diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/data_processing/__init__.py b/tests/data_processing/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/data_processing/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/data_processing/conftest.py b/tests/data_processing/conftest.py new file mode 100644 index 0000000..9eda2af --- /dev/null +++ b/tests/data_processing/conftest.py @@ -0,0 +1,53 @@ +"""Pytest configuration and shared fixtures.""" + +import pytest +import pandas as pd +from unittest.mock import MagicMock, patch +import sys +from dagster import build_op_context + +# Mock external dependencies that might not be available in test environment +sys.modules['spellchecker'] = MagicMock() + + +@pytest.fixture +def mock_context(): + """Create a mock Dagster context for testing operations.""" + context = build_op_context() + return context + + +@pytest.fixture +def sample_dataframe(): + """Create a sample DataFrame for testing.""" + return pd.DataFrame({ + 'Name': ['John Doe', 'jane smith', 'John Doe', 'bob johnson', 'John Doe'], + 'Age': [25, 30, 25, None, 25], + 'City': ['New York', 'los angeles', 'New York', 'chicago', 'New York'], + 'Status': ['Active', 'INACTIVE', 'Active', 'penDing', 'Active'] + }) + + +@pytest.fixture +def sample_dataframe_with_typos(): + """Create a sample DataFrame with typos for spell checking.""" + return pd.DataFrame({ + 'Name': ['jon doe', 'jane smith', 'bob jonson'], + 'Description': ['developer', 'analst', 'enginer'] + }) + + +@pytest.fixture +def empty_dataframe(): + """Create an empty DataFrame.""" + return pd.DataFrame() + + +@pytest.fixture +def dataframe_with_missing_values(): + """Create a DataFrame with various missing values.""" + return pd.DataFrame({ + 'Column1': [1, None, 3, None, 5], + 'Column2': ['a', 'b', None, 'd', None], + 'Column3': [None, None, None, None, None] + }) diff --git a/tests/data_processing/conftest_utils.py b/tests/data_processing/conftest_utils.py new file mode 100644 index 0000000..19d2f59 --- /dev/null +++ b/tests/data_processing/conftest_utils.py @@ -0,0 +1,7 @@ +"""Configuration utilities for testing.""" + +import os +import sys + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) diff --git a/tests/data_processing/test_config_models.py b/tests/data_processing/test_config_models.py new file mode 100644 index 0000000..989054f --- /dev/null +++ b/tests/data_processing/test_config_models.py @@ -0,0 +1,202 @@ +"""Unit tests for configuration models.""" + +import pytest +from pydantic import ValidationError +from template_code_location.data_processing.config_models import ( + FillMissingConfiguration, + ColumnsSelectConfiguration, + SpellCheckConfiguration, + AggregationConfiguration +) + + +class TestColumnsSelectConfiguration: + """Tests for ColumnsSelectConfiguration.""" + + def test_default_columns(self): + """Test default columns configuration.""" + config = ColumnsSelectConfiguration() + assert config.columns == ['Name'] + + def test_custom_columns(self): + """Test custom columns configuration.""" + config = ColumnsSelectConfiguration(columns=['Col1', 'Col2', 'Col3']) + assert config.columns == ['Col1', 'Col2', 'Col3'] + + def test_empty_columns_list(self): + """Test with empty columns list.""" + config = ColumnsSelectConfiguration(columns=[]) + assert config.columns == [] + + def test_single_column(self): + """Test with a single column.""" + config = ColumnsSelectConfiguration(columns=['SingleCol']) + assert config.columns == ['SingleCol'] + + def test_columns_with_special_characters(self): + """Test columns with special characters.""" + config = ColumnsSelectConfiguration(columns=['Col-1', 'Col_2', 'Col.3']) + assert config.columns == ['Col-1', 'Col_2', 'Col.3'] + + def test_duplicate_columns_are_removed(self): + """Verifica che i duplicati vengano rimossi mantenendo l'ordine (grazie a dict.fromkeys).""" + config = ColumnsSelectConfiguration(columns=['A', 'B', 'A', 'C', 'B']) + + assert config.columns == ['A', 'B', 'C'] + + def test_duplicate_default_behavior(self): + """Verifica che anche input estremi vengano gestiti correttamente.""" + config = ColumnsSelectConfiguration(columns=['Name', 'Name', 'Name']) + assert config.columns == ['Name'] + + +class TestFillMissingConfiguration: + """Tests for FillMissingConfiguration.""" + + def test_default_fill_map(self): + """Test default fill map configuration.""" + config = FillMissingConfiguration() + + assert config.fill_map == {'Age': 'UNKNOWN_AGE'} + + def test_custom_fill_map(self): + """Test custom fill map configuration.""" + fill_map = {'Age': '0', 'Name': 'UNKNOWN', 'City': 'N/A'} + config = FillMissingConfiguration(fill_map=fill_map) + + assert config.fill_map == fill_map + + def test_empty_fill_map(self): + """Test with empty fill map.""" + config = FillMissingConfiguration(fill_map={}) + + assert config.fill_map == {} + + def test_fill_map_with_numeric_values(self): + """Test fill map with numeric string values.""" + fill_map = {'Age': '0', 'Score': '-1', 'Count': '999'} + config = FillMissingConfiguration(fill_map=fill_map) + + assert config.fill_map == fill_map + + def test_fill_map_with_string_values(self): + """Test fill map with string values.""" + fill_map = {'Name': 'Unknown', 'Email': 'no-email'} + config = FillMissingConfiguration(fill_map=fill_map) + + assert config.fill_map == fill_map + + def test_fill_map_mixed_types(self): + """Test fill map with mixed value types (all strings).""" + fill_map = {'IntCol': '0', 'StrCol': 'Unknown', 'FloatCol': '0.0'} + config = FillMissingConfiguration(fill_map=fill_map) + + assert config.fill_map == fill_map + + +class TestSpellCheckConfiguration: + """Tests for SpellCheckConfiguration.""" + + def test_default_spell_check_config(self): + """Test default spell check configuration.""" + config = SpellCheckConfiguration() + + assert config.columns == ['Name'] + assert config.language == 'en' + + def test_custom_spell_check_config(self): + """Test custom spell check configuration.""" + config = SpellCheckConfiguration( + columns=['Description', 'Notes'], + language='es' + ) + + assert config.columns == ['Description', 'Notes'] + assert config.language == 'es' + + def test_spell_check_all_languages(self): + """Test spell check with all supported languages.""" + supported_languages = ['en', 'es', 'it', 'fr', 'pt', 'de', 'nl'] + + for lang in supported_languages: + config = SpellCheckConfiguration(language=lang) + assert config.language == lang + + def test_spell_check_invalid_language(self): + """Test spell check with invalid language.""" + with pytest.raises(ValidationError): + SpellCheckConfiguration(language='invalid') + + def test_spell_check_multiple_columns(self): + """Test spell check with multiple columns.""" + columns = ['Col1', 'Col2', 'Col3', 'Col4'] + config = SpellCheckConfiguration(columns=columns) + + assert config.columns == columns + + def test_spell_check_empty_columns(self): + """Test spell check with empty columns list.""" + config = SpellCheckConfiguration(columns=[]) + + assert config.columns == [] + assert config.language == 'en' + + def test_spell_check_inheritance(self): + """Test that SpellCheckConfiguration inherits from ColumnsSelectConfiguration.""" + config = SpellCheckConfiguration() + + assert isinstance(config, ColumnsSelectConfiguration) + assert hasattr(config, 'columns') + assert hasattr(config, 'language') + + @pytest.mark.parametrize("language", ['en', 'es', 'it', 'fr', 'pt', 'de', 'nl']) + def test_spell_check_languages_parametrized(self, language): + """Test spell check with parametrized languages.""" + config = SpellCheckConfiguration(language=language) + assert config.language == language + +class TestAggregationConfiguration: + """Tests for AggregationConfiguration.""" + + def test_aggregation_default_config(self): + """Test default aggregation configuration.""" + config = AggregationConfiguration() + + assert config.columns == ['Name'] + assert config.operation == 'sum' + + @pytest.mark.parametrize("op", ["sum", "mean", "min", "max", "count"]) + def test_aggregation_valid_operations(self, op): + """Test all allowed aggregation operations.""" + config = AggregationConfiguration(operation=op) + assert config.operation == op + + def test_aggregation_invalid_operation(self): + """Test that an invalid operation raises a ValidationError.""" + with pytest.raises(ValidationError) as excinfo: + AggregationConfiguration(operation="invalid_op") + + assert "Invalid aggregation operation 'invalid_op'" in str(excinfo.value) + + def test_aggregation_custom_columns(self): + """Test aggregation with custom columns.""" + config = AggregationConfiguration(columns=['Price', 'Quantity'], operation='mean') + + assert config.columns == ['Price', 'Quantity'] + assert config.operation == 'mean' + + def test_aggregation_inheritance(self): + """Test that AggregationConfiguration inherits from ColumnsSelectConfiguration.""" + config = AggregationConfiguration() + + assert isinstance(config, ColumnsSelectConfiguration) + assert hasattr(config, 'columns') + assert hasattr(config, 'operation') + + def test_aggregation_model_dump(self): + """Test that model_dump contains all expected fields (useful for the Dagster op).""" + config = AggregationConfiguration(columns=['Value'], operation='max') + dump = config.model_dump() + + assert dump['columns'] == ['Value'] + assert dump['operation'] == 'max' diff --git a/tests/data_processing/test_integration.py b/tests/data_processing/test_integration.py new file mode 100644 index 0000000..c9d01eb --- /dev/null +++ b/tests/data_processing/test_integration.py @@ -0,0 +1,185 @@ +"""Integration tests for data processing jobs.""" + +import pytest +import pandas as pd +from unittest.mock import patch, MagicMock +from template_code_location.data_processing.ops import ( + remove_duplicates, + fill_missing_values, + standardize_categorical_values, + correct_typos +) +from template_code_location.data_processing.config_models import ( + FillMissingConfiguration, + ColumnsSelectConfiguration, + SpellCheckConfiguration +) + + +class TestPipelineIntegration: + """Integration tests for data processing pipeline.""" + + def test_pipeline_remove_duplicates_then_standardize(self, mock_context): + """Test pipeline: remove duplicates then standardize.""" + df = pd.DataFrame({ + 'Name': [' JOHN DOE ', 'jane smith', ' JOHN DOE ', 'bob johnson'], + 'City': ['NEW YORK', 'los angeles', 'NEW YORK', 'chicago'] + }) + + # Step 1: Remove duplicates + df_no_dupes = remove_duplicates(mock_context, df) + assert df_no_dupes.shape[0] == 3 + + # Step 2: Standardize + config = ColumnsSelectConfiguration(columns=['Name', 'City']) + df_standardized = standardize_categorical_values(mock_context, config, df_no_dupes) + + assert df_standardized['Name'].iloc[0] == 'john doe' + assert df_standardized['City'].iloc[0] == 'new york' + + def test_pipeline_fill_missing_then_standardize(self, mock_context): + """Test pipeline: fill missing values then standardize.""" + df = pd.DataFrame({ + 'Category': [' ACTIVE ', None, ' PENDING '], + 'Value': ['1', '2', None] + }) + + # Step 1: Fill missing values + fill_config = FillMissingConfiguration(fill_map={'Value': '0'}) + df_filled = fill_missing_values(mock_context, fill_config, df) + + # Step 2: Standardize + std_config = ColumnsSelectConfiguration(columns=['Category']) + df_standardized = standardize_categorical_values(mock_context, std_config, df_filled) + + assert df_standardized['Category'].iloc[0] == 'active' + assert df_filled['Value'].iloc[2] == '0' + + def test_pipeline_all_operations(self, mock_context): + """Test complete pipeline with all operations.""" + df = pd.DataFrame({ + 'Name': [' john doe ', 'JANE SMITH', ' john doe ', None], + 'Value': ['1', None, '1', '2'] + }) + + # Step 1: Remove duplicates + df = remove_duplicates(mock_context, df) + assert df.shape[0] == 3 + + # Step 2: Fill missing + fill_config = FillMissingConfiguration(fill_map={'Value': '0'}) + df = fill_missing_values(mock_context, fill_config, df) + assert df['Value'].isna().sum() == 0 + + # Step 3: Standardize + std_config = ColumnsSelectConfiguration(columns=['Name']) + df = standardize_categorical_values(mock_context, std_config, df) + + assert df['Name'].iloc[0] == 'john doe' + + def test_pipeline_with_large_dataset(self, mock_context): + """Test pipeline performance with larger dataset.""" + # Create larger dataset + size = 1000 + df = pd.DataFrame({ + 'ID': list(range(size)), + 'Name': ['User_' + str(i % 50) for i in range(size)], + 'Status': ['ACTIVE', 'INACTIVE', 'PENDING'] * (size // 3) + ['ACTIVE'] * (size % 3), + 'Score': [i % 100 for i in range(size)] + }) + + # Add some duplicates + df = pd.concat([df, df.head(100)], ignore_index=True) + + # Process + df_cleaned = remove_duplicates(mock_context, df) + + assert df_cleaned.shape[0] == 1000 + assert df_cleaned.shape[1] == 4 + + +class TestErrorHandling: + """Tests for error handling and edge cases.""" + + def test_operation_with_corrupted_data(self, mock_context): + """Test operations with corrupted/unusual data.""" + df = pd.DataFrame({ + 'Col': [float('nan'), float('inf'), -float('inf'), 0, 1, 2] + }) + + # Should handle special float values + result = remove_duplicates(mock_context, df) + assert result.shape[0] > 0 + + def test_operation_preserves_index(self, mock_context): + """Test that index is handled correctly.""" + df = pd.DataFrame( + {'Col': [1, 2, 1, 3]}, + index=['a', 'b', 'c', 'd'] + ) + + result = remove_duplicates(mock_context, df) + # Index may be reset, so just check shape + assert result.shape[0] == 3 + + def test_standardize_with_unicode_characters(self, mock_context): + """Test standardization with unicode characters.""" + df = pd.DataFrame({ + 'Name': ['José', 'François', 'Müller'] + }) + + config = ColumnsSelectConfiguration(columns=['Name']) + result = standardize_categorical_values(mock_context, config, df) + + # Should handle unicode correctly + assert result.shape[0] == 3 + + def test_fill_with_same_key_multiple_times(self, mock_context): + """Test filling when fill_map has multiple entries.""" + df = pd.DataFrame({ + 'A': ['1', None, '3'], + 'B': [None, None, 'c'], + 'C': [None, '2', None] + }) + + config = FillMissingConfiguration(fill_map={ + 'A': '-1', + 'B': 'EMPTY', + 'C': '0' + }) + + result = fill_missing_values(mock_context, config, df) + + assert result.loc[1, 'A'] == '-1' + assert result.loc[0, 'B'] == 'EMPTY' + assert result.loc[0, 'C'] == '0' + + +class TestDataTypePreservation: + """Tests to ensure data types are preserved appropriately.""" + + def test_remove_duplicates_preserves_dtypes(self, mock_context): + """Test that remove_duplicates preserves column data types.""" + df = pd.DataFrame({ + 'int32': pd.array([1, 2, 1], dtype='int32'), + 'float64': pd.array([1.5, 2.5, 1.5], dtype='float64'), + 'str': ['a', 'b', 'a'] + }) + + result = remove_duplicates(mock_context, df) + + assert result['int32'].dtype == df['int32'].dtype + assert result['float64'].dtype == df['float64'].dtype + + def test_fill_missing_preserves_column_types_where_possible(self, mock_context): + """Test that fill_missing handles type preservation.""" + df = pd.DataFrame({ + 'A': pd.array(['1', None, '3'], dtype='string'), + 'B': ['x', 'y', 'z'] + }) + + config = FillMissingConfiguration(fill_map={'A': '0'}) + result = fill_missing_values(mock_context, config, df) + + assert result['A'].loc[1] == '0' + assert result['B'].dtype == df['B'].dtype diff --git a/tests/data_processing/test_jobs.py b/tests/data_processing/test_jobs.py new file mode 100644 index 0000000..5373f7c --- /dev/null +++ b/tests/data_processing/test_jobs.py @@ -0,0 +1,56 @@ +from template_code_location.data_processing.jobs import ( + remove_duplicates_job_s3, + fill_missing_values_job_s3, + standardize_categorical_values_job_s3, + correct_typos_job_s3, + normalize_numeric_min_max_job_s3, + normalize_datetime_job_s3, + normalize_coordinates_job_s3, + add_global_aggregations_job_s3 +) + + +def test_remove_duplicates_job_s3_is_callable(): + """Test remove_duplicates_job_s3 is a valid Dagster job""" + assert callable(remove_duplicates_job_s3) + assert hasattr(remove_duplicates_job_s3, 'execute_in_process') + + +def test_fill_missing_values_job_s3_is_callable(): + """Test fill_missing_values_job_s3 is a valid Dagster job""" + assert callable(fill_missing_values_job_s3) + assert hasattr(fill_missing_values_job_s3, 'execute_in_process') + + +def test_standardize_categorical_values_job_s3_is_callable(): + """Test standardize_categorical_values_job_s3 is a valid Dagster job""" + assert callable(standardize_categorical_values_job_s3) + assert hasattr(standardize_categorical_values_job_s3, 'execute_in_process') + + +def test_correct_typos_job_s3_is_callable(): + """Test correct_typos_job_s3 is a valid Dagster job""" + assert callable(correct_typos_job_s3) + assert hasattr(correct_typos_job_s3, 'execute_in_process') + + +def test_normalize_numeric_min_max_job_s3_is_callable(): + """Test normalize_numeric_min_max_job_s3 is a valid Dagster job""" + assert callable(normalize_numeric_min_max_job_s3) + assert hasattr(normalize_numeric_min_max_job_s3, 'execute_in_process') + + +def test_normalize_datetime_job_s3_is_callable(): + """Test normalize_datetime_job_s3 is a valid Dagster job""" + assert callable(normalize_datetime_job_s3) + assert hasattr(normalize_datetime_job_s3, 'execute_in_process') + +def test_normalize_coordinates_job_s3_is_callable(): + """Test normalize_coordinates_job_s3 is a valid Dagster job""" + assert callable(normalize_coordinates_job_s3) + assert hasattr(normalize_coordinates_job_s3, 'execute_in_process') + +def test_add_global_aggregations_job_s3_is_callable(): + """Test add_global_aggregations_job_s3 is a valid Dagster job""" + assert callable(add_global_aggregations_job_s3) + assert hasattr(add_global_aggregations_job_s3, 'execute_in_process') diff --git a/tests/data_processing/test_ops.py b/tests/data_processing/test_ops.py new file mode 100644 index 0000000..def913b --- /dev/null +++ b/tests/data_processing/test_ops.py @@ -0,0 +1,700 @@ +"""Unit tests for data processing operations.""" + +import pytest +import pandas as pd +from template_code_location.data_processing.ops import ( + remove_duplicates, + fill_missing_values, + standardize_categorical_values, + correct_typos, + normalize_datetime, + normalize_numeric_min_max, + normalize_coordinates, + add_global_aggregations +) +from template_code_location.data_processing.config_models import ( + FillMissingConfiguration, + ColumnsSelectConfiguration, + SpellCheckConfiguration, + AggregationConfiguration, + CoordinatesNormalizationConfiguration +) + + +class TestRemoveDuplicates: + """Tests for the remove_duplicates operation.""" + + def test_remove_duplicates_basic(self, mock_context, sample_dataframe): + """Test basic duplicate removal.""" + result = remove_duplicates(mock_context, sample_dataframe) + + # Should have 3 unique rows (john doe appears 3x, jane smith 1x, bob johnson 1x) + assert result.shape[0] == 3 + assert len(result) < len(sample_dataframe) + + def test_remove_duplicates_no_duplicates(self, mock_context): + """Test remove_duplicates when there are no duplicates.""" + df = pd.DataFrame({ + 'A': [1, 2, 3], + 'B': ['x', 'y', 'z'] + }) + result = remove_duplicates(mock_context, df) + + assert result.shape[0] == 3 + pd.testing.assert_frame_equal(result, df) + + def test_remove_duplicates_all_duplicates(self, mock_context): + """Test remove_duplicates when all rows are identical.""" + df = pd.DataFrame({ + 'A': [1, 1, 1], + 'B': ['x', 'x', 'x'] + }) + result = remove_duplicates(mock_context, df) + + assert result.shape[0] == 1 + + def test_remove_duplicates_empty_dataframe(self, mock_context, empty_dataframe): + """Test remove_duplicates with empty DataFrame.""" + result = remove_duplicates(mock_context, empty_dataframe) + + assert result.shape[0] == 0 + assert result.shape[1] == 0 + + def test_remove_duplicates_preserves_data_types(self, mock_context): + """Test that remove_duplicates preserves data types.""" + df = pd.DataFrame({ + 'int_col': [1, 2, 1], + 'str_col': ['a', 'b', 'a'], + 'float_col': [1.5, 2.5, 1.5] + }) + result = remove_duplicates(mock_context, df) + + assert result['int_col'].dtype == df['int_col'].dtype + assert result['str_col'].dtype == df['str_col'].dtype + assert result['float_col'].dtype == df['float_col'].dtype + + +class TestFillMissingValues: + """Tests for the fill_missing_values operation.""" + + def test_fill_missing_values_basic(self, mock_context, dataframe_with_missing_values): + """Test basic missing value filling.""" + config = FillMissingConfiguration(fill_map={'Column1': '0', 'Column2': 'N/A'}) + result = fill_missing_values(mock_context, config, dataframe_with_missing_values) + + # Check that no NaN values remain + assert result['Column1'].isna().sum() == 0 + assert result['Column2'].isna().sum() == 0 + + def test_fill_missing_values_with_different_values(self, mock_context): + """Test filling with different replacement values.""" + df = pd.DataFrame({ + 'A': [1, None, 3], + 'B': [None, 'b', 'c'] + }) + config = FillMissingConfiguration(fill_map={'A': '-1', 'B': 'UNKNOWN'}) + result = fill_missing_values(mock_context, config, df) + + assert result.loc[1, 'A'] == '-1' + assert result.loc[0, 'B'] == 'UNKNOWN' + + def test_fill_missing_values_partial_columns(self, mock_context): + """Test filling only specified columns.""" + df = pd.DataFrame({ + 'A': [1, None, 3], + 'B': [None, 'b', 'c'] + }) + config = FillMissingConfiguration(fill_map={'A': '999'}) + result = fill_missing_values(mock_context, config, df) + + assert result.loc[1, 'A'] == '999' + assert pd.isna(result.loc[0, 'B']) # B should still have NaN + + def test_fill_missing_values_no_missing(self, mock_context): + """Test when there are no missing values.""" + df = pd.DataFrame({ + 'A': ['1', '2', '3'], + 'B': ['a', 'b', 'c'] + }) + config = FillMissingConfiguration(fill_map={'A': '0'}) + result = fill_missing_values(mock_context, config, df) + + pd.testing.assert_frame_equal(result, df) + + def test_fill_missing_values_empty_dataframe(self, mock_context, empty_dataframe): + """Test with empty DataFrame.""" + config = FillMissingConfiguration(fill_map={}) + result = fill_missing_values(mock_context, config, empty_dataframe) + + assert result.shape[0] == 0 + + +class TestStandardizeCategoricalValues: + """Tests for the standardize_categorical_values operation.""" + + def test_standardize_categorical_basic(self, mock_context, sample_dataframe): + """Test basic categorical standardization.""" + config = ColumnsSelectConfiguration(columns=['Name', 'City', 'Status']) + result = standardize_categorical_values(mock_context, config, sample_dataframe) + + # Check that values are lowercase and stripped + assert result['Name'].iloc[0] == 'john doe' + assert result['City'].iloc[1] == 'los angeles' + assert result['Status'].iloc[1] == 'inactive' + + def test_standardize_categorical_single_column(self, mock_context): + """Test standardization on a single column.""" + df = pd.DataFrame({ + 'City': [' NEW YORK ', 'LOS ANGELES', ' chicago '] + }) + config = ColumnsSelectConfiguration(columns=['City']) + result = standardize_categorical_values(mock_context, config, df) + + assert result['City'].iloc[0] == 'new york' + assert result['City'].iloc[1] == 'los angeles' + assert result['City'].iloc[2] == 'chicago' + + def test_standardize_categorical_missing_column(self, mock_context, sample_dataframe): + """Test with non-existent column (should skip).""" + config = ColumnsSelectConfiguration(columns=['NonExistent', 'Name']) + result = standardize_categorical_values(mock_context, config, sample_dataframe) + + # Should process 'Name' column without error + assert result['Name'].iloc[0] == 'john doe' + + def test_standardize_categorical_with_missing_values(self, mock_context): + """Test standardization with missing values.""" + df = pd.DataFrame({ + 'Category': [' ACTIVE ', None, ' pending '] + }) + config = ColumnsSelectConfiguration(columns=['Category']) + result = standardize_categorical_values(mock_context, config, df) + + assert result['Category'].iloc[0] == 'active' + assert result['Category'].iloc[1] == '' + assert result['Category'].iloc[2] == 'pending' + + def test_standardize_categorical_empty_dataframe(self, mock_context, empty_dataframe): + """Test with empty DataFrame.""" + config = ColumnsSelectConfiguration(columns=['A', 'B']) + result = standardize_categorical_values(mock_context, config, empty_dataframe) + + assert result.shape[0] == 0 + + def test_standardize_categorical_numeric_columns(self, mock_context): + """Test that numeric columns are converted to strings.""" + df = pd.DataFrame({ + 'NumCol': [1, 2, 3] + }) + config = ColumnsSelectConfiguration(columns=['NumCol']) + result = standardize_categorical_values(mock_context, config, df) + + assert result['NumCol'].iloc[0] == '1' + assert isinstance(result['NumCol'].iloc[0], str) + + +class TestCorrectTypos: + """Tests for the correct_typos operation.""" + + def test_correct_typos_basic(self, mock_context): + """Test basic typo correction.""" + df = pd.DataFrame({ + 'Name': ['jon', 'jayne', 'bob'] + }) + config = SpellCheckConfiguration(columns=['Name'], language='en') + result = correct_typos(mock_context, config, df) + + # Result should have corrections applied + assert result.shape[0] == 3 + + def test_correct_typos_missing_column(self, mock_context): + """Test with non-existent column (should skip).""" + df = pd.DataFrame({ + 'Name': ['jon', 'jayne'] + }) + config = SpellCheckConfiguration(columns=['NonExistent'], language='en') + result = correct_typos(mock_context, config, df) + + # Should not raise error, just skip + pd.testing.assert_frame_equal(result, df) + + def test_correct_typos_with_missing_values(self, mock_context): + """Test typo correction with missing values.""" + df = pd.DataFrame({ + 'Text': ['helo', '', 'wrld'] + }) + config = SpellCheckConfiguration(columns=['Text'], language='en') + result = correct_typos(mock_context, config, df) + + # Empty strings should be preserved + assert result.loc[1, 'Text'] == '' + + def test_correct_typos_empty_dataframe(self, mock_context, empty_dataframe): + """Test with empty DataFrame.""" + config = SpellCheckConfiguration(columns=['A'], language='en') + result = correct_typos(mock_context, config, empty_dataframe) + + assert result.shape[0] == 0 + + def test_correct_typos_different_languages(self, mock_context): + """Test typo correction with different languages.""" + df = pd.DataFrame({ + 'Text': ['ciao', 'mondo'] + }) + + for lang in ['en', 'es', 'it']: + config = SpellCheckConfiguration(columns=['Text'], language=lang) + result = correct_typos(mock_context, config, df) + + # Should process without error + assert result.shape[0] == 2 + + def test_correct_typos_numeric_values(self, mock_context): + """Test typo correction on numeric values converted to strings.""" + df = pd.DataFrame({ + 'Values': [123, 456, 789] + }) + config = SpellCheckConfiguration(columns=['Values'], language='en') + result = correct_typos(mock_context, config, df) + + # Numeric values should be converted to string and processed + assert result.shape[0] == 3 + +class TestNormalizeDatetime: + """Tests for the normalize_datetime operation.""" + + def test_normalize_datetime_basic(self, mock_context): + """Test basic datetime normalization to ISO format.""" + df = pd.DataFrame({ + 'date_col': ['2023-01-01 10:00:00', '2023-12-31T23:59:59'] + }) + + config = ColumnsSelectConfiguration(columns=['date_col']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'date_col_iso' in result.columns + assert result['date_col_iso'].iloc[0] == '2023-01-01T10:00:00Z' + assert result['date_col_iso'].iloc[1] == '2023-12-31T23:59:59Z' + + def test_normalize_datetime_missing_column(self, mock_context, sample_dataframe): + """Test behavior when a configured column is missing in the DataFrame.""" + config = ColumnsSelectConfiguration(columns=['non_existent_column']) + + result = normalize_datetime(mock_context, config, sample_dataframe.copy()) + + pd.testing.assert_frame_equal(result, sample_dataframe) + + def test_normalize_datetime_unparseable_values(self, mock_context): + """Test column with values that cannot be parsed as dates.""" + df = pd.DataFrame({ + 'invalid_col': ['not-a-date', 'completely-random-text'] + }) + config = ColumnsSelectConfiguration(columns=['invalid_col']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'invalid_col_iso' not in result.columns + + def test_normalize_datetime_mixed_and_nulls(self, mock_context): + """Test column with mixed valid dates, invalid dates, and NaNs.""" + df = pd.DataFrame({ + 'mixed_col': ['2023-05-01', None, 'invalid-date'] + }) + config = ColumnsSelectConfiguration(columns=['mixed_col']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'mixed_col_iso' in result.columns + assert result['mixed_col_iso'].iloc[0] == '2023-05-01T00:00:00Z' + + assert result['mixed_col_iso'].iloc[1] == "" + assert result['mixed_col_iso'].iloc[2] == "" + + def test_normalize_datetime_empty_dataframe(self, mock_context, empty_dataframe): + """Test with an empty DataFrame.""" + config = ColumnsSelectConfiguration(columns=['some_col']) + + result = normalize_datetime(mock_context, config, empty_dataframe) + + assert result.empty + + def test_normalize_datetime_epoch_only(self, mock_context, capsys): + """If parsing a column yields only the Unix epoch date, it should be skipped.""" + df = pd.DataFrame({ + 'weird_col': ['0', 0, '0000', ''] + }) + + config = ColumnsSelectConfiguration(columns=['weird_col']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'weird_col_iso' not in result.columns + + captured = capsys.readouterr() + assert "all normalized values are '1970-01-01'" in captured.err + + def test_normalize_datetime_all_1970_skipped(self, mock_context, capsys): + """If all formatted values are '1970-01-01', the column should be skipped with a warning.""" + df = pd.DataFrame({ + 'ts_col': ['1970-01-01 05:30:00', '1970-01-01 12:00:00'] + }) + + config = ColumnsSelectConfiguration(columns=['ts_col']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'ts_col_iso' not in result.columns + + captured = capsys.readouterr() + assert "all normalized values are '1970-01-01'" in captured.err + + def test_normalize_datetime_integer_age_column_skipped(self, mock_context, capsys): + """If an integer column like 'age' is passed, all values become 1970-01-01 and should be skipped.""" + df = pd.DataFrame({ + 'age': [66, 45, 40, 43, 20, 26, 69, 21, 46] + }) + + config = ColumnsSelectConfiguration(columns=['age']) + + result = normalize_datetime(mock_context, config, df.copy()) + + assert 'age_iso' not in result.columns + + captured = capsys.readouterr() + assert "all normalized values are '1970-01-01'" in captured.err + +class TestNormalizeNumericMinMax: + """Tests for the normalize_numeric_min_max operation.""" + + def test_normalize_numeric_basic(self, mock_context): + """Test standard min-max normalization between 0 and 1.""" + df = pd.DataFrame({ + 'score': [10, 20, 30, 40, 50] + }) + config = ColumnsSelectConfiguration(columns=['score']) + + result = normalize_numeric_min_max(mock_context, config, df.copy()) + + assert 'score_norm' in result.columns + assert result['score_norm'].min() == 0.0 + assert result['score_norm'].max() == 1.0 + + assert result['score_norm'].iloc[2] == 0.5 + + def test_normalize_numeric_missing_column(self, mock_context): + """Test skipping of non-existent columns.""" + df = pd.DataFrame({'existing': [1, 2, 3]}) + config = ColumnsSelectConfiguration(columns=['missing_col']) + + result = normalize_numeric_min_max(mock_context, config, df.copy()) + + assert 'missing_col_norm' not in result.columns + + def test_normalize_numeric_constant_values(self, mock_context): + """Test skipping when min == max to avoid division by zero.""" + df = pd.DataFrame({ + 'constant': [10, 10, 10] + }) + config = ColumnsSelectConfiguration(columns=['constant']) + + result = normalize_numeric_min_max(mock_context, config, df.copy()) + + assert 'constant_norm' not in result.columns + + def test_normalize_numeric_with_nans(self, mock_context): + """Test normalization with NaN values (pandas min/max ignore NaNs by default).""" + df = pd.DataFrame({ + 'with_nans': [10, None, 50] + }) + config = ColumnsSelectConfiguration(columns=['with_nans']) + + result = normalize_numeric_min_max(mock_context, config, df.copy()) + + assert 'with_nans_norm' in result.columns + assert result['with_nans_norm'].iloc[0] == 0.0 + assert result['with_nans_norm'].iloc[2] == 1.0 + assert pd.isna(result['with_nans_norm'].iloc[1]) + + def test_normalize_numeric_multiple_columns(self, mock_context): + """Test processing multiple columns in one call.""" + df = pd.DataFrame({ + 'A': [1, 2], + 'B': [10, 20] + }) + config = ColumnsSelectConfiguration(columns=['A', 'B']) + + result = normalize_numeric_min_max(mock_context, config, df.copy()) + + assert 'A_norm' in result.columns + assert 'B_norm' in result.columns + +class TestNormalizeCoordinates: + """Tests for the normalize_coordinates operation.""" + + def test_normalize_coordinates_basic(self, mock_context): + """Test rounding and basic coordinate normalization.""" + df = pd.DataFrame({ + 'lat': [45.123456, 46.0], + 'lon': [9.123456, 10.0] + }) + config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon') + + result = normalize_coordinates(mock_context, config, df.copy()) + + assert result['lat'].iloc[0] == 45.1235 + assert result['lon'].iloc[0] == 9.1235 + + assert len(result) == 2 + + def test_normalize_coordinates_filtering(self, mock_context): + """Test filtering of out-of-range coordinates.""" + df = pd.DataFrame({ + 'lat': [45.0, 100.0, -91.0, 0.0], # 100 e -91 sono out of range + 'lon': [9.0, 0.0, 0.0, 200.0] # 200 è out of range + }) + config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon') + + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 1 + assert result['lat'].iloc[0] == 45.0 + + def test_normalize_coordinates_invalid_types(self, mock_context): + """Test conversion of strings to numeric and handling of NaNs.""" + df = pd.DataFrame({ + 'lat': ["45.5", "invalid", None], + 'lon': ["9.5", "10.0", "11.0"] + }) + config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon') + + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 1 + assert isinstance(result['lat'].iloc[0], float) + + def test_normalize_coordinates_empty_df(self, mock_context, empty_dataframe): + """Test with an empty DataFrame.""" + + df = pd.DataFrame(columns=['lat', 'lon']) + config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon') + + result = normalize_coordinates(mock_context, config, df) + + assert len(result) == 0 + assert result.empty + + def test_normalize_coordinates_default_config(self, mock_context): + """Test that normalize_coordinates uses default 'lat'/'lon' columns when no config is provided.""" + df = pd.DataFrame({ + 'lat': [45.123456, 46.0], + 'lon': [9.123456, 10.0] + }) + config = CoordinatesNormalizationConfiguration() + + result = normalize_coordinates(mock_context, config, df.copy()) + + assert result['lat'].iloc[0] == 45.1235 + assert result['lon'].iloc[0] == 9.1235 + assert len(result) == 2 + + def test_normalize_coordinates_null_config_values(self, mock_context): + """Test that null lat/lon column names fall back to defaults ('lat'/'lon').""" + df = pd.DataFrame({ + 'lat': [45.123456, 46.0], + 'lon': [9.123456, 10.0] + }) + config = CoordinatesNormalizationConfiguration(latColumn=None, lonColumn=None) + + assert config.latColumn == "lat" + assert config.lonColumn == "lon" + + result = normalize_coordinates(mock_context, config, df.copy()) + + assert result['lat'].iloc[0] == 45.1235 + assert result['lon'].iloc[0] == 9.1235 + assert len(result) == 2 + + def test_normalize_coordinates_dms_degree_symbol(self, mock_context): + """Test DMS parsing with degree/minute/second symbols like 40°26'46\"N.""" + df = pd.DataFrame({ + 'lat': ["40°26'46\"N", "51°30'26\"N"], + 'lon': ["79°58'56\"W", "0°7'39\"W"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 2 + # 40°26'46"N ≈ 40.4461 + assert abs(result['lat'].iloc[0] - 40.4461) < 0.001 + # 79°58'56"W ≈ -79.9822 + assert abs(result['lon'].iloc[0] - (-79.9822)) < 0.001 + + def test_normalize_coordinates_dms_spaced_format(self, mock_context): + """Test DMS parsing with space-separated format like '40 26 46 N'.""" + df = pd.DataFrame({ + 'lat': ["40 26 46 N"], + 'lon': ["79 58 56 W"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 1 + assert abs(result['lat'].iloc[0] - 40.4461) < 0.001 + assert abs(result['lon'].iloc[0] - (-79.9822)) < 0.001 + + def test_normalize_coordinates_dms_already_decimal(self, mock_context): + """Test that string columns with decimal values are auto-parsed correctly.""" + df = pd.DataFrame({ + 'lat': ["45.5", "46.0"], + 'lon': ["9.5", "10.0"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 2 + assert result['lat'].iloc[0] == 45.5 + assert result['lon'].iloc[0] == 9.5 + + def test_normalize_coordinates_dms_mixed_valid_invalid(self, mock_context): + """Test auto-detection with a mix of valid DMS, valid decimal, and unparseable values.""" + df = pd.DataFrame({ + 'lat': ["40°26'46\"N", "not_a_coord", "51.5"], + 'lon': ["79°58'56\"W", "10.0", "0.1"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + # Row with "not_a_coord" for lat should be dropped (NaN lat) + assert len(result) == 2 + + def test_normalize_coordinates_dms_out_of_range(self, mock_context): + """Test that DMS-parsed coordinates outside valid range are filtered out.""" + df = pd.DataFrame({ + 'lat': ["91°0'0\"N", "45°0'0\"N"], + 'lon': ["0°0'0\"E", "9°0'0\"E"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + # First row has lat=91° which is out of [-90, 90] + assert len(result) == 1 + assert abs(result['lat'].iloc[0] - 45.0) < 0.001 + + def test_normalize_coordinates_dms_south_and_east(self, mock_context): + """Test DMS parsing with south latitude and east longitude.""" + df = pd.DataFrame({ + 'lat': ["33°51'54\"S"], + 'lon': ["151°12'36\"E"] + }) + config = CoordinatesNormalizationConfiguration( + latColumn='lat', lonColumn='lon' + ) + result = normalize_coordinates(mock_context, config, df.copy()) + + assert len(result) == 1 + # 33°51'54"S ≈ -33.865 + assert result['lat'].iloc[0] < 0 + assert abs(result['lat'].iloc[0] - (-33.865)) < 0.001 + # 151°12'36"E ≈ 151.21 + assert result['lon'].iloc[0] > 0 + assert abs(result['lon'].iloc[0] - 151.21) < 0.01 + + def test_normalize_coordinates_autodetect_numeric_vs_dms(self, mock_context): + """Test that numeric columns are coerced directly while string columns are parsed as DMS.""" + # Numeric columns — should go through pd.to_numeric path + df_numeric = pd.DataFrame({ + 'lat': [45.123456, 46.0], + 'lon': [9.123456, 10.0] + }) + config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon') + result_numeric = normalize_coordinates(mock_context, config, df_numeric.copy()) + + assert result_numeric['lat'].iloc[0] == 45.1235 + assert len(result_numeric) == 2 + + # String DMS columns — should go through _parse_dms_to_decimal path + df_dms = pd.DataFrame({ + 'lat': ["40°26'46\"N"], + 'lon': ["79°58'56\"W"] + }) + result_dms = normalize_coordinates(mock_context, config, df_dms.copy()) + + assert len(result_dms) == 1 + assert abs(result_dms['lat'].iloc[0] - 40.4461) < 0.001 + +class TestAddGlobalAggregations: + """Tests for the add_global_aggregations operation.""" + + def test_add_global_aggregations_success(self, mock_context): + """Test a successful group by and aggregation.""" + df = pd.DataFrame({ + 'category': ['A', 'A', 'B'], + 'value': [10, 20, 100], + 'ignored_str': ['x', 'y', 'z'] + }) + + config = AggregationConfiguration( + columns=['category'], + operation='sum' + ) + + result = add_global_aggregations(mock_context, config, df.copy()) + + assert len(result) == 2 + assert result.loc[result['category'] == 'A', 'value'].values[0] == 30 + assert result.loc[result['category'] == 'B', 'value'].values[0] == 100 + assert 'ignored_str' not in result.columns + mock_context.log.info.assert_called() + + def test_add_global_aggregations_missing_column(self, mock_context): + """Test skipping a column that does not exist in the dataframe.""" + df = pd.DataFrame({'value': [1, 2, 3]}) + config = AggregationConfiguration( + columns=['missing_col'], + operation='count' + ) + + result = add_global_aggregations(mock_context, config, df.copy()) + + mock_context.log.warning.assert_any_call("Column 'missing_col' not found, skipping aggregation.") + assert len(result) == 1 + + def test_add_global_aggregations_unsupported_op(self, mock_context): + """Test the warning when an unsupported operation is provided.""" + df = pd.DataFrame({'category': ['A'], 'value': [1]}) + + config = AggregationConfiguration( + columns=['category'], + operation='unsupported' + ) + + with pytest.raises(Exception): + add_global_aggregations(mock_context, config, df.copy()) + + mock_context.log.warning.assert_any_call("Unsupported aggregation 'unsupported'") + + def test_add_global_aggregations_only_numeric_kept(self, mock_context): + """Verify that non-numeric and non-grouping columns are dropped.""" + df = pd.DataFrame({ + 'group': ['A', 'A'], + 'num': [1, 2], + 'text': ['hello', 'world'] + }) + config = AggregationConfiguration(columns=['group'], operation='mean') + + result = add_global_aggregations(mock_context, config, df.copy()) + + assert 'text' not in result.columns + assert 'num' in result.columns + assert 'group' in result.columns diff --git a/tests/dataframe_level_anonymisation/__init__.py b/tests/dataframe_level_anonymisation/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/dataframe_level_anonymisation/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/dataframe_level_anonymisation/config_models/__init__.py b/tests/dataframe_level_anonymisation/config_models/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/dataframe_level_anonymisation/config_models/test_base_config.py b/tests/dataframe_level_anonymisation/config_models/test_base_config.py new file mode 100644 index 0000000..92e599b --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/test_base_config.py @@ -0,0 +1,54 @@ +import pytest +from pydantic import ValidationError + +from template_code_location.dataframe_level_anonymisation.config_models.base_config import BaseConfiguration + + +def test_valid_configuration_with_overrides(): + cfg = BaseConfiguration( + ident=["id"], + quasi_identifiers=["age"], + supp_level=10.0, + generalisation_hierarchies={"age": "age_hierarchy"}, + ) + assert cfg.ident == ["id"] + assert cfg.quasi_identifiers == ["age"] + assert cfg.supp_level == 10.0 + assert cfg.generalisation_hierarchies == {"age": "age_hierarchy"} + + +def test_default_values_are_loaded(): + cfg = BaseConfiguration() + assert cfg.ident == ["Name"] + assert cfg.quasi_identifiers == ["Age"] + assert cfg.supp_level == 50.0 + assert cfg.generalisation_hierarchies == {"Age": "simpl_age"} + + +def test_missing_ident_raises_error(): + with pytest.raises(ValidationError): + BaseConfiguration( + ident=[] + ) + + +def test_missing_quasi_ident_raises_error(): + with pytest.raises(ValidationError): + BaseConfiguration( + quasi_identifiers=[] + ) + + +def test_overlap_between_ident_and_quasi_identifiers(): + with pytest.raises(ValidationError): + BaseConfiguration( + ident=["age"], + quasi_identifiers=["age"] + ) + + +def test_supp_level_bounds(): + with pytest.raises(ValidationError): + BaseConfiguration( + supp_level=150.0 # fuori range + ) diff --git a/tests/dataframe_level_anonymisation/config_models/test_hierarchies.py b/tests/dataframe_level_anonymisation/config_models/test_hierarchies.py new file mode 100644 index 0000000..c6994a9 --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/test_hierarchies.py @@ -0,0 +1,48 @@ +from template_code_location.dataframe_level_anonymisation.config_models.hierarchies import ( + simpl_age, + simpl_age2, + simpl_gender, + get_all_hierarchies, +) + + +def test_simpl_age_structure(): + assert isinstance(simpl_age, dict) + assert 0 in simpl_age + assert isinstance(simpl_age[0], list) + # verify first level contains 100 ages + assert len(simpl_age[0]) == 100 + assert simpl_age[0][0] == 0 + assert simpl_age[0][-1] == 99 + + +def test_simpl_age2_structure(): + assert isinstance(simpl_age2, dict) + assert 0 in simpl_age2 + assert 1 in simpl_age2 + assert isinstance(simpl_age2[0], list) + assert isinstance(simpl_age2[1], list) + + +def test_simpl_gender_structure(): + assert isinstance(simpl_gender, dict) + assert 0 in simpl_gender + assert 1 in simpl_gender + assert simpl_gender[0] == ["M", "F", "O"] + assert simpl_gender[1] == ["*", "*", "*"] + + +def test_get_all_hierarchies(): + hier = get_all_hierarchies() + + # the function should return dicts only + assert isinstance(hier, dict) + + # ensure expected dicts are included + assert "simpl_age" in hier + assert "simpl_age2" in hier + assert "simpl_gender" in hier + + # ensure the values returned are references to the actual dicts + assert hier["simpl_age"] is simpl_age + assert hier["simpl_gender"] is simpl_gender diff --git a/tests/dataframe_level_anonymisation/config_models/test_k_anonymity_config.py b/tests/dataframe_level_anonymisation/config_models/test_k_anonymity_config.py new file mode 100644 index 0000000..ef6e2c8 --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/test_k_anonymity_config.py @@ -0,0 +1,41 @@ +import pytest +from pydantic import ValidationError + +from template_code_location.dataframe_level_anonymisation.config_models.k_anonymity_configuration import ( + KAnonymityConfiguration, +) + + +def test_valid_k_anonymity_config_with_overrides(): + cfg = KAnonymityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + supp_level=5.0, + generalisation_hierarchies={"age": "age_hier"}, + k=3, + sensitive_attributes=["disease"], + ) + assert cfg.k == 3 + assert cfg.sensitive_attributes == ["disease"] + assert cfg.generalisation_hierarchies == {"age": "age_hier"} + + +def test_default_values_are_loaded(): + cfg = KAnonymityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"} + ) + assert cfg.k == 3 + assert cfg.sensitive_attributes == ["Disease"] + + +def test_invalid_k_value_raises_error(): + with pytest.raises(ValidationError): + KAnonymityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"}, + k=1, # invalid, must be >= 2 + sensitive_attributes=["disease"], + ) diff --git a/tests/dataframe_level_anonymisation/config_models/test_l_diversity_config.py b/tests/dataframe_level_anonymisation/config_models/test_l_diversity_config.py new file mode 100644 index 0000000..c94db3e --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/test_l_diversity_config.py @@ -0,0 +1,44 @@ +import pytest +from pydantic import ValidationError + +from template_code_location.dataframe_level_anonymisation.config_models.l_diversity_configuration import ( + LDiversityConfiguration, +) + + +def test_valid_l_diversity_config_with_overrides(): + cfg = LDiversityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + supp_level=5.0, + generalisation_hierarchies={"age": "age_hier"}, + k=3, + l=2, + sensitive_attribute="disease", + ) + assert cfg.k == 3 + assert cfg.l == 2 + assert cfg.sensitive_attribute == "disease" + + +def test_default_values_are_loaded(): + cfg = LDiversityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"} + ) + assert cfg.k == 2 + assert cfg.l == 3 + assert cfg.sensitive_attribute == "Disease" + + +def test_invalid_l_value_raises_error(): + with pytest.raises(ValidationError): + LDiversityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"}, + k=3, + l=0, # invalid, must be >= 1 + sensitive_attribute="disease", + ) diff --git a/tests/dataframe_level_anonymisation/config_models/test_t_closeness_config.py b/tests/dataframe_level_anonymisation/config_models/test_t_closeness_config.py new file mode 100644 index 0000000..615bd27 --- /dev/null +++ b/tests/dataframe_level_anonymisation/config_models/test_t_closeness_config.py @@ -0,0 +1,56 @@ +import pytest +from pydantic import ValidationError + +from template_code_location.dataframe_level_anonymisation.config_models.t_closeness_configuration import ( + TClosenessConfiguration, +) + + +def test_valid_t_closeness_config_with_overrides(): + cfg = TClosenessConfiguration( + ident=["id"], + quasi_identifiers=["age"], + supp_level=5.0, + generalisation_hierarchies={"age": "age_hier"}, + k=3, + t=0.4, + sensitive_attribute="disease", + ) + assert cfg.k == 3 + assert cfg.t == 0.4 + assert cfg.sensitive_attribute == "disease" + + +def test_default_values_are_loaded(): + cfg = TClosenessConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"} + ) + assert cfg.k == 2 + assert cfg.t == 0.5 + assert cfg.sensitive_attribute == "Disease" + + +def test_invalid_t_value_low(): + with pytest.raises(ValidationError): + TClosenessConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"}, + k=3, + t=-0.1, # invalid + sensitive_attribute="disease", + ) + + +def test_invalid_t_value_high(): + with pytest.raises(ValidationError): + TClosenessConfiguration( + ident=["id"], + quasi_identifiers=["age"], + generalisation_hierarchies={"age": "age_hier"}, + k=3, + t=2.0, # invalid > 1 + sensitive_attribute="disease", + ) diff --git a/tests/dataframe_level_anonymisation/test_jobs.py b/tests/dataframe_level_anonymisation/test_jobs.py new file mode 100644 index 0000000..f890e2d --- /dev/null +++ b/tests/dataframe_level_anonymisation/test_jobs.py @@ -0,0 +1,44 @@ +from template_code_location.dataframe_level_anonymisation.jobs import ( + k_anonymity_job, + l_diversity_job, + t_closeness_job, + k_anonymity_job_s3, + l_diversity_job_s3, + t_closeness_job_s3 +) + + +def test_k_anonymity_job_is_callable(): + """Test k_anonymity_job is a valid Dagster job""" + assert callable(k_anonymity_job) + assert hasattr(k_anonymity_job, 'execute_in_process') + + +def test_l_diversity_job_is_callable(): + """Test l_diversity_job is a valid Dagster job""" + assert callable(l_diversity_job) + assert hasattr(l_diversity_job, 'execute_in_process') + + +def test_t_closeness_job_is_callable(): + """Test t_closeness_job is a valid Dagster job""" + assert callable(t_closeness_job) + assert hasattr(t_closeness_job, 'execute_in_process') + + +def test_k_anonymity_job_s3_is_callable(): + """Test k_anonymity_job_s3 is a valid Dagster job""" + assert callable(k_anonymity_job_s3) + assert hasattr(k_anonymity_job_s3, 'execute_in_process') + + +def test_l_diversity_job_s3_is_callable(): + """Test l_diversity_job_s3 is a valid Dagster job""" + assert callable(l_diversity_job_s3) + assert hasattr(l_diversity_job_s3, 'execute_in_process') + + +def test_t_closeness_job_s3_is_callable(): + """Test t_closeness_job_s3 is a valid Dagster job""" + assert callable(t_closeness_job_s3) + assert hasattr(t_closeness_job_s3, 'execute_in_process') diff --git a/tests/dataframe_level_anonymisation/test_ops.py b/tests/dataframe_level_anonymisation/test_ops.py new file mode 100644 index 0000000..90c01aa --- /dev/null +++ b/tests/dataframe_level_anonymisation/test_ops.py @@ -0,0 +1,230 @@ +import pytest +import pandas as pd +from unittest.mock import patch +from dagster import DagsterInvalidInvocationError, build_op_context + +from template_code_location.dataframe_level_anonymisation.ops import ( + apply_k_anonymity, + apply_l_diversity, + apply_t_closeness, +) +from template_code_location.dataframe_level_anonymisation.config_models import ( + KAnonymityConfiguration, + LDiversityConfiguration, + TClosenessConfiguration, +) + + +# --------------------------- +# Fixtures +# --------------------------- +@pytest.fixture +def fake_df(): + return pd.DataFrame({"id": [1, 2], "age": [30, 40]}) + + +@pytest.fixture +def k_config(): + return KAnonymityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + sensitive_attributes=["age"], + k=2, + supp_level=0.0, + generalisation_hierarchies={"age": "simpl_age"}, + ) + + +@pytest.fixture +def l_config(): + return LDiversityConfiguration( + ident=["id"], + quasi_identifiers=["age"], + sensitive_attribute="age", + k=2, + l=1, + supp_level=0.0, + generalisation_hierarchies={"age": "simpl_age"}, + ) + + +@pytest.fixture +def t_config(): + return TClosenessConfiguration( + ident=["id"], + quasi_identifiers=["age"], + sensitive_attribute="age", + k=2, + t=0.5, + supp_level=0.0, + generalisation_hierarchies={"age": "simpl_age"}, + ) + + +@pytest.fixture +def op_context(): + return build_op_context() + + +# --------------------------- +# Helper for patching external functions +# --------------------------- +@pytest.fixture(autouse=True) +def patch_external_ops(): + with ( + patch( + "dataframe_level_anonymisation.ops.get_all_hierarchies", + return_value={"simpl_age": {0: [30, 40]}}, + ), + patch( + "dataframe_level_anonymisation.ops.k_anonymity", + return_value=pd.DataFrame({"id": [1, 2], "age": [30, 40]}), + ), + patch( + "dataframe_level_anonymisation.ops.l_diversity", + return_value=pd.DataFrame({"id": [1, 2], "age": [30, 40]}), + ), + patch( + "dataframe_level_anonymisation.ops.t_closeness", + return_value=pd.DataFrame({"id": [1, 2], "age": [30, 40]}), + ), + ): + yield + + +# --------------------------- +# Tests for apply_k_anonymity +# --------------------------- +def test_apply_k_anonymity_outputs(op_context, k_config, fake_df): + results = list(apply_k_anonymity(op_context, k_config, fake_df)) + assert len(results) == 2 + + data_output = results[0].value + metrics_output = results[1].value + + # Check types + assert isinstance(data_output, pd.DataFrame) + assert isinstance(metrics_output, dict) + assert "k_anon" in metrics_output + assert "l_div" in metrics_output + assert "t_clos" in metrics_output + + +# --------------------------- +# Tests for apply_l_diversity +# --------------------------- +def test_apply_l_diversity_outputs(op_context, l_config, fake_df): + results = list(apply_l_diversity(op_context, l_config, fake_df)) + assert len(results) == 2 + + data_output = results[0].value + metrics_output = results[1].value + + assert isinstance(data_output, pd.DataFrame) + assert isinstance(metrics_output, dict) + assert "k_anon" in metrics_output + assert "l_div" in metrics_output + assert "t_clos" in metrics_output + + +def test_apply_l_diversity_empty_raises(op_context, l_config): + with patch("dataframe_level_anonymisation.ops.l_diversity", return_value=pd.DataFrame()): + + with pytest.raises(DagsterInvalidInvocationError): + list(apply_l_diversity(op_context, l_config, pd.DataFrame({"id": [1], "age": [30]}))) + + +# --------------------------- +# Tests for apply_t_closeness +# --------------------------- +def test_apply_t_closeness_outputs(op_context, t_config, fake_df): + results = list(apply_t_closeness(op_context, t_config, fake_df)) + assert len(results) == 2 + + data_output = results[0].value + metrics_output = results[1].value + + assert isinstance(data_output, pd.DataFrame) + assert isinstance(metrics_output, dict) + assert "k_anon" in metrics_output + assert "l_div" in metrics_output + assert "t_clos" in metrics_output + + +def test_apply_t_closeness_empty_raises(op_context, t_config): + with patch("dataframe_level_anonymisation.ops.t_closeness", return_value=pd.DataFrame()): + with pytest.raises(DagsterInvalidInvocationError): + list(apply_t_closeness(op_context, t_config, pd.DataFrame({"id": [1], "age": [30]}))) + + +# --------------------------- +# Additional tests for _validate_and_get_hierarchies +# --------------------------- +def test_validate_hierarchies_dataset_too_small(k_config): + small_df = pd.DataFrame({"id": [1], "age": [30]}) + from template_code_location.dataframe_level_anonymisation.ops import _validate_and_get_hierarchies + + with pytest.raises(DagsterInvalidInvocationError): + _validate_and_get_hierarchies(k_config, small_df) + + +def test_validate_hierarchies_missing_hierarchy(k_config, fake_df): + from template_code_location.dataframe_level_anonymisation.ops import _validate_and_get_hierarchies + + bad_config = k_config.model_copy(update={"generalisation_hierarchies": {}}) + + with pytest.raises(DagsterInvalidInvocationError): + _validate_and_get_hierarchies(bad_config, fake_df) + + +def test_validate_hierarchies_hierarchy_not_in_code(k_config, fake_df): + from template_code_location.dataframe_level_anonymisation.ops import _validate_and_get_hierarchies + + with patch("dataframe_level_anonymisation.ops.get_all_hierarchies", return_value={}): + with pytest.raises(DagsterInvalidInvocationError): + _validate_and_get_hierarchies(k_config, fake_df) + + +# --------------------------- +# Additional tests for _calc_dataframe_metrics +# --------------------------- +def test_calc_dataframe_metrics_basic(): + from template_code_location.dataframe_level_anonymisation.ops import _calc_dataframe_metrics + + df_org = pd.DataFrame({"age": [30, 40], "id": [1, 2]}) + df_anon = df_org.copy() + + with ( + patch("dataframe_level_anonymisation.ops.anonymity.k_anonymity", return_value=2), + patch("dataframe_level_anonymisation.ops.anonymity.l_diversity", return_value=1), + patch("dataframe_level_anonymisation.ops.anonymity.t_closeness", return_value=0.1), + ): + + report, metrics = _calc_dataframe_metrics(df_anon, df_org, ["age"], ["age"]) + + assert "k-anonymity" in report + assert metrics["k_anon"] == 2 + assert metrics["l_div"] == 1 + assert metrics["t_clos"] == 0.1 + + +# --------------------------- +# Tests for apply_t_closeness exception branches +# --------------------------- +def test_apply_t_closeness_value_error_quasi_identifiers(op_context, t_config, fake_df): + """Covers the branch where ValueError contains 'Cannot be quasi-identifiers'.""" + with patch( + "dataframe_level_anonymisation.ops.t_closeness", + side_effect=ValueError("Cannot be quasi-identifiers invalid"), + ): + with pytest.raises(DagsterInvalidInvocationError): + list(apply_t_closeness(op_context, t_config, fake_df)) + + +def test_apply_t_closeness_value_error_other_message(op_context, t_config, fake_df): + """Covers the branch where ValueError is raised but message does NOT contain that substring.""" + with patch( + "dataframe_level_anonymisation.ops.t_closeness", side_effect=ValueError("Some other error") + ): + with pytest.raises(DagsterInvalidInvocationError): + list(apply_t_closeness(op_context, t_config, fake_df)) diff --git a/tests/dataframe_level_anonymisation/test_utils.py b/tests/dataframe_level_anonymisation/test_utils.py new file mode 100644 index 0000000..3fa1841 --- /dev/null +++ b/tests/dataframe_level_anonymisation/test_utils.py @@ -0,0 +1,70 @@ +import numpy as np + +from template_code_location.dataframe_level_anonymisation.utils import ( + parse_value_list, + normalize_hierarchy_levels, +) + + +# ------------------------------------ +# Tests for parse_value_list +# ------------------------------------ +def test_parse_value_list_all_strings_digits(): + values = ["1", "2", "3"] + assert parse_value_list(values) == [1, 2, 3] + + +def test_parse_value_list_mixed_values(): + values = ["1", 2, "abc", "5"] + assert parse_value_list(values) == [1, 2, "abc", 5] + + +def test_parse_value_list_no_digits(): + values = ["a", "b", "c"] + assert parse_value_list(values) == ["a", "b", "c"] + + +# ------------------------------------ +# Tests for normalize_hierarchy_levels +# ------------------------------------ +def test_normalize_hierarchy_levels_level_0_converted_to_numpy_array(): + hierarchy = {"age": {"0": ["1", "2", "3"], "1": ["0-10", "11-20"]}} + + normalized = normalize_hierarchy_levels(hierarchy) + + assert "age" in normalized + assert 0 in normalized["age"] + assert isinstance(normalized["age"][0], np.ndarray) + assert normalized["age"][0].tolist() == [1, 2, 3] # converted via parse_value_list + assert normalized["age"][1] == ["0-10", "11-20"] # untouched + + +def test_normalize_hierarchy_levels_multiple_columns(): + hierarchy = {"age": {"0": ["10", "20"]}, "gender": {"0": ["M", "F"], "1": ["*"]}} + + normalized = normalize_hierarchy_levels(hierarchy) + + # First column + assert isinstance(normalized["age"][0], np.ndarray) + assert normalized["age"][0].tolist() == [10, 20] + + # Second column + assert isinstance(normalized["gender"][0], np.ndarray) + assert normalized["gender"][0].tolist() == ["M", "F"] + assert normalized["gender"][1] == ["*"] + + +def test_normalize_hierarchy_levels_mixed_digit_non_digit_at_level_0(): + hierarchy = {"test": {"0": ["1", "x", "3"]}} + + normalized = normalize_hierarchy_levels(hierarchy) + + assert isinstance(normalized["test"][0], np.ndarray) + assert normalized["test"][0].tolist() == ["1", "x", "3"] + + +def test_normalize_hierarchy_levels_empty_mapping(): + hierarchy = {"col": {}} + normalized = normalize_hierarchy_levels(hierarchy) + + assert normalized == {"col": {}} diff --git a/tests/field_level_pseudo_anonymisation/__init__.py b/tests/field_level_pseudo_anonymisation/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/field_level_pseudo_anonymisation/conftest.py b/tests/field_level_pseudo_anonymisation/conftest.py new file mode 100644 index 0000000..ee54069 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/conftest.py @@ -0,0 +1,444 @@ +""" +Shared pytest fixtures and helpers for field-level pseudonymisation tests. + +This module provides: +- Mock Vault client for testing without real Vault connections +- Sample data fixtures +- Configuration fixtures for encryption/decryption operations +- Helper functions for running ops and managing test Vault storage +""" + +import pandas as pd +import pytest +from dagster import build_op_context +from cryptography.fernet import Fernet +from hvac.exceptions import InvalidPath, Forbidden +from unittest.mock import patch, MagicMock + +from template_code_location.field_level_pseudo_anonymisation.config_models.structured_config import ( + AnonymisePseudonymizeStructuredConfig, + DepseudonymizeStructuredConfig, + EncryptConfig, + DecryptConfig, + PseudoTechniqueConfig, + DepseudoTechniqueConfig, +) +from template_code_location.field_level_pseudo_anonymisation.ops import ( + anonymize_pseudonymize_structured, + depseudonymize_structured, +) + + +# -------------------------------- Mock Vault Storage ---------------------------------------- + +# In-memory Vault simulation for tests +_test_vault_storage = {} +_test_vault_access_control = {} # For simulating access control + + +@pytest.fixture(autouse=True) +def mock_vault_client(): + """ + Auto-use fixture that mocks the hvac.Client to avoid real Vault connections. + Uses an in-memory dict to simulate Vault storage for tests. + Includes access control simulation for AC3. + """ + global _test_vault_storage, _test_vault_access_control + _test_vault_storage = {} # Reset storage before each test + _test_vault_access_control = {} # Reset access control + + def mock_read_secret(path, mount_point): + """Mock reading secret from Vault with access control""" + full_path = f"{mount_point}/{path}" + + # Check access control first + if full_path in _test_vault_access_control: + if not _test_vault_access_control[full_path]: + raise Forbidden(f"Access denied to secret: {full_path}") + + if full_path not in _test_vault_storage: + raise InvalidPath(f"Secret not found: {full_path}") + return {"data": {"data": {"value": _test_vault_storage[full_path]}}} + + def mock_create_or_update_secret(path, mount_point, secret): + """Mock creating/updating secret in Vault""" + full_path = f"{mount_point}/{path}" + _test_vault_storage[full_path] = secret["value"] + + def mock_delete_metadata(path, mount_point): + """Mock deleting secret from Vault""" + full_path = f"{mount_point}/{path}" + if full_path in _test_vault_storage: + del _test_vault_storage[full_path] + if full_path in _test_vault_access_control: + del _test_vault_access_control[full_path] + + with patch("hvac.Client") as mock_client_class: + mock_instance = MagicMock() + mock_instance.secrets.kv.v2.read_secret_version.side_effect = mock_read_secret + mock_instance.secrets.kv.v2.create_or_update_secret.side_effect = ( + mock_create_or_update_secret + ) + mock_instance.secrets.kv.v2.delete_metadata_and_all_versions.side_effect = ( + mock_delete_metadata + ) + mock_client_class.return_value = mock_instance + yield mock_instance + + +# -------------------------------- Sample Data Fixtures ---------------------------------------- + + +@pytest.fixture +def sample_df(): + """ + Fixture providing a sample structured dataset with PII data. + Represents typical data that requires pseudonymisation and restoration. + """ + return pd.DataFrame( + { + "id": [1, 2, 3, 4, 5], + "name": [ + "Alice Smith", + "Bob Jones", + "Charlie Brown", + "David Wilson", + "Eva Garcia", + ], + "email": [ + "alice@example.com", + "bob@example.com", + "charlie@example.com", + "david@example.com", + "eva@example.com", + ], + "ssn": [ + "123-45-6789", + "234-56-7890", + "345-67-8901", + "456-78-9012", + "567-89-0123", + ], + "age": [25, 30, 35, 40, 45], + "salary": [50000.0, 60000.0, 70000.0, 80000.0, 90000.0], + "department": ["HR", "IT", "Finance", "IT", "HR"], + } + ) + + +# -------------------------------- Configuration Fixtures ---------------------------------------- + + +@pytest.fixture +def encrypt_config_single_field(): + """ + Configuration for encrypting a single field (email). + Used to create pseudonymised data for restoration tests. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + +@pytest.fixture +def decrypt_config_single_field(): + """ + Configuration for decrypting a single field (email). + Used to restore original values. + """ + return DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + +@pytest.fixture +def encrypt_config_multiple_fields(): + """ + Configuration for encrypting multiple fields (name, email, ssn). + Tests restoration of multiple sensitive fields. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + columns=["name", "email", "ssn"], + key_name="test_restoration_key_multi", + ) + ) + ] + ) + + +@pytest.fixture +def decrypt_config_multiple_fields(): + """ + Configuration for decrypting multiple fields (name, email, ssn). + """ + return DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["name", "email", "ssn"], + key_name="test_restoration_key_multi", + ) + ) + ] + ) + + +@pytest.fixture +def encrypt_config_partial_fields(): + """ + Configuration for encrypting only some fields (email, ssn). + Tests partial restoration scenarios. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + columns=["email", "ssn"], + key_name="test_restoration_key_partial", + ) + ) + ] + ) + + +@pytest.fixture +def decrypt_config_partial_fields(): + """ + Configuration for decrypting only some fields (email, ssn). + """ + return DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email", "ssn"], + key_name="test_restoration_key_partial", + ) + ) + ] + ) + + +@pytest.fixture +def authorized_multi_key_scenario(): + """ + Fixture for testing multi-key authorization scenarios. + Sets up two keys: one authorized, one denied. + """ + clear_vault_key("authorized_key") + clear_vault_key("unauthorized_key") + + # Create authorized key by generating it + authorized_key = Fernet.generate_key().decode() + set_vault_key("authorized_key", authorized_key) + + # Create unauthorized key and deny access + unauthorized_key = Fernet.generate_key().decode() + set_vault_key("unauthorized_key", unauthorized_key) + deny_vault_access("unauthorized_key") + + yield {"authorized": "authorized_key", "unauthorized": "unauthorized_key"} + + # Cleanup + clear_vault_key("authorized_key") + clear_vault_key("unauthorized_key") + + +@pytest.fixture +def large_dataset(): + """ + Fixture providing a large dataset (10,000 rows) for performance testing. + Reusable across multiple performance tests. + """ + return pd.DataFrame( + { + "id": range(1, 10001), + "email": [f"user{i}@example.com" for i in range(1, 10001)], + "name": [f"User {i}" for i in range(1, 10001)], + "ssn": [f"{i:03d}-{i:02d}-{i:04d}" for i in range(1, 10001)], + "age": [20 + (i % 50) for i in range(1, 10001)], + "salary": [30000.0 + (i * 10) for i in range(1, 10001)], + "department": [["HR", "IT", "Finance", "Sales"][i % 4] for i in range(1, 10001)], + } + ) + + +@pytest.fixture(scope="session") +def vault_test_keys(): + """ + Session-scoped fixture to pre-generate test keys for faster test execution. + Avoids repeated key generation in each test. + """ + keys = {f"test_key_{i}": Fernet.generate_key().decode() for i in range(10)} + + return keys + + +@pytest.fixture +def cleanup_test_keys(request): + """ + Fixture to automatically cleanup test keys after each test. + Use with: @pytest.mark.usefixtures("cleanup_test_keys") + """ + yield + + # Cleanup all test keys from mock Vault + test_keys = [k for k in _test_vault_storage.keys() if "test_" in k] + for key in test_keys: + _test_vault_storage.pop(key, None) + + +# -------------------------------- Helper Functions ---------------------------------------- + + +def config_to_dagster_dict(config): + """ + Convert Pydantic config to Dagster-compatible dictionary. + + For AnonymisePseudonymizeStructuredConfig (uses discriminated Union): + Pydantic v2 outputs: {'technique': {'type': 'encrypt', 'columns': [...], 'key_name': '...'}} + Dagster expects: {'technique': {'encrypt': {'columns': [...], 'key_name': '...'}}} + + For DepseudonymizeStructuredConfig (direct DecryptConfig, no Union): + Pydantic v2 outputs: + {'technique': {'type': 'decrypt', 'columns': [...], 'key_name': '...'}} + Dagster expects: Same flat structure with 'type' field + + Args: + config: Pydantic config instance + (AnonymisePseudonymizeStructuredConfig or + DepseudonymizeStructuredConfig) + + Returns: + dict: Dagster-compatible configuration dictionary + """ + from template_code_location.field_level_pseudo_anonymisation.config_models.structured_config import ( + AnonymisePseudonymizeStructuredConfig, + ) + + config_dict = config.model_dump() + + # Only convert discriminated unions for AnonymisePseudonymizeStructuredConfig + # DepseudonymizeStructuredConfig uses direct DecryptConfig (no discriminated union) + if isinstance(config, AnonymisePseudonymizeStructuredConfig): + if "used_function" in config_dict: + for func_config in config_dict["used_function"]: + if "technique" in func_config: + technique = func_config["technique"] + # Pydantic outputs flat dict with 'type' field for discriminated unions + if isinstance(technique, dict) and "type" in technique: + # Extract the type discriminator + technique_type = technique["type"] + # Create nested structure without the 'type' field + technique_data = {k: v for k, v in technique.items() if k != "type"} + # Nest under the discriminator key for Dagster + func_config["technique"] = {technique_type: technique_data} + + return config_dict + + +def run_encrypt_op(config, df): + """ + Helper function to execute the anonymize_pseudonymize_structured op. + + Args: + config: AnonymisePseudonymizeStructuredConfig instance + df: Input pandas DataFrame + + Returns: + tuple: (result_df, metrics) - Output DataFrame and metrics dict + """ + context = build_op_context(op_config=config_to_dagster_dict(config)) + result_df, metrics = anonymize_pseudonymize_structured(context, df=df) + return result_df.value, metrics.value + + +def run_decrypt_op(config, df): + """ + Helper function to execute the depseudonymize_structured op. + + Args: + config: DepseudonymizeStructuredConfig instance + df: Input pandas DataFrame + + Returns: + tuple: (result_df, metrics) - Output DataFrame and metrics dict + """ + context = build_op_context(op_config=config_to_dagster_dict(config)) + result_df, metrics = depseudonymize_structured(context, df=df) + return result_df.value, metrics.value + + +def clear_vault_key(key_name: str): + """ + Helper function to clear a key from the simulated Vault storage for test isolation. + + Args: + key_name: Name of the key to delete from Vault + """ + full_path = f"secret/PseudonymKeys/{key_name}" + if full_path in _test_vault_storage: + del _test_vault_storage[full_path] + if full_path in _test_vault_access_control: + del _test_vault_access_control[full_path] + + +def set_vault_key(key_name: str, key_value: str): + """ + Helper function to set a key in the simulated Vault storage. + + Args: + key_name: Name of the key + key_value: Value of the key (Fernet key as string) + """ + full_path = f"secret/PseudonymKeys/{key_name}" + _test_vault_storage[full_path] = key_value + + +def deny_vault_access(key_name: str): + """ + Helper function to deny access to a key for authorization testing (AC3). + + Args: + key_name: Name of the key to deny access to + """ + full_path = f"secret/PseudonymKeys/{key_name}" + _test_vault_access_control[full_path] = False + + +def get_vault_key(key_name: str) -> bytes: + """ + Helper function to retrieve a key from the simulated Vault storage. + + Args: + key_name: Name of the key to retrieve + + Returns: + bytes: The encryption key + """ + full_path = f"secret/PseudonymKeys/{key_name}" + if full_path not in _test_vault_storage: + raise InvalidPath(f"Key not found: {key_name}") + return _test_vault_storage[full_path].encode() diff --git a/tests/field_level_pseudo_anonymisation/test_config_models_coverage.py b/tests/field_level_pseudo_anonymisation/test_config_models_coverage.py new file mode 100644 index 0000000..010b9a6 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_config_models_coverage.py @@ -0,0 +1,633 @@ +import pytest +from pydantic import ValidationError + +from template_code_location.field_level_pseudo_anonymisation.config_models.structured_config import ( + AnonymisePseudonymizeStructuredConfig, + DepseudonymizeStructuredConfig, + PseudoTechniqueConfig, + DepseudoTechniqueConfig, + HashConfig, + EncryptConfig, + RedactConfig, + ReplaceConfig, + DecryptConfig, +) +from template_code_location.field_level_pseudo_anonymisation.config_models.unstructured_config import ( + AnonymisePseudonymizeUnstructuredConfig, + DepseudonymizeUnstructuredConfig, + PseudoTechniqueConfig as UnstructuredPseudoTechniqueConfig, + DepseudoTechniqueConfig as UnstructuredDepseudoTechniqueConfig, + HashConfig as UnstructuredHashConfig, + EncryptConfig as UnstructuredEncryptConfig, + RedactConfig as UnstructuredRedactConfig, + ReplaceConfig as UnstructuredReplaceConfig, + RetainConfig, + DecryptConfig as UnstructuredDecryptConfig, +) +from template_code_location.field_level_pseudo_anonymisation.config_models.languages import LanguageEnum +from template_code_location.field_level_pseudo_anonymisation.config_models.pii_entities import PIIEntityEnum + + +# ==================== Structured Config Tests ==================== + +class TestStructuredConfigValidators: + """Tests for structured_config.py validators and validators.""" + + def test_ensure_unique_columns_valid_single_technique(self): + """Test that single technique with single column passes validation.""" + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + columns=["email"], + key_name="key1" + ) + ) + ] + ) + assert config is not None + assert len(config.used_function) == 1 + + def test_ensure_unique_columns_valid_multiple_techniques_different_columns(self): + """Test that multiple techniques with different columns passes validation.""" + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + columns=["email"], + key_name="key1" + ) + ), + PseudoTechniqueConfig( + technique=HashConfig( + columns=["ssn"], + algorithm="sha256" + ) + ) + ] + ) + assert config is not None + assert len(config.used_function) == 2 + + def test_ensure_unique_columns_duplicate_columns_same_technique(self): + """Test that duplicate columns in different techniques raises error.""" + with pytest.raises(ValueError) as exc_info: + AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + columns=["email"], + key_name="key1" + ) + ), + PseudoTechniqueConfig( + technique=HashConfig( + columns=["email"], + algorithm="sha256" + ) + ) + ] + ) + assert "Duplicate column" in str(exc_info.value) + assert "email" in str(exc_info.value) + + def test_ensure_unique_columns_multiple_duplicates(self): + """Test error message with multiple duplicate columns.""" + with pytest.raises(ValueError) as exc_info: + AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + columns=["email", "phone"], + key_name="key1" + ) + ), + PseudoTechniqueConfig( + technique=HashConfig( + columns=["email", "phone"], + algorithm="sha256" + ) + ) + ] + ) + error_msg = str(exc_info.value) + assert "Duplicate column" in error_msg + assert "email" in error_msg + assert "phone" in error_msg + + def test_collect_column_to_techniques_single_technique(self): + """Test _collect_column_to_techniques with single technique.""" + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + columns=["email", "phone"], + key_name="key1" + ) + ) + ] + ) + mapping = config._collect_column_to_techniques() + assert mapping == { + "email": ["encrypt"], + "phone": ["encrypt"] + } + + def test_extract_technique_and_columns_dict_with_type_field(self): + """Test _extract_technique_and_columns with dict containing 'type' field.""" + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns( + { + "technique": { + "type": "encrypt", + "columns": ["email", "ssn"], + "key_name": "test_key" + } + } + ) + assert technique_type == "encrypt" + assert columns == ["email", "ssn"] + + def test_extract_technique_and_columns_dict_with_variant_mapping(self): + """Test _extract_technique_and_columns with variant-key mapping {'hash': {...}}.""" + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns( + { + "technique": { + "encrypt": { + "columns": ["ssn"], + "key_name": "test_key" + } + } + } + ) + assert technique_type == "encrypt" + assert columns == ["ssn"] + + def test_extract_technique_and_columns_model_instance(self): + """Test _extract_technique_and_columns with PseudoTechniqueConfig model instance.""" + pseudo_config = PseudoTechniqueConfig( + technique=RedactConfig(columns=["address"]) + ) + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns(pseudo_config) + assert technique_type == "redact" + assert columns == ["address"] + + def test_extract_technique_and_columns_empty_dict(self): + """Test _extract_technique_and_columns with empty dict.""" + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns( + {"technique": {}} + ) + assert technique_type is None + assert columns == [] + + def test_extract_technique_and_columns_none_technique(self): + """Test _extract_technique_and_columns with None technique.""" + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns( + {"technique": None} + ) + assert technique_type is None + assert columns == [] + + def test_extract_technique_and_columns_missing_columns_key(self): + """Test _extract_technique_and_columns when 'columns' key is missing.""" + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns( + { + "technique": { + "type": "encrypt", + "key_name": "test_key" + } + } + ) + assert technique_type == "encrypt" + assert columns == [] + + def test_extract_technique_and_columns_model_without_columns_attr(self): + """Test _extract_technique_and_columns with model instance missing columns attribute.""" + pseudo_config = PseudoTechniqueConfig( + technique=ReplaceConfig(columns=["old_value"], new_value="NEW") + ) + config = AnonymisePseudonymizeStructuredConfig() + technique_type, columns = config._extract_technique_and_columns(pseudo_config) + assert technique_type == "replace" + assert columns == ["old_value"] + + +class TestStructuredDepseudonymizeConfig: + """Tests for DepseudonymizeStructuredConfig.""" + + def test_depseudonymize_config_normalize_used_function_with_dict(self): + """Test _normalize_depseudo_used_function with dict input.""" + config = DepseudonymizeStructuredConfig( + used_function=[ + { + "technique": { + "type": "decrypt", + "columns": ["email"], + "key_name": "key1" + } + } + ] + ) + assert len(config.used_function) == 1 + assert isinstance(config.used_function[0], DepseudoTechniqueConfig) + assert config.used_function[0].technique.type == "decrypt" + + def test_depseudonymize_config_normalize_used_function_with_model(self): + """Test _normalize_depseudo_used_function with model instance.""" + depseudo_tech = DepseudoTechniqueConfig( + technique=DecryptConfig( + columns=["email"], + key_name="key1" + ) + ) + config = DepseudonymizeStructuredConfig( + used_function=[depseudo_tech] + ) + assert len(config.used_function) == 1 + assert config.used_function[0] is depseudo_tech + + def test_depseudonymize_config_ensure_unique_columns_no_op(self): + """Test that ensure_unique_columns is a no-op for depseudonymize.""" + # For depseudonymize, there's no per-column uniqueness constraint + config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + columns=["email"], + key_name="key1" + ) + ), + DepseudoTechniqueConfig( + technique=DecryptConfig( + columns=["email"], + key_name="key2" + ) + ) + ] + ) + # Should not raise - no-op validator + assert config is not None + + +# ==================== Unstructured Config Tests ==================== + +class TestUnstructuredConfigValidators: + """Tests for unstructured_config.py validators.""" + + def test_normalize_used_function_with_dict(self): + """Test _normalize_used_function with dict input.""" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + { + "technique": { + "encrypt": { + "pii": [PIIEntityEnum.EMAIL.value], + "key_name": "key1" + } + } + } + ] + ) + assert len(config.used_function) == 1 + + def test_normalize_used_function_with_model(self): + """Test _normalize_used_function with model instance.""" + pseudo_tech = UnstructuredPseudoTechniqueConfig( + technique=UnstructuredEncryptConfig( + pii=[PIIEntityEnum.EMAIL.value], + key_name="key1" + ) + ) + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[pseudo_tech] + ) + assert len(config.used_function) == 1 + + def test_ensure_unique_pii_valid_different_pii_types(self): + """Test that different PII types pass validation.""" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredEncryptConfig( + pii=[PIIEntityEnum.EMAIL.value], + key_name="key1" + ) + ), + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredHashConfig( + pii=[PIIEntityEnum.PERSON.value], + algorithm="sha256" + ) + ) + ] + ) + assert config is not None + assert len(config.used_function) == 2 + + def test_ensure_unique_pii_duplicate_pii_types(self): + """Test that duplicate PII types raise error.""" + with pytest.raises(ValueError) as exc_info: + AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredEncryptConfig( + pii=[PIIEntityEnum.EMAIL.value], + key_name="key1" + ) + ), + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredHashConfig( + pii=[PIIEntityEnum.EMAIL.value], + algorithm="sha256" + ) + ) + ] + ) + assert "Duplicate PII" in str(exc_info.value) + # Error message shows PIIEntityEnum.EMAIL (the enum repr) rather than the value + assert "EMAIL" in str(exc_info.value) + + def test_collect_pii_to_techniques_single_technique(self): + """Test _collect_pii_to_techniques with single technique.""" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredEncryptConfig( + pii=[PIIEntityEnum.EMAIL.value, PIIEntityEnum.PERSON.value], + key_name="key1" + ) + ) + ] + ) + mapping = config._collect_pii_to_techniques() + assert mapping == { + PIIEntityEnum.EMAIL.value: ["encrypt"], + PIIEntityEnum.PERSON.value: ["encrypt"] + } + + def test_extract_technique_and_pii_dict_with_type_field(self): + """Test _extract_technique_and_pii with dict containing 'type' field.""" + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii( + { + "technique": { + "type": "encrypt", + "pii": [PIIEntityEnum.EMAIL.value], + "key_name": "test_key" + } + } + ) + assert technique_type == "encrypt" + assert piis == [PIIEntityEnum.EMAIL.value] + + def test_extract_technique_and_pii_dict_with_variant_mapping(self): + """Test _extract_technique_and_pii with variant-key mapping.""" + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii( + { + "technique": { + "hash": { + "pii": [PIIEntityEnum.PERSON.value], + "algorithm": "sha256" + } + } + } + ) + assert technique_type == "hash" + assert piis == [PIIEntityEnum.PERSON.value] + + def test_extract_technique_and_pii_dict_fallback_to_columns(self): + """Test _extract_technique_and_pii fallback to 'columns' key when 'pii' is missing.""" + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii( + { + "technique": { + "type": "redact", + "columns": ["fallback_col"] + } + } + ) + assert technique_type == "redact" + assert piis == ["fallback_col"] + + def test_extract_technique_and_pii_model_instance(self): + """Test _extract_technique_and_pii with model instance.""" + pseudo_tech = UnstructuredPseudoTechniqueConfig( + technique=UnstructuredRedactConfig( + pii=[PIIEntityEnum.EMAIL.value] + ) + ) + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii(pseudo_tech) + assert technique_type == "redact" + assert piis == [PIIEntityEnum.EMAIL.value] + + def test_extract_technique_and_pii_model_with_getattr_fallback(self): + """Test _extract_technique_and_pii model with getattr fallback to columns.""" + # Create a mock-like scenario where pii attribute doesn't exist + pseudo_tech = UnstructuredPseudoTechniqueConfig( + technique=RetainConfig(pii=[PIIEntityEnum.PERSON.value]) + ) + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii(pseudo_tech) + assert technique_type == "retain" + assert piis == [PIIEntityEnum.PERSON.value] + + def test_extract_technique_and_pii_empty_dict(self): + """Test _extract_technique_and_pii with empty dict.""" + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii( + {"technique": {}} + ) + assert technique_type is None + assert piis == [] + + def test_extract_technique_and_pii_missing_pii_key(self): + """Test _extract_technique_and_pii when 'pii' key is missing.""" + config = AnonymisePseudonymizeUnstructuredConfig(language=LanguageEnum.en) + technique_type, piis = config._extract_technique_and_pii( + { + "technique": { + "type": "encrypt", + "key_name": "test_key" + } + } + ) + assert technique_type == "encrypt" + assert piis == [] + + +class TestUnstructuredDepseudonymizeConfig: + """Tests for DepseudonymizeUnstructuredConfig.""" + + def test_depseudonymize_unstructured_config_default(self): + """Test default DepseudonymizeUnstructuredConfig.""" + config = DepseudonymizeUnstructuredConfig() + assert config is not None + assert len(config.used_function) >= 1 + + def test_depseudonymize_unstructured_config_with_custom_function(self): + """Test DepseudonymizeUnstructuredConfig with custom function.""" + config = DepseudonymizeUnstructuredConfig( + used_function=[ + UnstructuredDepseudoTechniqueConfig( + technique=UnstructuredDecryptConfig( + key_name="custom_key" + ) + ) + ] + ) + assert len(config.used_function) == 1 + assert config.used_function[0].technique.key_name == "custom_key" + + +class TestLanguageSupport: + """Tests for language configuration support.""" + + def test_all_supported_languages(self): + """Test that all supported languages can be set.""" + supported_languages = [ + LanguageEnum.hr, LanguageEnum.da, LanguageEnum.nl, LanguageEnum.en, + LanguageEnum.fi, LanguageEnum.fr, LanguageEnum.de, LanguageEnum.el, + LanguageEnum.it, LanguageEnum.lt, LanguageEnum.pl, LanguageEnum.pt, + LanguageEnum.ro, LanguageEnum.sl, LanguageEnum.es, LanguageEnum.sv + ] + + for lang in supported_languages: + config = AnonymisePseudonymizeUnstructuredConfig(language=lang) + assert config.language == lang + + def test_default_language_is_english(self): + """Test that default language is English.""" + config = AnonymisePseudonymizeUnstructuredConfig() + assert config.language == LanguageEnum.en + + +class TestTechniqueConfigDefaults: + """Tests for technique config defaults.""" + + def test_hash_config_default_algorithm(self): + """Test HashConfig default algorithm.""" + config = HashConfig() + assert config.algorithm == "sha256" + assert config.type == "hash" + + def test_encrypt_config_defaults(self): + """Test EncryptConfig defaults.""" + config = EncryptConfig() + assert config.type == "encrypt" + assert config.key_name == "my_key" + + def test_redact_config_defaults(self): + """Test RedactConfig defaults.""" + config = RedactConfig() + assert config.type == "redact" + + def test_replace_config_defaults(self): + """Test ReplaceConfig defaults.""" + config = ReplaceConfig() + assert config.type == "replace" + assert config.new_value == "REPLACED" + + def test_decrypt_config_defaults(self): + """Test DecryptConfig defaults.""" + config = DecryptConfig() + assert config.type == "decrypt" + assert config.key_name == "my_key" + + def test_unstructured_retain_config_defaults(self): + """Test RetainConfig defaults.""" + config = RetainConfig() + assert config.type == "retain" + + +class TestPseudoTechniqueConfigDefaults: + """Tests for PseudoTechniqueConfig defaults.""" + + def test_pseudo_technique_default_to_hash(self): + """Test PseudoTechniqueConfig defaults to hash technique.""" + config = PseudoTechniqueConfig() + # For Dagster Config, technique may be a dict with the discriminator structure + if isinstance(config.technique, dict): + # Check if it has hash configuration + assert "hash" in config.technique or config.technique.get("type") == "hash" + else: + assert config.technique.type == "hash" + + def test_unstructured_pseudo_technique_default_to_hash(self): + """Test UnstructuredPseudoTechniqueConfig defaults to hash technique.""" + config = UnstructuredPseudoTechniqueConfig() + # For Dagster Config, technique may be a dict with the discriminator structure + if isinstance(config.technique, dict): + # Check if it has hash configuration + assert "hash" in config.technique or config.technique.get("type") == "hash" + else: + assert config.technique.type == "hash" + + +class TestConfigModelIntegration: + """Integration tests for config models.""" + + def test_structured_config_with_all_technique_types(self): + """Test structured config with all technique types.""" + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=HashConfig(columns=["col1"]) + ), + PseudoTechniqueConfig( + technique=EncryptConfig(columns=["col2"], key_name="k1") + ), + PseudoTechniqueConfig( + technique=RedactConfig(columns=["col3"]) + ), + PseudoTechniqueConfig( + technique=ReplaceConfig(columns=["col4"], new_value="X") + ) + ] + ) + assert len(config.used_function) == 4 + techniques = {f.technique.type for f in config.used_function} + assert techniques == {"hash", "encrypt", "redact", "replace"} + + def test_unstructured_config_with_all_technique_types(self): + """Test unstructured config with all technique types.""" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredHashConfig(pii=[PIIEntityEnum.EMAIL.value]) + ), + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredEncryptConfig( + pii=[PIIEntityEnum.PERSON.value], + key_name="k1" + ) + ), + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredRedactConfig(pii=[PIIEntityEnum.PHONE_NUMBERS.value]) + ), + UnstructuredPseudoTechniqueConfig( + technique=UnstructuredReplaceConfig( + pii=[PIIEntityEnum.CREDIT_CARD.value], + new_value="X" + ) + ), + UnstructuredPseudoTechniqueConfig( + technique=RetainConfig(pii=[PIIEntityEnum.DATE_OF_BIRTH.value]) + ) + ] + ) + assert len(config.used_function) == 5 + techniques = {f.technique.type for f in config.used_function} + assert techniques == {"hash", "encrypt", "redact", "replace", "retain"} diff --git a/tests/field_level_pseudo_anonymisation/test_decrypt_structured.py b/tests/field_level_pseudo_anonymisation/test_decrypt_structured.py new file mode 100644 index 0000000..9ed013a --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_decrypt_structured.py @@ -0,0 +1,1090 @@ +""" +Test suite for data restoration (depseudonymization) operations. + +This test suite validates the data restoration feature against the following Acceptance Criteria: + +## Test Coverage Summary + +### Acceptance Criteria Coverage: +- AC1 (Data Restoration with Valid Key): 7 tests +- AC2 (Restoration Denial - Missing Key): 3 tests +- AC3 (Restoration Denial - Unauthorized Access): 2 tests +- AC4 (Restoration Denial - Invalid Key): 3 tests +- Additional Coverage: 3 tests + +### Test Pattern: +- Each test uses build_op_context with .model_dump() for configuration +- Tests validate dual outputs (data, metrics) +- Tests verify complete restoration of original values +- Tests validate security controls and error handling + +""" + +import pandas as pd +import pytest +from cryptography.fernet import Fernet + +from template_code_location.field_level_pseudo_anonymisation.config_models.structured_config import ( + AnonymisePseudonymizeStructuredConfig, + DepseudonymizeStructuredConfig, + EncryptConfig, + DecryptConfig, + PseudoTechniqueConfig, + DepseudoTechniqueConfig, +) + +# Import helper functions (fixtures are auto-discovered by pytest) +from .conftest import ( + run_encrypt_op, + run_decrypt_op, + clear_vault_key, + set_vault_key, + deny_vault_access, + get_vault_key, +) + + +# -------------------------------- Test Markers Configuration -------------------------------- + +# Register custom markers +pytest.mark.slow = pytest.mark.slow +pytest.mark.security = pytest.mark.security +pytest.mark.edge_case = pytest.mark.edge_case +pytest.mark.integration = pytest.mark.integration + + +# ---------------------- AC1: Data Restoration with Valid Key -------------------------------- + + +def test_ac1_restore_single_encrypted_field_with_valid_key( + sample_df, encrypt_config_single_field, decrypt_config_single_field +): + """ + AC1: Data Restoration using Secret Management Tool-Stored Decryption Key + + Scenario: Restore encrypted field with a valid key + Given: A pseudonymised dataset with encrypted email field + And: A valid decryption key stored in secret management tool + And: The participant provided the field that needs to be restored (email) + And: The participant is authorized + When: The participant requests data restoration + And: Provides the correct key name + Then: The system retrieves the key from secret management tool + And: Decrypts the dataset accurately + And: All original values are restored + And: A success message is presented to the user (via successful return) + And: The result is presented to the user + """ + # Clear any existing test key + clear_vault_key("test_restoration_key_single") + + # Step 1: Encrypt the data (pseudonymisation phase) + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Verify encryption occurred + assert not encrypted_df["email"].equals(sample_df["email"]), "Email field should be encrypted" + + # Verify key was created in Vault + key = get_vault_key("test_restoration_key_single") + assert key is not None, "Encryption key should exist in Vault" + + # Step 2: Restore the data (depseudonymisation phase) + restored_df, metrics = run_decrypt_op(decrypt_config_single_field, encrypted_df.copy()) + + # Verify restoration succeeded + assert restored_df is not None, "Restored DataFrame should not be None" + assert metrics is not None, "Metrics should not be None" + + # Verify all original values are restored exactly + assert restored_df["email"].equals( + sample_df["email"] + ), "Email field should be restored to original values" + + # Verify each individual value + for idx, (original, restored) in enumerate(zip(sample_df["email"], restored_df["email"])): + assert ( + original == restored + ), f"Row {idx}: Original '{original}' should match restored '{restored}'" + + # Verify row count preserved + assert len(restored_df) == len(sample_df), "Row count should be preserved during restoration" + + # Verify non-encrypted columns remain unchanged + assert restored_df["name"].equals( + sample_df["name"] + ), "Non-encrypted fields should remain unchanged" + assert restored_df["age"].equals( + sample_df["age"] + ), "Non-encrypted fields should remain unchanged" + assert restored_df["department"].equals( + sample_df["department"] + ), "Non-encrypted fields should remain unchanged" + + +def test_ac1_restore_multiple_encrypted_fields_with_valid_key( + sample_df, encrypt_config_multiple_fields, decrypt_config_multiple_fields +): + """ + AC1: Data Restoration of multiple encrypted fields with a valid key + + Scenario: Restore multiple encrypted fields (name, email, ssn) with a valid key + Given: A pseudonymised dataset with multiple encrypted fields + And: A valid decryption key stored in secret management tool + And: The participant provided the fields that need to be restored + When: The participant requests data restoration + Then: All specified fields are decrypted accurately + And: All original values are restored + """ + clear_vault_key("test_restoration_key_multi") + + # Encrypt multiple fields + encrypted_df, _ = run_encrypt_op(encrypt_config_multiple_fields, sample_df.copy()) + + # Verify all specified fields were encrypted + assert not encrypted_df["name"].equals(sample_df["name"]), "Name should be encrypted" + assert not encrypted_df["email"].equals(sample_df["email"]), "Email should be encrypted" + assert not encrypted_df["ssn"].equals(sample_df["ssn"]), "SSN should be encrypted" + + # Restore all encrypted fields + restored_df, _ = run_decrypt_op(decrypt_config_multiple_fields, encrypted_df.copy()) + + # Verify all fields restored to original values + assert restored_df["name"].equals( + sample_df["name"] + ), "Name field should be restored to original values" + assert restored_df["email"].equals( + sample_df["email"] + ), "Email field should be restored to original values" + assert restored_df["ssn"].equals( + sample_df["ssn"] + ), "SSN field should be restored to original values" + + # Verify non-encrypted columns remain unchanged + assert restored_df["age"].equals( + sample_df["age"] + ), "Non-encrypted fields should remain unchanged" + assert restored_df["salary"].equals( + sample_df["salary"] + ), "Non-encrypted fields should remain unchanged" + + +def test_ac1_restore_partial_fields_leaves_others_encrypted( + sample_df, encrypt_config_multiple_fields +): + """ + AC1: Partial restoration - participant specifies only some fields to restore + + Scenario: Restore only selected fields while leaving others encrypted + Given: A pseudonymised dataset with multiple encrypted fields (name, email, ssn) + And: The participant specifies only some fields to restore (e.g., only email) + When: The participant requests partial restoration + Then: Only the specified fields are decrypted + And: Other encrypted fields remain encrypted + """ + clear_vault_key("test_restoration_key_multi") + + # Encrypt multiple fields + encrypted_df, _ = run_encrypt_op(encrypt_config_multiple_fields, sample_df.copy()) + + # Create config to restore only email field + partial_decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], # Only restore email + key_name="test_restoration_key_multi", + ) + ) + ] + ) + + # Restore only email field + restored_df, _ = run_decrypt_op(partial_decrypt_config, encrypted_df.copy()) + + # Verify email is restored + assert restored_df["email"].equals( + sample_df["email"] + ), "Email field should be restored to original values" + + # Verify other fields remain encrypted (different from original) + assert not restored_df["name"].equals(sample_df["name"]), "Name field should remain encrypted" + assert not restored_df["ssn"].equals(sample_df["ssn"]), "SSN field should remain encrypted" + + +def test_ac1_restore_preserves_data_types(sample_df): + """ + AC1: Data restoration preserves original data types for all fields + + Scenario: Restore encrypted numeric and string fields + Given: A dataset with mixed data types (strings, integers, floats) + When: Fields are encrypted and then restored + Then: Original data types are preserved after restoration + """ + # Create config to encrypt mixed types + encrypt_config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + columns=["name", "age", "salary"], + key_name="test_restoration_types", + ) + ) + ] + ) + + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["name", "age", "salary"], + key_name="test_restoration_types", + ) + ) + ] + ) + + clear_vault_key("test_restoration_types") + + # Encrypt and restore + encrypted_df, _ = run_encrypt_op(encrypt_config, sample_df.copy()) + restored_df, _ = run_decrypt_op(decrypt_config, encrypted_df.copy()) + + # Verify values are restored (as strings due to encryption/decryption) + # Note: Fernet encryption/decryption converts everything to strings + # This is expected behavior - original types are preserved via string representation + assert ( + restored_df["name"].tolist() == sample_df["name"].tolist() + ), "String values should be restored" + assert ( + restored_df["age"].tolist() == sample_df["age"].astype(str).tolist() + ), "Integer values should be restored as strings" + assert ( + restored_df["salary"].tolist() == sample_df["salary"].astype(str).tolist() + ), "Float values should be restored as strings" + + +def test_ac1_restore_empty_dataframe(encrypt_config_single_field, decrypt_config_single_field): + """ + AC1: Edge case - restore an empty dataset + + Scenario: Attempt to restore an empty pseudonymised dataset + Given: An empty DataFrame with correct schema + When: Restoration is attempted + Then: Operation completes successfully without errors + And: Returns an empty DataFrame + """ + clear_vault_key("test_restoration_key_single") + + # Create empty DataFrame with same schema + empty_df = pd.DataFrame(columns=["id", "name", "email", "ssn", "age", "salary", "department"]) + + # Encrypt (should handle empty DataFrame) + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, empty_df.copy()) + + # Restore (should also handle empty DataFrame) + restored_df, metrics = run_decrypt_op(decrypt_config_single_field, encrypted_df.copy()) + + # Verify empty DataFrame returned + assert len(restored_df) == 0, "Restored DataFrame should be empty" + assert list(restored_df.columns) == list(empty_df.columns), "Column schema should be preserved" + + +def test_ac1_restore_with_special_characters( + encrypt_config_single_field, decrypt_config_single_field +): + """ + AC1: Data restoration with special characters and edge case values + + Scenario: Restore data containing special characters, unicode, etc. + Given: A dataset with special characters in string fields + When: Data is encrypted and then restored + Then: All special characters are preserved accurately + """ + clear_vault_key("test_restoration_key_single") + + # Create DataFrame with special characters + special_df = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "name": ["José García", "François Müller", "李明", "O'Brien"], + "email": [ + "josé@example.com", + "françois@example.com", + "li@example.cn", + "o'brien@example.ie", + ], + "ssn": ["123-45-6789", "234-56-7890", "345-67-8901", "456-78-9012"], + "age": [25, 30, 35, 40], + "salary": [50000.0, 60000.0, 70000.0, 80000.0], + "department": ["HR", "IT", "Finance", "IT"], + } + ) + + # Encrypt and restore + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, special_df.copy()) + restored_df, _ = run_decrypt_op(decrypt_config_single_field, encrypted_df.copy()) + + # Verify special characters preserved + assert restored_df["email"].equals( + special_df["email"] + ), "Special characters should be preserved during restoration" + + for idx, (original, restored) in enumerate(zip(special_df["email"], restored_df["email"])): + assert ( + original == restored + ), f"Row {idx}: Special characters in '{original}' should be preserved" + + +# ------------------- AC2: Restoration Denial when Key is Missing ---------------------------- + + +def test_ac2_restore_fails_when_key_missing(sample_df, encrypt_config_single_field): + """ + AC2: Restoration Denial when Decryption Key is missing + + Scenario: Attempt to restore encrypted fields when decryption key is missing + Given: A pseudonymised dataset + And: The decryption key is missing from Vault + And: The participant provides the correct key name + When: The participant attempts to restore the data + Then: The system fails the restoration request + And: Logs the failed key retrieval for auditing (via exception) + And: An error message is presented to the user + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data first + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Delete the key from Vault to simulate missing key + clear_vault_key("test_restoration_key_single") + + # Create decrypt config with missing key + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Attempt restoration - should fail with clear error + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + # Verify error message is informative + error_message = str(exc_info.value) + assert ( + "not found" in error_message.lower() or "decrypt" in error_message.lower() + ), "Error message should indicate key not found for decrypt operation" + assert ( + "test_restoration_key_single" in error_message + ), "Error message should include the key name for auditing" + + +def test_ac2_restore_fails_with_nonexistent_key_name(sample_df, encrypt_config_single_field): + """ + AC2: Restoration fails when using a key name that never existed + + Scenario: Attempt to restore with a key name that was never created + Given: A pseudonymised dataset + And: A key name that does not exist in Vault + When: The participant attempts to restore the data + Then: The system fails the restoration request with appropriate error + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data with one key + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Try to decrypt with a different, non-existent key + decrypt_config_wrong_key = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", columns=["email"], key_name="nonexistent_key_name" + ) + ) + ] + ) + + # Attempt restoration - should fail + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config_wrong_key, encrypted_df.copy()) + + error_message = str(exc_info.value) + assert "not found" in error_message.lower(), "Error message should indicate key not found" + + +def test_ac2_restore_fails_when_key_corrupted(sample_df, encrypt_config_single_field): + """ + AC2: Restoration Denial when Decryption Key is corrupted + + Scenario: Attempt to restore when key is corrupted in Vault + Given: A pseudonymised dataset + And: The decryption key is corrupted (invalid format) + When: The participant attempts to restore the data + Then: The system fails the restoration request + And: An appropriate error message is presented + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data first + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Corrupt the key by replacing it with invalid data + set_vault_key("test_restoration_key_single", "corrupted_invalid_key_data") + + # Create decrypt config + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Attempt restoration - should fail due to corrupted key + with pytest.raises(Exception) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + # Should raise either ValueError or Fernet-related exception + assert "Fernet" in str(type(exc_info.value)) or "ValueError" in str( + type(exc_info.value) + ), "Should raise Fernet or ValueError for corrupted key" + + +# ------------- AC3: Restoration Denial when Access is Unauthorized -------------------------- + + +def test_ac3_restore_fails_when_access_unauthorized(sample_df, encrypt_config_single_field): + """ + AC3: Restoration Denial when Decryption Key access is unauthorized + + Scenario: Attempt to restore encrypted fields without authorization + Given: A pseudonymised dataset + And: A decryption key in secret management tool + And: The participant is not authorized to access the key + When: The participant attempts to restore the data + Then: The system denies the participant access to the key + And: The system denies the initiation of the restoration process + And: The system logs the unauthorized access attempt (via exception) + And: An appropriate error message is presented to the user + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data first + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Set access control to deny access + deny_vault_access("test_restoration_key_single") + + # Create decrypt config + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Attempt restoration - should fail with ValueError (wrapping Forbidden) + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + # Verify error indicates access denial + error_message = str(exc_info.value) + assert ( + "access denied" in error_message.lower() or "error while reading" in error_message.lower() + ), "Error message should indicate access denial or error reading key" + assert ( + "test_restoration_key_single" in error_message + ), "Error message should include the key name for auditing" + + +def test_ac3_restore_multiple_keys_with_mixed_authorization(sample_df): + """ + AC3: Restoration with mixed authorization - some keys authorized, others not + + Scenario: Attempt to restore multiple fields where user has access to some keys but not others + Given: A pseudonymised dataset with multiple encrypted fields using different keys + And: The participant is authorized for some keys but not others + When: The participant attempts to restore all fields + Then: The system denies access when unauthorized key is encountered + """ + # Encrypt email with one key, ssn with another + encrypt_config_multi_keys = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="authorized_key" + ) + ) + ] + ) + + clear_vault_key("authorized_key") + clear_vault_key("unauthorized_key") + + # Encrypt data + encrypted_df, _ = run_encrypt_op(encrypt_config_multi_keys, sample_df.copy()) + + # Manually encrypt another field with different key (simulating separate encryption) + encrypt_config_ssn = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["ssn"], key_name="unauthorized_key" + ) + ) + ] + ) + encrypted_df, _ = run_encrypt_op(encrypt_config_ssn, encrypted_df.copy()) + + # Deny access to unauthorized_key + deny_vault_access("unauthorized_key") + + # Try to decrypt both fields + decrypt_config_both = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", columns=["email"], key_name="authorized_key" + ) + ), + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", columns=["ssn"], key_name="unauthorized_key" + ) + ), + ] + ) + + # Should fail when trying to access unauthorized_key with ValueError (wrapping Forbidden) + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config_both, encrypted_df.copy()) + + # Verify error indicates access issue with unauthorized key + error_message = str(exc_info.value) + assert ( + "access denied" in error_message.lower() or "error while reading" in error_message.lower() + ), "Error message should indicate access denial" + assert "unauthorized_key" in error_message, "Error message should mention the unauthorized key" + + +# ------------------- AC4: Restoration Denial when Key is Invalid ---------------------------- + + +def test_ac4_restore_fails_with_wrong_key(sample_df): + """ + AC4: Restoration Denial when Decryption Key is invalid + + Scenario: Attempt to restore encrypted fields with a key that doesn't match the encryption key + Given: A pseudonymised dataset encrypted with key A + And: A different valid decryption key B is stored in secret management tool + And: The participant provides key B (which is not the correct key) + And: Key B does not correspond to the fields to be restored + When: The participant attempts to restore the data + Then: The system fails the restoration request + And: Logs the failed decryption attempt for auditing (via exception) + And: An error message is presented to the user + """ + # Encrypt with one key + encrypt_config_key_a = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="encryption_key_a" + ) + ) + ] + ) + + clear_vault_key("encryption_key_a") + clear_vault_key("encryption_key_b") + + # Encrypt data with key A + encrypted_df, _ = run_encrypt_op(encrypt_config_key_a, sample_df.copy()) + + # Generate a different valid key B in Vault + different_key = Fernet.generate_key().decode() + set_vault_key("encryption_key_b", different_key) + + # Try to decrypt with key B (wrong key) + decrypt_config_key_b = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", columns=["email"], key_name="encryption_key_b" + ) + ) + ] + ) + + # Attempt restoration - should fail with InvalidToken or ValueError + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config_key_b, encrypted_df.copy()) + + # Verify error message indicates decryption failure + error_message = str(exc_info.value) + assert ( + "invalid" in error_message.lower() or "token" in error_message.lower() + ), "Error message should indicate invalid token or decryption failure" + assert ( + "encryption_key_b" in error_message + ), "Error message should include the key name for auditing" + + +def test_ac4_restore_fails_with_key_from_different_field(sample_df): + """ + AC4: Restoration fails when using a key intended for a different field + + Scenario: Attempt to restore field A using the key for field B + Given: A dataset with multiple fields encrypted with different keys + And: The participant provides the key for field B to decrypt field A + When: The participant attempts to restore field A + Then: The system fails the restoration request + """ + # Encrypt email and ssn with different keys + encrypt_config_email = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["email"], key_name="email_key") + ) + ] + ) + + encrypt_config_ssn = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["ssn"], key_name="ssn_key") + ) + ] + ) + + clear_vault_key("email_key") + clear_vault_key("ssn_key") + + # Encrypt both fields + encrypted_df, _ = run_encrypt_op(encrypt_config_email, sample_df.copy()) + encrypted_df, _ = run_encrypt_op(encrypt_config_ssn, encrypted_df.copy()) + + # Try to decrypt email field using ssn_key + decrypt_config_wrong_field = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], # Trying to decrypt email + key_name="ssn_key", # But using ssn's key + ) + ) + ] + ) + + # Should fail with InvalidToken + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config_wrong_field, encrypted_df.copy()) + + error_message = str(exc_info.value) + assert ( + "invalid" in error_message.lower() or "token" in error_message.lower() + ), "Error message should indicate invalid token" + + +def test_ac4_restore_fails_with_tampered_encrypted_data(sample_df, encrypt_config_single_field): + """ + AC4: Restoration fails when encrypted data has been tampered with + + Scenario: Attempt to restore encrypted data that has been modified + Given: A pseudonymised dataset + And: Some encrypted values have been tampered with + And: The correct decryption key is provided + When: The participant attempts to restore the data + Then: The system fails the restoration for tampered values + And: An appropriate error message is presented + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Tamper with encrypted data (modify one encrypted value) + encrypted_df.loc[0, "email"] = "tampered_invalid_encrypted_data" + + # Create decrypt config + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Attempt restoration - should fail on tampered data + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + error_message = str(exc_info.value) + assert ( + "invalid" in error_message.lower() or "token" in error_message.lower() + ), "Error message should indicate invalid token due to tampering" + + +# ---------------- Additional Edge Cases and Integration Tests ------------------------------- + + +def test_integration_full_cycle_encrypt_decrypt_multiple_operations(sample_df): + """ + Integration test: Full cycle of multiple encrypt/decrypt operations + + Scenario: Complex workflow with multiple encryption and restoration operations + Given: A dataset + When: Multiple fields are encrypted at different times + And: Fields are restored in different orders + Then: All operations complete successfully + And: Final restored data matches original + """ + # Phase 1: Encrypt email + encrypt_config_1 = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["email"], key_name="key_1") + ) + ] + ) + clear_vault_key("key_1") + encrypted_df_1, _ = run_encrypt_op(encrypt_config_1, sample_df.copy()) + + # Phase 2: Encrypt name and ssn + encrypt_config_2 = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["name", "ssn"], key_name="key_2") + ) + ] + ) + clear_vault_key("key_2") + encrypted_df_2, _ = run_encrypt_op(encrypt_config_2, encrypted_df_1.copy()) + + # Phase 3: Restore email first + decrypt_config_1 = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", columns=["email"], key_name="key_1") + ) + ] + ) + restored_df_1, _ = run_decrypt_op(decrypt_config_1, encrypted_df_2.copy()) + assert restored_df_1["email"].equals(sample_df["email"]), "Email should be restored" + + # Phase 4: Restore name and ssn + decrypt_config_2 = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", columns=["name", "ssn"], key_name="key_2") + ) + ] + ) + restored_df_2, _ = run_decrypt_op(decrypt_config_2, restored_df_1.copy()) + + # Verify all fields restored + assert restored_df_2["email"].equals(sample_df["email"]), "Email should remain restored" + assert restored_df_2["name"].equals(sample_df["name"]), "Name should be restored" + assert restored_df_2["ssn"].equals(sample_df["ssn"]), "SSN should be restored" + + +def test_restore_with_null_values(encrypt_config_single_field, decrypt_config_single_field): + """ + Edge case: Restoration of dataset with null/NaN values + + Scenario: Dataset contains null values in encrypted fields + Given: A dataset with null values in fields to be encrypted + When: Data is encrypted and then restored + Then: Null values are handled appropriately + """ + clear_vault_key("test_restoration_key_single") + + # Create DataFrame with null values + df_with_nulls = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "name": ["Alice", "Bob", None, "David"], + "email": [ + "alice@example.com", + None, + "charlie@example.com", + "david@example.com", + ], + "ssn": ["123-45-6789", "234-56-7890", "345-67-8901", None], + "age": [25, 30, 35, 40], + "salary": [50000.0, 60000.0, 70000.0, 80000.0], + "department": ["HR", "IT", "Finance", "IT"], + } + ) + + # Note: Encryption of NaN/None values will convert them to string "nan" or "None" + # This is expected behavior - Fernet encryption requires string input + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, df_with_nulls.copy()) + restored_df, _ = run_decrypt_op(decrypt_config_single_field, encrypted_df.copy()) + + # Verify non-null values are restored correctly + assert restored_df.loc[0, "email"] == "alice@example.com" + assert restored_df.loc[2, "email"] == "charlie@example.com" + assert restored_df.loc[3, "email"] == "david@example.com" + + +def test_restore_large_dataset_performance(): + """ + Performance test: Restoration of large dataset + + Scenario: Restore a large dataset with many rows + Given: A large dataset with 10,000 rows + When: Data is encrypted and then restored + Then: Operation completes without errors or timeout + And: All values are restored correctly + """ + # Create large dataset + large_df = pd.DataFrame( + { + "id": range(1, 10001), + "email": [f"user{i}@example.com" for i in range(1, 10001)], + "name": [f"User {i}" for i in range(1, 10001)], + "ssn": [f"{i:03d}-{i:02d}-{i:04d}" for i in range(1, 10001)], + "age": [20 + (i % 50) for i in range(1, 10001)], + "salary": [30000 + (i * 10) for i in range(1, 10001)], + "department": [["HR", "IT", "Finance", "Sales"][i % 4] for i in range(1, 10001)], + } + ) + + encrypt_config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="test_large_dataset" + ) + ) + ] + ) + + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", columns=["email"], key_name="test_large_dataset" + ) + ) + ] + ) + + clear_vault_key("test_large_dataset") + + # Encrypt and restore + encrypted_df, _ = run_encrypt_op(encrypt_config, large_df.copy()) + restored_df, _ = run_decrypt_op(decrypt_config, encrypted_df.copy()) + + # Verify sample of values + assert len(restored_df) == 10000, "Should restore all 10,000 rows" + assert restored_df["email"].equals(large_df["email"]), "All emails should be restored" + + # Spot check specific values + assert restored_df.loc[0, "email"] == "user1@example.com" + assert restored_df.loc[5000, "email"] == "user5001@example.com" + assert restored_df.loc[9999, "email"] == "user10000@example.com" + + +@pytest.mark.edge_case +@pytest.mark.security +def test_restore_after_key_rotation(sample_df, encrypt_config_single_field): + """ + AC4: Restoration fails after key rotation (key changed in Vault) + + Scenario: Key is rotated in Vault after encryption + Given: Data encrypted with key version 1 + And: Key is rotated to version 2 in Vault + When: Participant attempts to restore using new key version + Then: Restoration fails with clear error message + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt with original key + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Rotate key (replace with new key) + new_key = Fernet.generate_key().decode() + set_vault_key("test_restoration_key_single", new_key) + + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Should fail - key mismatch + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + assert ( + "invalid" in str(exc_info.value).lower() or "decrypt" in str(exc_info.value).lower() + ), "Should indicate invalid token due to key rotation" + + +@pytest.mark.edge_case +def test_restore_partially_encrypted_column(sample_df, encrypt_config_single_field): + """ + Edge case: Attempt to restore column where only some rows are encrypted + + Scenario: Column has mixed encrypted/plaintext values (data corruption scenario) + """ + clear_vault_key("test_restoration_key_single") + + # Encrypt data + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Corrupt by replacing some encrypted values with plaintext + encrypted_df.loc[0, "email"] = "plaintext@example.com" + encrypted_df.loc[2, "email"] = "another_plaintext@example.com" + + decrypt_config = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig( + type="decrypt", + columns=["email"], + key_name="test_restoration_key_single", + ) + ) + ] + ) + + # Should fail on plaintext values + with pytest.raises(ValueError) as exc_info: + run_decrypt_op(decrypt_config, encrypted_df.copy()) + + assert ( + "invalid" in str(exc_info.value).lower() or "decrypt" in str(exc_info.value).lower() + ), "Should indicate invalid token for plaintext values" + + +@pytest.mark.edge_case +def test_restore_with_missing_column_in_encrypted_data( + sample_df, encrypt_config_single_field, decrypt_config_single_field +): + """ + AC2: Restoration fails when specified column doesn't exist in encrypted dataset + """ + clear_vault_key("test_restoration_key_single") + + # First encrypt the sample data to create the key + encrypted_df, _ = run_encrypt_op(encrypt_config_single_field, sample_df.copy()) + + # Create encrypted DataFrame missing the 'email' column + incomplete_df = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + # Missing 'email' column that decrypt config expects + "age": [25, 30, 35], + "salary": [50000.0, 60000.0, 70000.0], + "department": ["HR", "IT", "Finance"], + } + ) + + with pytest.raises((ValueError, KeyError)) as exc_info: + run_decrypt_op(decrypt_config_single_field, incomplete_df) + + error_msg = str(exc_info.value) + assert ( + "email" in error_msg or "not present" in error_msg or "not found" in error_msg + ), f"Error should indicate missing column, got: {error_msg}" + + +@pytest.mark.integration +def test_restore_with_multiple_encryption_keys(sample_df): + """ + Integration test: Restore data encrypted with multiple different keys + + Scenario: Different fields encrypted with different keys + Given: name encrypted with key_a, email encrypted with key_b + When: Participant provides both keys for restoration + Then: Both fields are restored correctly + """ + clear_vault_key("key_a") + clear_vault_key("key_b") + + # Encrypt name with key_a + encrypt_config_name = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["name"], key_name="key_a") + ) + ] + ) + + # Encrypt email with key_b + encrypt_config_email = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["email"], key_name="key_b") + ) + ] + ) + + # Encrypt both fields + df_encrypted = sample_df.copy() + df_encrypted, _ = run_encrypt_op(encrypt_config_name, df_encrypted) + df_encrypted, _ = run_encrypt_op(encrypt_config_email, df_encrypted) + + # Decrypt name with key_a + decrypt_config_name = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", columns=["name"], key_name="key_a") + ) + ] + ) + + # Decrypt email with key_b + decrypt_config_email = DepseudonymizeStructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", columns=["email"], key_name="key_b") + ) + ] + ) + + # Restore both fields + df_restored = df_encrypted.copy() + df_restored, _ = run_decrypt_op(decrypt_config_name, df_restored) + df_restored, _ = run_decrypt_op(decrypt_config_email, df_restored) + + # Verify both fields restored + assert df_restored["name"].equals(sample_df["name"]), "Name field should be restored with key_a" + assert df_restored["email"].equals( + sample_df["email"] + ), "Email field should be restored with key_b" diff --git a/tests/field_level_pseudo_anonymisation/test_decrypt_unstructured.py b/tests/field_level_pseudo_anonymisation/test_decrypt_unstructured.py new file mode 100644 index 0000000..1ce8585 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_decrypt_unstructured.py @@ -0,0 +1,288 @@ +""" +Test suite for data restoration (depseudonymisation) of unstructured text. + +## Test Coverage Summary + +### Acceptance Criteria Coverage: +- AC1 (Data Restoration with Valid Key): 2 tests +- AC2 (Restoration Denial - Missing Key): 1 test +- AC3 (Restoration Denial - Unauthorized Access): 1 test +- AC4 (Restoration Denial - Invalid Key): 1 test +- Additional Coverage: 2 tests (edge cases) + +### Test Pattern: +- Each test uses build_op_context with .model_dump() for configuration +- Tests validate dual outputs (data, metrics) +- Tests verify complete restoration of original text +- Tests validate security controls and error handling +- Tests use descriptive names mapping to AC scenarios + +""" + +import pytest +from unittest.mock import patch +from cryptography.fernet import Fernet +from dagster import build_op_context + +from src.field_level_pseudo_anonymisation.unstructured_ops import ( + depseudonymize_unstructured, +) +from src.field_level_pseudo_anonymisation.config_models.unstructured_config import ( + DepseudonymizeUnstructuredConfig, + DecryptConfig, + DepseudoTechniqueConfig, +) + + +@pytest.fixture +def fernet_key() -> bytes: + """Generate a valid Fernet key for encryption in tests.""" + return Fernet.generate_key() + + +@pytest.fixture +def encrypted_text_data(fernet_key: bytes) -> dict: + """ + Create encrypted data for testing decryption. + + Returns a dict with: + - original_text: The unencrypted text + - encrypted_text: Text with PII values encrypted in {encrypt:...} format + """ + original_text = "My name is John Doe and my email is john.doe@example.com." + fernet = Fernet(fernet_key) + encrypted_name = fernet.encrypt(b"John Doe").decode() + encrypted_email = fernet.encrypt(b"john.doe@example.com").decode() + encrypted_text = ( + f"My name is {{encrypt:{encrypted_name}}} and my email is {{encrypt:{encrypted_email}}}." + ) + return { + "original_text": original_text, + "encrypted_text": encrypted_text, + } + + +# ---------------------- AC1: Data Restoration with Valid Key -------------------------------- + + +@patch("src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key") +def test_ac1_restore_encrypted_pii_entities_with_valid_key( + mock_create_get_key, fernet_key: bytes, encrypted_text_data: dict +): + """AC1: Restore encrypted PII entities with a valid key from secret management tool.""" + # Arrange - Mock the Vault key retrieval to return the valid key + mock_create_get_key.return_value = fernet_key + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig(technique=DecryptConfig(type="decrypt", key_name="test_key")) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act - Request data restoration + result_gen = depseudonymize_unstructured( + context, input_text=encrypted_text_data["encrypted_text"] + ) + data_output = next(result_gen) + metrics_output = next(result_gen) + + # Assert - Verify successful restoration + # 1. All original values are restored exactly + assert ( + data_output.value == encrypted_text_data["original_text"] + ), "Original text should be fully restored" + + # 2. Correct output structure + assert data_output.output_name == "data", "Output should be named 'data'" + + # 3. Metrics show correct number of restored entities + assert ( + metrics_output.value["total_depseudo_count"] == 2 + ), "Should restore 2 encrypted entities (name and email)" + + # 4. System retrieved key from secret management tool + mock_create_get_key.assert_called_once_with("decrypt", "test_key") + + +@patch("src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key") +def test_ac1_restore_multiple_pii_types_with_valid_key(mock_create_get_key, fernet_key: bytes): + """AC1: Restore multiple encrypted PII entity types (name, email, phone) with a valid key.""" + # Arrange - Create text with multiple PII types encrypted + original_text = "Contact John Doe at john.doe@example.com or call 555-1234." + fernet = Fernet(fernet_key) + encrypted_name = fernet.encrypt(b"John Doe").decode() + encrypted_email = fernet.encrypt(b"john.doe@example.com").decode() + encrypted_phone = fernet.encrypt(b"555-1234").decode() + encrypted_text = ( + f"Contact {{encrypt:{encrypted_name}}} at " + f"{{encrypt:{encrypted_email}}} or call {{encrypt:{encrypted_phone}}}." + ) + + mock_create_get_key.return_value = fernet_key + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", key_name="multi_pii_key") + ) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act + result_gen = depseudonymize_unstructured(context, input_text=encrypted_text) + data_output = next(result_gen) + metrics_output = next(result_gen) + + # Assert + assert data_output.value == original_text, "All PII types should be restored" + assert ( + metrics_output.value["total_depseudo_count"] == 3 + ), "Should restore 3 encrypted entities (name, email, phone)" + mock_create_get_key.assert_called_once_with("decrypt", "multi_pii_key") + + +# ------------------- AC2: Restoration Denial when Key is Missing ---------------------------- + + +@patch("src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key") +def test_ac2_restoration_denial_when_key_missing(mock_create_get_key, encrypted_text_data: dict): + """AC2: Deny restoration when decryption key is missing from secret management tool.""" + # Arrange - Mock Vault to indicate key is missing + mock_create_get_key.side_effect = ValueError( + "Fernet key 'non_existent_key' not found in Vault for decrypt." + ) + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", key_name="non_existent_key") + ) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act & Assert - Verify system fails the restoration request + with pytest.raises( + ValueError, + match="Fernet key 'non_existent_key' not found in Vault for decrypt.", + ) as exc_info: + list(depseudonymize_unstructured(context, input_text=encrypted_text_data["encrypted_text"])) + + # Verify error message is clear and actionable + assert "not found in Vault" in str( + exc_info.value + ), "Error message should indicate key is missing from Vault" + + # Verify system attempted to retrieve the key (logged attempt) + mock_create_get_key.assert_called_once_with("decrypt", "non_existent_key") + + +# ------------- AC3: Restoration Denial when Access is Unauthorized -------------------------- + + +@patch("src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key") +def test_ac3_restoration_denial_when_unauthorized_access( + mock_create_get_key, encrypted_text_data: dict +): + """AC3: Deny restoration when participant is not authorized to access the decryption key.""" + # Arrange - Mock Vault to deny access + mock_create_get_key.side_effect = ValueError("Access denied to secret: unauthorized_key") + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig( + technique=DecryptConfig(type="decrypt", key_name="unauthorized_key") + ) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act & Assert - Verify system denies access + with pytest.raises(ValueError, match="Access denied to secret: unauthorized_key") as exc_info: + list(depseudonymize_unstructured(context, input_text=encrypted_text_data["encrypted_text"])) + + # Verify error message clearly indicates access denial + assert "Access denied" in str( + exc_info.value + ), "Error message should clearly indicate access was denied" + + # Verify the unauthorized access attempt was logged (function was called) + mock_create_get_key.assert_called_once_with("decrypt", "unauthorized_key") + + +# ------------------- AC4: Restoration Denial when Key is Invalid ---------------------------- + + +@patch("src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key") +def test_ac4_restoration_denial_when_key_invalid(mock_create_get_key, encrypted_text_data: dict): + """AC4: Deny restoration when decryption key does not correspond to the encrypted fields.""" + # Arrange - Mock Vault to return a different (wrong) key + invalid_key = Fernet.generate_key() # A different, incorrect key + mock_create_get_key.return_value = invalid_key + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig(technique=DecryptConfig(type="decrypt", key_name="wrong_key")) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act & Assert - Verify system fails the restoration + with pytest.raises(ValueError, match="Invalid Fernet token") as exc_info: + list(depseudonymize_unstructured(context, input_text=encrypted_text_data["encrypted_text"])) + + # Verify error message indicates decryption failure + assert "Invalid Fernet token" in str( + exc_info.value + ), "Error message should indicate the key is invalid for this data" + + # Verify key was retrieved (system attempted decryption) + mock_create_get_key.assert_called_once_with("decrypt", "wrong_key") + + +# -------------------------------- Additional Edge Cases ---------------------------------------- + + +def test_depseudonymize_unstructured_no_decrypt_config(): + """Edge case: Text is returned unchanged when no decryption techniques are configured.""" + # Arrange + original_text = "This text has no {encrypt:values} to decrypt." + config = DepseudonymizeUnstructuredConfig(used_function=[]) # No techniques + context = build_op_context(op_config=config.model_dump()) + + # Act + result_gen = depseudonymize_unstructured(context, input_text=original_text) + result_output = next(result_gen) + metrics_output = next(result_gen) + + # Assert + assert ( + result_output.value == original_text + ), "Text should remain unchanged when no decryption is configured" + assert ( + metrics_output.value["total_depseudo_count"] == 0 + ), "Should report zero decryptions performed" + + +def test_depseudonymize_unstructured_empty_text(): + """Edge case: Empty input text is returned unchanged with zero decryptions performed.""" + # Arrange + empty_text = "" + config = DepseudonymizeUnstructuredConfig( + used_function=[ + DepseudoTechniqueConfig(technique=DecryptConfig(type="decrypt", key_name="test_key")) + ] + ) + context = build_op_context(op_config=config.model_dump()) + + # Act + with patch( + "src.field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key" + ) as mock_key: + mock_key.return_value = Fernet.generate_key() + result_gen = depseudonymize_unstructured(context, input_text=empty_text) + result_output = next(result_gen) + metrics_output = next(result_gen) + + # Assert + assert result_output.value == "", "Empty text should remain empty" + assert ( + metrics_output.value["total_depseudo_count"] == 0 + ), "Should report zero decryptions for empty text" diff --git a/tests/field_level_pseudo_anonymisation/test_encrypt_structured.py b/tests/field_level_pseudo_anonymisation/test_encrypt_structured.py new file mode 100644 index 0000000..b89fad3 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_encrypt_structured.py @@ -0,0 +1,1119 @@ +""" +Test suite for field-level pseudonymisation operations (encrypt technique). + +This test suite covers the encryption pseudonymisation technique for structured dataframes, +validating the following Acceptance Criteria: + +## Test Coverage Summary + +### Acceptance Criteria Coverage: +- AC1 (Supported Technique Applied Correctly): 7 tests +- AC2 (Invalid Execution Handling): 7 tests +- AC3 (DataFrame Compliance): 6 tests +- AC4 (Audit Logging - Success): 2 tests +- AC5 (Audit Logging - Failure): 3 tests +- Additional Coverage: 7 tests + +### Test Pattern: +- Each test uses build_op_context with config_to_dagster_dict for configuration +- Tests validate dual outputs (data, metrics) +- Vault access is mocked for isolation + +""" + +import pandas as pd +import pytest +from dagster import build_op_context +from cryptography.fernet import Fernet +from hvac.exceptions import InvalidPath +from unittest.mock import patch, MagicMock + +from template_code_location.field_level_pseudo_anonymisation.config_models.structured_config import ( + AnonymisePseudonymizeStructuredConfig, + EncryptConfig, + HashConfig, + PseudoTechniqueConfig, +) +from template_code_location.field_level_pseudo_anonymisation.ops import anonymize_pseudonymize_structured + +# Import helper functions (fixtures are auto-discovered by pytest) +from .conftest import ( + run_encrypt_op, + clear_vault_key, + get_vault_key, + config_to_dagster_dict, +) + + +# -------------------------------- Test Markers Configuration -------------------------------- + +# Register custom markers +pytest.mark.slow = pytest.mark.slow +pytest.mark.security = pytest.mark.security +pytest.mark.edge_case = pytest.mark.edge_case + + +# -------------------------------- Test-Specific Fixtures ---------------------------------------- + + +@pytest.fixture +def encrypt_single_column_config(): + """ + Configuration for encrypting a single column (email). + Tests basic encryption functionality. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="test_email_key" + ) + ) + ] + ) + + +@pytest.fixture +def encrypt_multiple_columns_config(): + """ + Configuration for encrypting multiple columns (name, email). + Tests encryption across multiple fields. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["name", "email"], key_name="test_multi_key" + ) + ) + ] + ) + + +@pytest.fixture +def encrypt_mixed_types_config(): + """ + Configuration for encrypting columns with different data types. + Tests that encryption handles type conversion (int, float -> string). + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + columns=["id", "age", "salary"], + key_name="test_numeric_key", + ) + ) + ] + ) + + +@pytest.fixture +def encrypt_with_unchanged_columns_config(): + """ + Configuration that encrypts some columns while leaving others unchanged. + Tests AC3 requirement for unchanged column preservation. + """ + return AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="test_partial_key" + ) + ) + ] + ) + + +# -------------------------------- Test-Specific Fixtures ---------------------------------------- + + +def test_encrypt_single_column_applied_correctly(sample_df, encrypt_single_column_config): + """ + AC1: Tests that encryption is applied correctly to a single column. + + Scenario: The system applies encryption to the 'email' field + Given: A structured dataset with an email column + And: A valid encryption configuration for the email field + When: The participant triggers the execution + Then: The email field must be transformed with Fernet encryption + And: The encrypted values must be different from the original values + And: The encrypted values must be valid Fernet tokens (decodable) + """ + # Clear any existing test key + clear_vault_key("test_email_key") + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + + # Verify output structure + assert result_df is not None, "Result DataFrame should not be None" + assert metrics is not None, "Metrics should not be None" + + # Verify email column is encrypted (values changed) + assert not result_df["email"].equals( + sample_df["email"] + ), "Email column should be encrypted (values should change)" + + # Verify all encrypted values are different from originals + for orig, enc in zip(sample_df["email"], result_df["email"]): + assert orig != enc, f"Original value '{orig}' should be encrypted" + + # Verify encrypted values are valid Fernet tokens (can be decrypted) + key = get_vault_key("test_email_key") + f = Fernet(key) + for enc_value in result_df["email"]: + decrypted = f.decrypt(enc_value.encode()).decode() + assert ( + decrypted in sample_df["email"].values + ), f"Decrypted value '{decrypted}' should match an original email" + + # Verify row count is preserved + assert len(result_df) == len(sample_df), "Row count should be preserved" + + +def test_encrypt_multiple_columns_applied_correctly(sample_df, encrypt_multiple_columns_config): + """ + AC1: Tests that encryption is applied correctly to multiple columns. + + Scenario: The system applies encryption to multiple fields (name, email) + Given: A structured dataset with name and email columns + And: A valid encryption configuration for both fields + When: The participant triggers the execution + Then: Both fields must be transformed with Fernet encryption + And: Each field uses the same encryption key (as specified) + """ + clear_vault_key("test_multi_key") + + result_df, metrics = run_encrypt_op(encrypt_multiple_columns_config, sample_df.copy()) + + # Verify both columns are encrypted + assert not result_df["name"].equals(sample_df["name"]), "Name column should be encrypted" + assert not result_df["email"].equals(sample_df["email"]), "Email column should be encrypted" + + # Verify all values are encrypted + key = get_vault_key("test_multi_key") + f = Fernet(key) + + for enc_name in result_df["name"]: + decrypted = f.decrypt(enc_name.encode()).decode() + assert decrypted in sample_df["name"].values + + for enc_email in result_df["email"]: + decrypted = f.decrypt(enc_email.encode()).decode() + assert decrypted in sample_df["email"].values + + +def test_encrypt_numeric_columns_applied_correctly(sample_df, encrypt_mixed_types_config): + """ + AC1: Tests that encryption handles numeric data types correctly. + + Scenario: The system applies encryption to numeric fields (id, age, salary) + Given: A structured dataset with integer and float columns + And: A valid encryption configuration for numeric fields + When: The participant triggers the execution + Then: Numeric values must be converted to strings and encrypted + And: Original numeric values should be recoverable via decryption + """ + clear_vault_key("test_numeric_key") + + result_df, metrics = run_encrypt_op(encrypt_mixed_types_config, sample_df.copy()) + + # Verify all numeric columns are now string type (encrypted) + assert result_df["id"].dtype == object, "Encrypted id should be object/string type" + assert result_df["age"].dtype == object, "Encrypted age should be object/string type" + assert result_df["salary"].dtype == object, "Encrypted salary should be object/string type" + + # Verify original numeric values can be recovered + key = get_vault_key("test_numeric_key") + f = Fernet(key) + + for enc_id in result_df["id"]: + decrypted = int(f.decrypt(enc_id.encode()).decode()) + assert decrypted in sample_df["id"].values + + +def test_encrypt_key_generation_on_first_use(sample_df, encrypt_single_column_config): + """ + AC1: Tests that encryption key is automatically generated and stored in Vault. + + Scenario: First-time encryption generates a key automatically + Given: A structured dataset with valid configuration + And: No encryption key exists in Vault for the specified key_name + When: The participant triggers the execution + Then: The system must generate a new Fernet key + And: Store it in Vault at the specified path + And: Use it for encryption + """ + clear_vault_key("test_email_key") + + # Verify key doesn't exist before encryption + with pytest.raises(InvalidPath): + get_vault_key("test_email_key") + + result_df, _ = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + + # Verify key was created + key = get_vault_key("test_email_key") + assert key is not None, "Encryption key should be created in Vault" + assert len(key) == 44, "Fernet key should be 44 bytes (base64 encoded 32 bytes)" + + # Verify the key works for decryption + f = Fernet(key) + for enc_email in result_df["email"]: + decrypted = f.decrypt(enc_email.encode()).decode() + assert decrypted in sample_df["email"].values + + +def test_encrypt_uses_existing_vault_key(sample_df, encrypt_single_column_config): + """ + AC1: Tests that encryption uses an existing key from Vault if present. + + Scenario: Encryption reuses existing key for consistent pseudonymisation + Given: A structured dataset + And: An encryption key already exists in Vault + When: The participant triggers the execution + Then: The system must use the existing key (not generate a new one) + And: The same input produces the same encrypted output (deterministic with same key) + """ + clear_vault_key("test_email_key") + + # First encryption - generates key + result_df_1, _ = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + key_1 = get_vault_key("test_email_key") + + # Second encryption - should use same key + result_df_2, _ = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + key_2 = get_vault_key("test_email_key") + + # Verify same key is used + assert key_1 == key_2, "Encryption should reuse existing Vault key" + + +# ----------------------- AC2: Invalid Execution Handling ------------------------------------ + + +def test_encrypt_missing_column_error(encrypt_single_column_config): + """ + AC2: Tests graceful error handling when a specified column doesn't exist. + + Scenario: The system aborts gracefully when column is missing + Given: A structured dataset + And: A configuration specifying a non-existent column + When: The participant triggers the execution + Then: The system must raise a clear ValueError + And: The error message must indicate which columns are missing + """ + df_missing_column = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "age": [25, 30, 35], + # Missing 'email' column + } + ) + + with pytest.raises(ValueError) as exc_info: + run_encrypt_op(encrypt_single_column_config, df_missing_column) + + assert "not present in the DataFrame" in str( + exc_info.value + ), "Error message should indicate missing columns" + assert "email" in str(exc_info.value), "Error message should mention the missing 'email' column" + + +def test_encrypt_empty_dataframe_handled(encrypt_single_column_config): + """ + AC2: Tests graceful handling of empty DataFrame input. + + Scenario: The system processes empty DataFrame without errors + Given: An empty structured dataset (no rows) + And: A valid encryption configuration + When: The participant triggers the execution + Then: The system must return an empty DataFrame with correct schema + And: No errors should be raised + """ + clear_vault_key("test_email_key") + + empty_df = pd.DataFrame(columns=["id", "name", "email", "age", "salary", "department"]) + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, empty_df) + + assert len(result_df) == 0, "Result should be empty" + assert "email" in result_df.columns, "Email column should exist in schema" + + +def test_encrypt_vault_connection_error(): + """ + AC2: Tests error handling when Vault is unreachable. + + Scenario: The system fails gracefully when Vault is unavailable + Given: A structured dataset with valid configuration + When: Vault service is unreachable or misconfigured + Then: The system must raise a clear error + And: The error message must indicate the Vault connection issue + + Note: This test requires Vault to be down or uses a bad URL. + For testing purposes, we simulate by using invalid credentials. + """ + # Create a mock client that raises an exception when accessing Vault + mock_client_instance = MagicMock() + mock_client_instance.secrets.kv.v2.read_secret_version.side_effect = Exception( + "Simulated Vault connection error" + ) + + with patch("hvac.Client", return_value=mock_client_instance): + df = pd.DataFrame( + { + "id": [1], + "name": ["Test"], + "email": ["test@example.com"], + "age": [30], + "salary": [50000.0], + "department": ["IT"], + } + ) + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="test_email_key" + ) + ) + ] + ) + with pytest.raises(ValueError) as exc_info: + run_encrypt_op(config, df) + + error_message = str(exc_info.value) + assert ( + "Simulated Vault connection error" in error_message + ), "Error should indicate Vault connection issue" + + +def test_encrypt_null_values_handled(encrypt_single_column_config): + """ + AC2: Tests handling of NULL/NaN values in encrypted columns. + + Scenario: The system handles null values appropriately + Given: A structured dataset with NULL values in the column to encrypt + And: A valid encryption configuration + When: The participant triggers the execution + Then: The system must process null values (encrypt "nan" string or handle appropriately) + And: Not raise an exception + """ + clear_vault_key("test_email_key") + + df_with_nulls = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "name": ["Alice", "Bob", "Charlie", "David"], + "email": ["alice@example.com", None, "charlie@example.com", pd.NA], + "age": [25, 30, 35, 40], + "salary": [50000.0, 60000.0, 70000.0, 80000.0], + "department": ["HR", "IT", "Finance", "IT"], + } + ) + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, df_with_nulls) + + # Verify execution completed without errors + assert result_df is not None + assert len(result_df) == 4 + + # Verify null values were processed (encrypted as string "None" or "nan") + key = get_vault_key("test_email_key") + f = Fernet(key) + + # The null values get converted to string "None" or "nan" before encryption + for enc_email in result_df["email"]: + decrypted = f.decrypt(enc_email.encode()).decode() + # Decrypted value should be original or string representation of null + assert decrypted in [ + "alice@example.com", + "charlie@example.com", + "None", + "nan", + "", + ] + + +def test_encrypt_duplicate_column_configuration_error(): + """ + AC2: Tests that duplicate columns across techniques are rejected. + + Scenario: Configuration validation prevents duplicate column assignments + Given: A configuration that assigns the same column to multiple techniques + When: The configuration is validated + Then: The system must raise a ValueError during configuration creation + And: The error message must indicate duplicate column assignment + """ + with pytest.raises(ValueError) as exc_info: + AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig(type="encrypt", columns=["email"], key_name="key1") + ), + PseudoTechniqueConfig( + technique=HashConfig( + type="hash", + columns=["email"], # Duplicate column + algorithm="sha256", + ) + ), + ] + ) + + assert "Duplicate column" in str( + exc_info.value + ), "Error should indicate duplicate column configuration" + + +# ------------------ AC3: DataFrame Input and Output Compliance ------------------------------ + + +def test_encrypt_dataframe_input_output_format(sample_df, encrypt_single_column_config): + """ + AC3: Tests that input and output are both pandas DataFrames. + + Scenario: The system accepts DataFrame input and returns DataFrame output + Given: A structured dataset as pandas DataFrame + And: A valid encryption configuration + When: The participant triggers the execution + Then: The system must return a pandas DataFrame + And: The DataFrame structure must be preserved + """ + clear_vault_key("test_email_key") + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + + # Verify output is a DataFrame + assert isinstance(result_df, pd.DataFrame), "Output must be a pandas DataFrame" + + # Verify DataFrame structure preserved + assert list(result_df.columns) == list(sample_df.columns), "Column names should be preserved" + assert len(result_df) == len(sample_df), "Row count should be preserved" + + +def test_encrypt_data_types_transformed_correctly(sample_df, encrypt_mixed_types_config): + """ + AC3: Tests that data types are transformed appropriately after encryption. + + Scenario: Encrypted columns change to string type + Given: A structured dataset with various data types (int, float, str) + And: An encryption configuration for multiple columns + When: The participant triggers the execution + Then: All encrypted columns must be of type object/string + And: This transformation is valid and consistent with encryption technique + """ + clear_vault_key("test_numeric_key") + + # Store original types + original_types = sample_df.dtypes.to_dict() + + result_df, _ = run_encrypt_op(encrypt_mixed_types_config, sample_df.copy()) + + # Verify encrypted columns are now object/string type + assert result_df["id"].dtype == object, "Encrypted integer column should become object type" + assert result_df["age"].dtype == object, "Encrypted integer column should become object type" + assert result_df["salary"].dtype == object, "Encrypted float column should become object type" + + # Verify data types changed (not same as original) + assert result_df["id"].dtype != original_types["id"], "Data type should change after encryption" + + +def test_encrypt_unchanged_columns_preserved(sample_df, encrypt_with_unchanged_columns_config): + """ + AC3: Tests that columns not specified for encryption remain unchanged. + + Scenario: Non-encrypted columns remain identical + Given: A structured dataset with multiple columns + And: An encryption configuration for only one column (email) + When: The participant triggers the execution + Then: Columns not specified (id, name, age, salary, department) must remain unchanged + And: Their values and data types must be identical to the input + """ + clear_vault_key("test_partial_key") + + result_df, _ = run_encrypt_op(encrypt_with_unchanged_columns_config, sample_df.copy()) + + # Verify unchanged columns are identical + assert result_df["id"].equals(sample_df["id"]), "ID column should remain unchanged" + assert result_df["name"].equals(sample_df["name"]), "Name column should remain unchanged" + assert result_df["age"].equals(sample_df["age"]), "Age column should remain unchanged" + assert result_df["salary"].equals(sample_df["salary"]), "Salary column should remain unchanged" + assert result_df["department"].equals( + sample_df["department"] + ), "Department column should remain unchanged" + + # Verify encrypted column is changed + assert not result_df["email"].equals( + sample_df["email"] + ), "Email column should be encrypted (changed)" + + +def test_encrypt_schema_consistency(sample_df, encrypt_multiple_columns_config): + """ + AC3: Tests that DataFrame schema is consistent and coherent. + + Scenario: Output DataFrame has consistent schema + Given: A structured dataset + And: A multi-column encryption configuration + When: The participant triggers the execution + Then: Output DataFrame must have same column names as input + And: Column order must be preserved + And: No columns should be added or removed + """ + clear_vault_key("test_multi_key") + + result_df, _ = run_encrypt_op(encrypt_multiple_columns_config, sample_df.copy()) + + # Verify column names are identical + assert list(result_df.columns) == list(sample_df.columns), "Column names must be identical" + + # Verify column order is preserved + for i, col in enumerate(sample_df.columns): + assert result_df.columns[i] == col, f"Column order should be preserved at position {i}" + + # Verify no extra columns added + assert len(result_df.columns) == len( + sample_df.columns + ), "Number of columns should remain the same" + + +def test_encrypt_index_preservation(sample_df, encrypt_single_column_config): + """ + AC3: Tests that DataFrame index is preserved after encryption. + + Scenario: DataFrame index remains unchanged + Given: A structured dataset with default index + And: A valid encryption configuration + When: The participant triggers the execution + Then: The output DataFrame must preserve the original index + And: No extraneous index column should be added + """ + clear_vault_key("test_email_key") + + # Set custom index to verify preservation + sample_df_with_index = sample_df.copy() + sample_df_with_index.index = [10, 20, 30, 40, 50] + + result_df, _ = run_encrypt_op(encrypt_single_column_config, sample_df_with_index) + + # Verify index is preserved + assert list(result_df.index) == list( + sample_df_with_index.index + ), "DataFrame index should be preserved" + + +# ------------- AC4: Execution Audit & Logging - Positive Scenario --------------------------- + + +def test_encrypt_successful_execution_logging(sample_df, encrypt_single_column_config): + """ + AC4: Tests that successful execution produces appropriate logs/metadata. + + Scenario: Successful pseudonymisation execution is logged + Given: A structured dataset with valid configuration + When: The participant triggers the execution + And: The execution completes successfully + Then: The system must return metrics output + And: Metrics should confirm successful operation + + Note: Dagster automatically logs: + - Timestamp of execution (run start/end times) + - Workflow run identifier (run_id) + - Configuration parameters (captured in op_config) + - Success status (run status in Dagster UI) + + This test validates the op returns proper outputs for Dagster to log. + """ + clear_vault_key("test_email_key") + + op_config_dict = config_to_dagster_dict(encrypt_single_column_config) + context = build_op_context(op_config=op_config_dict) + + # Capture run context information + run_id = context.run_id + + # Execute the operation + result_df, metrics = anonymize_pseudonymize_structured(context, df=sample_df.copy()) + + # Verify outputs for logging + assert result_df is not None, "Data output should be present for logging" + assert metrics is not None, "Metrics output should be present for logging" + assert isinstance(metrics.value, dict), "Metrics should be a dict" + + # Verify run context is available (Dagster provides this automatically) + assert run_id is not None, "Run ID should be available for audit logging" + + # Verify configuration is captured (can be logged) + assert "used_function" in op_config_dict, "Configuration should be captured for audit" + # In Dagster format, technique is nested under the discriminator key + technique_config = op_config_dict["used_function"][0]["technique"] + assert "encrypt" in technique_config, "Encrypt technique should be present" + assert ( + technique_config["encrypt"]["key_name"] == "test_email_key" + ), "Key name should be logged (but not key value)" + + # Verify no PII is in metrics (compliance requirement) + metrics_str = str(metrics.value) + for email in sample_df["email"]: + assert email not in metrics_str, "PII values should not appear in metrics/logs" + + +def test_encrypt_configuration_parameters_logged(sample_df, encrypt_multiple_columns_config): + """ + AC4: Tests that configuration parameters are properly captured for audit. + + Scenario: Configuration details are available for compliance logging + Given: A multi-column encryption configuration + When: The participant triggers the execution + Then: The system must capture configuration parameters including: + - Selected technique (encrypt) + - Columns to encrypt + - Key name (but not key value) + And: These parameters should be accessible for audit logging + """ + clear_vault_key("test_multi_key") + + op_config_dict = config_to_dagster_dict(encrypt_multiple_columns_config) + context = build_op_context(op_config=op_config_dict) + + result_df, metrics = anonymize_pseudonymize_structured(context, df=sample_df.copy()) + + # Verify configuration details are captured + technique_config = op_config_dict["used_function"][0]["technique"] + assert "encrypt" in technique_config, "Encrypt technique should be present" + assert set(technique_config["encrypt"]["columns"]) == {"name", "email"} + assert technique_config["encrypt"]["key_name"] == "test_multi_key" + + # Verify encryption key itself is NOT in config (security) + config_str = str(op_config_dict) + try: + key = get_vault_key("test_multi_key") + assert ( + key.decode() not in config_str + ), "Encryption key value should never be in logged configuration" + except Exception: + pass # Key might not exist yet + + +# ------------- AC5: Execution Audit & Logging - Negative Scenario --------------------------- + + +def test_encrypt_failed_execution_logging(encrypt_single_column_config): + """ + AC5: Tests that failed execution provides error details for audit. + + Scenario: Failed pseudonymisation execution is logged with error details + Given: A structured dataset with valid configuration + When: The participant triggers the execution + And: The execution fails (e.g., missing column) + Then: The system must raise an exception with clear error message + And: The error message should indicate the failure reason + And: Configuration parameters should still be accessible for audit + And: No PII should be exposed in error messages + """ + df_missing_column = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + # Missing 'email' column - will cause failure + } + ) + + op_config_dict = config_to_dagster_dict(encrypt_single_column_config) + context = build_op_context(op_config=op_config_dict) + run_id = context.run_id + + # Execute and capture failure + with pytest.raises(ValueError) as exc_info: + # Need to consume the generator to trigger execution + list(anonymize_pseudonymize_structured(context, df=df_missing_column)) + + # Verify error details are available for logging + error_message = str(exc_info.value) + assert ( + "not present in the DataFrame" in error_message + ), "Error message should explain failure reason" + assert "email" in error_message, "Error message should mention the problematic column" + + # Verify run context is available for failure logging + assert run_id is not None, "Run ID should be available for failure audit" + + # Verify configuration is still accessible for audit + assert op_config_dict is not None, "Configuration should be accessible for failure audit" + + # Verify no actual data values in error message (PII protection) + for name in ["Alice", "Bob", "Charlie"]: + assert name not in error_message, "PII values should not appear in error messages" + + +def test_encrypt_stack_trace_available_on_failure(encrypt_single_column_config): + """ + AC5: Tests that stack trace is available for debugging failed executions. + + Scenario: Failed execution provides stack trace for troubleshooting + Given: A configuration that will cause failure + When: The execution fails + Then: Python exception with stack trace should be raised + And: Stack trace should be available for logging (Dagster captures this) + And: Stack trace should not contain PII values + """ + df_missing_column = pd.DataFrame({"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}) + + try: + run_encrypt_op(encrypt_single_column_config, df_missing_column) + pytest.fail("Should have raised ValueError") + except ValueError: + # Verify exception information is available + import traceback + + stack_trace = traceback.format_exc() + + assert "ValueError" in stack_trace, "Exception type should be in stack trace" + assert ( + "not present in the DataFrame" in stack_trace + ), "Error message should be in stack trace" + + # Verify stack trace contains code location + assert ( + "ops.py" in stack_trace or "anonymize_pseudonymize_structured" in stack_trace + ), "Stack trace should indicate error location" + + +def test_encrypt_vault_error_logged_appropriately(sample_df): + """ + AC5: Tests that Vault-related errors are logged with appropriate detail. + + Scenario: Vault connection/authentication errors are captured + Given: A configuration with invalid Vault setup + When: The execution attempts to access Vault + And: Vault access fails + Then: The system must raise an error with Vault-specific details + And: The error should indicate the Vault-related nature of the failure + + Note: This test validates error handling structure; actual Vault errors + depend on Vault availability. + """ + # Create a mock client that raises an exception when accessing Vault + mock_client_instance = MagicMock() + mock_client_instance.secrets.kv.v2.read_secret_version.side_effect = Exception( + "Simulated Vault authentication error" + ) + + with patch("hvac.Client", return_value=mock_client_instance): + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["email"], key_name="test_email_key" + ) + ) + ] + ) + with pytest.raises(ValueError) as exc_info: + run_encrypt_op(config, sample_df) + + error_message = str(exc_info.value) + assert ( + "Simulated Vault authentication error" in error_message + ), "Error should indicate Vault-related failure" + + +# --------------- Additional Edge Cases & Integration Tests ---------------------------------- + + +def test_encrypt_large_dataset_performance(encrypt_single_column_config): + """ + Additional test: Validates encryption works with larger datasets. + + Tests that encryption scales to realistic dataset sizes without errors. + """ + clear_vault_key("test_email_key") + + # Create a larger dataset (1000 rows) + large_df = pd.DataFrame( + { + "id": range(1000), + "name": [f"Person{i}" for i in range(1000)], + "email": [f"person{i}@example.com" for i in range(1000)], + "age": [25 + (i % 50) for i in range(1000)], + "salary": [50000.0 + (i * 100) for i in range(1000)], + "department": ["HR", "IT", "Finance"] * 333 + ["HR"], + } + ) + + # Save original values for comparison + original_emails = large_df["email"].copy() + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, large_df) + + assert len(result_df) == 1000, "All rows should be processed" + assert not result_df["email"].equals(original_emails), "All email values should be encrypted" + + +def test_encrypt_special_characters_in_data(encrypt_single_column_config): + """ + Additional test: Validates encryption handles special characters correctly. + + Tests that encryption works with unicode, special chars, emojis, etc. + """ + clear_vault_key("test_email_key") + + df_special = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "name": ["Müller", "José", "李明", "🙂 John"], + "email": [ + "test@müller.de", + "josé@example.com", + "李明@example.cn", + "emoji@😀.com", + ], + "age": [25, 30, 35, 40], + "salary": [50000.0, 60000.0, 70000.0, 80000.0], + "department": ["HR", "IT", "Finance", "IT"], + } + ) + + # Save original values for comparison + original_emails = df_special["email"].copy().tolist() + + result_df, metrics = run_encrypt_op(encrypt_single_column_config, df_special) + + # Verify special characters are encrypted and recoverable + key = get_vault_key("test_email_key") + f = Fernet(key) + + decrypted_emails = [f.decrypt(enc.encode()).decode() for enc in result_df["email"]] + assert set(decrypted_emails) == set( + original_emails + ), "Special characters should be preserved through encryption/decryption" + + +def test_encrypt_deterministic_within_session(sample_df, encrypt_single_column_config): + """ + Additional test: Validates encryption produces consistent results with same key. + + Note: Fernet encryption includes a timestamp, so it's NOT deterministic. + This test validates that decryption recovers the original value consistently. + """ + clear_vault_key("test_email_key") + + # First encryption + result_df_1, _ = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + + # Get the key used + key = get_vault_key("test_email_key") + f = Fernet(key) + + # Verify first encryption decrypts correctly + decrypted_1 = [f.decrypt(enc.encode()).decode() for enc in result_df_1["email"]] + assert decrypted_1 == sample_df["email"].tolist(), "Decryption should recover original values" + + # Second encryption with same key (different encrypted values due to timestamp) + result_df_2, _ = run_encrypt_op(encrypt_single_column_config, sample_df.copy()) + + # Verify second encryption also decrypts correctly + decrypted_2 = [f.decrypt(enc.encode()).decode() for enc in result_df_2["email"]] + assert ( + decrypted_2 == sample_df["email"].tolist() + ), "Decryption should consistently recover original values" + + # Note: Encrypted values will be different due to Fernet's timestamp + assert not result_df_1["email"].equals( + result_df_2["email"] + ), "Fernet encryption includes timestamp, so outputs differ" + + +def test_encrypt_empty_string_values(encrypt_single_column_config): + """ + Additional test: Validates encryption handles empty strings correctly. + """ + clear_vault_key("test_email_key") + + df_empty_strings = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "", "Charlie"], + "email": ["alice@example.com", "", "charlie@example.com"], + "age": [25, 30, 35], + "salary": [50000.0, 60000.0, 70000.0], + "department": ["HR", "IT", "Finance"], + } + ) + + result_df, _ = run_encrypt_op(encrypt_single_column_config, df_empty_strings) + + # Verify empty strings are encrypted + key = get_vault_key("test_email_key") + f = Fernet(key) + + decrypted_emails = [f.decrypt(enc.encode()).decode() for enc in result_df["email"]] + assert "" in decrypted_emails, "Empty strings should be encrypted and recoverable" + + +@pytest.mark.edge_case +def test_encrypt_very_long_strings(encrypt_single_column_config): + """ + Edge case: Encryption of very long string values (e.g., 10KB+) + + Validates that Fernet encryption handles large strings without truncation. + """ + clear_vault_key("test_email_key") + + # Create DataFrame with very long strings + long_string = "x" * 10000 # 10KB string + df_long_strings = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "email": [ + f"{long_string}@example.com", + "bob@example.com", + "charlie@example.com", + ], + "age": [25, 30, 35], + "salary": [50000.0, 60000.0, 70000.0], + "department": ["HR", "IT", "Finance"], + } + ) + + result_df, _ = run_encrypt_op(encrypt_single_column_config, df_long_strings) + + # Verify long string is encrypted and recoverable + key = get_vault_key("test_email_key") + f = Fernet(key) + decrypted = f.decrypt(result_df.loc[0, "email"].encode()).decode() + assert ( + decrypted == f"{long_string}@example.com" + ), "Very long strings should be encrypted and recoverable" + + +@pytest.mark.edge_case +def test_encrypt_column_with_all_identical_values(encrypt_single_column_config): + """ + Edge case: Encryption when all values in a column are identical + + Validates that encryption produces different outputs for identical inputs + (due to Fernet's timestamp-based nonce). + """ + clear_vault_key("test_email_key") + + df_identical = pd.DataFrame( + { + "id": [1, 2, 3, 4, 5], + "name": ["Alice"] * 5, + "email": ["same@example.com"] * 5, # All identical + "age": [30] * 5, + "salary": [60000.0] * 5, + "department": ["IT"] * 5, + } + ) + + result_df, _ = run_encrypt_op(encrypt_single_column_config, df_identical) + + # Verify all encrypted values are unique (due to Fernet timestamp) + encrypted_values = result_df["email"].tolist() + assert ( + len(set(encrypted_values)) == 5 + ), "Fernet should produce unique ciphertexts even for identical plaintexts" + + # Verify all decrypt to same original value + key = get_vault_key("test_email_key") + f = Fernet(key) + decrypted_values = [f.decrypt(enc.encode()).decode() for enc in encrypted_values] + assert all( + val == "same@example.com" for val in decrypted_values + ), "All encrypted values should decrypt to same original" + + +@pytest.mark.edge_case +def test_encrypt_whitespace_only_values(encrypt_single_column_config): + """ + Edge case: Encryption of whitespace-only values + """ + clear_vault_key("test_email_key") + + df_whitespace = pd.DataFrame( + { + "id": [1, 2, 3], + "name": ["Alice", "Bob", "Charlie"], + "email": [" ", "\t\t", "\n\n"], # Various whitespace + "age": [25, 30, 35], + "salary": [50000.0, 60000.0, 70000.0], + "department": ["HR", "IT", "Finance"], + } + ) + + # Store original values before encryption + original_emails = df_whitespace["email"].tolist() + + result_df, _ = run_encrypt_op(encrypt_single_column_config, df_whitespace) + + # Verify whitespace values are encrypted and recoverable + key = get_vault_key("test_email_key") + f = Fernet(key) + encrypted_emails = result_df["email"].tolist() + + for orig_ws, enc_val in zip(original_emails, encrypted_emails): + decrypted = f.decrypt(enc_val.encode()).decode() + assert ( + decrypted == orig_ws + ), f"Whitespace value {repr(orig_ws)} should be preserved, but got {repr(decrypted)}" + + +@pytest.mark.edge_case +@pytest.mark.parametrize( + "column_type,test_values", + [ + ("integer", [1, 2, 3, 4, 5]), + ("float", [1.1, 2.2, 3.3, 4.4, 5.5]), + ("string", ["a", "b", "c", "d", "e"]), + ], +) +def test_encrypt_various_data_types(column_type, test_values): + """ + Parameterized test: Encryption across different pandas data types + """ + clear_vault_key("test_type_key") + + df = pd.DataFrame( + { + "id": range(len(test_values)), + "test_column": test_values, + "name": ["Person"] * len(test_values), + "email": ["test@example.com"] * len(test_values), + "age": [30] * len(test_values), + "salary": [60000.0] * len(test_values), + "department": ["IT"] * len(test_values), + } + ) + + config = AnonymisePseudonymizeStructuredConfig( + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", columns=["test_column"], key_name="test_type_key" + ) + ) + ] + ) + + result_df, _ = run_encrypt_op(config, df) + + # Verify encryption occurred (values changed to strings) + assert ( + result_df["test_column"].dtype == object + ), f"Encrypted {column_type} should become object type" + + # Verify decryption recovers original values + key = get_vault_key("test_type_key") + f = Fernet(key) + for idx, orig_val in enumerate(test_values): + decrypted = f.decrypt(result_df.loc[idx, "test_column"].encode()).decode() + assert decrypted == str( + orig_val + ), f"Decrypted value should match original {column_type} value" diff --git a/tests/field_level_pseudo_anonymisation/test_encrypt_unstructured.py b/tests/field_level_pseudo_anonymisation/test_encrypt_unstructured.py new file mode 100644 index 0000000..8d6a3cc --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_encrypt_unstructured.py @@ -0,0 +1,853 @@ +""" +Test suite for field-level pseudonymisation operations on unstructured data. + +This test suite validates the pseudonymisation of unstructured text with PII detection, +covering the following Acceptance Criteria: + +## Test Coverage Summary + +### Acceptance Criteria Coverage: +- AC1 (Pseudonymisation and Retention Applied Correctly): 8 tests +- AC2 (Invalid Execution Handling): 5 tests +- AC3 (Execution Audit & Logging - Positive Scenario): 3 tests +- AC4 (Execution Audit & Logging - Negative Scenario): 4 tests +- Additional Coverage: 3 tests + +### Test Pattern: +- Each test uses build_op_context with config_to_dagster_dict for configuration +- Tests validate dual outputs (data, metrics) +- Vault access is mocked for isolation +- Tests validate Scrubadub automatic PII detection +- Tests ensure placeholder replacement for unconfigured PII +""" + +import pytest +import re +from dagster import build_op_context +from unittest.mock import patch, MagicMock + +from template_code_location.field_level_pseudo_anonymisation.config_models.unstructured_config import ( + AnonymisePseudonymizeUnstructuredConfig, + EncryptConfig, + RetainConfig, + PseudoTechniqueConfig, +) +from template_code_location.field_level_pseudo_anonymisation.config_models import PIIEntityEnum, LanguageEnum +from template_code_location.field_level_pseudo_anonymisation.unstructured_ops import ( + anonymize_pseudonymize_unstructured, +) + +from .conftest import clear_vault_key + + +def config_to_dagster_dict_unstructured(config): + """Convert unstructured config to Dagster format.""" + config_dict = {"language": config.language.value, "used_function": []} + + for func_config in config.used_function: + technique = func_config.technique + technique_type = technique.type + technique_dict = technique.model_dump() + + if "pii" in technique_dict: + technique_dict["pii"] = [pii_enum.name for pii_enum in technique.pii] + + technique_dict_without_type = {k: v for k, v in technique_dict.items() if k != "type"} + + config_dict["used_function"].append( + {"technique": {technique_type: technique_dict_without_type}} + ) + + return config_dict + + +def run_unstructured_op(config, text): + """ + Helper to run unstructured pseudonymisation op. + + Returns: + tuple: (result_text: str, metrics_markdown: str) + """ + context = build_op_context(op_config=config_to_dagster_dict_unstructured(config)) + result_text, metrics = anonymize_pseudonymize_unstructured(context, text=text) + + # Extract actual values from Output objects + return result_text.value, metrics.value + + +def parse_metrics_markdown(metrics_md: str) -> dict: + """ + Parse markdown metrics into structured dict for easier testing. + + Args: + metrics_md: Markdown metrics string from op output + + Returns: + dict with keys: total_pii_detected, pii_by_type, techniques_applied, language + """ + result = { + "total_pii_detected": 0, + "pii_by_type": {}, + "techniques_applied": {}, + "language": "", + } + + # Extract total PII detected + total_match = re.search(r"\*\*Total PII Detected\*\*:\s*(\d+)", metrics_md) + if total_match: + result["total_pii_detected"] = int(total_match.group(1)) + + # Extract language + lang_match = re.search(r"\*\*Language\*\*:\s*(\w+)", metrics_md) + if lang_match: + result["language"] = lang_match.group(1) + + # Extract PII by type from table + pii_table_section = re.search( + r"### PII by Type\n\| Entity Type \| Count \|\n\|[^\n]+\n((?:\|[^\n]+\n)+)", + metrics_md, + ) + if pii_table_section: + for line in pii_table_section.group(1).strip().split("\n"): + parts = [p.strip() for p in line.split("|") if p.strip()] + if len(parts) == 2: + entity_type, count = parts + result["pii_by_type"][entity_type] = int(count) + + # Extract techniques applied + techniques_section = re.search(r"### Techniques Applied\n((?:- \*\*[^\n]+\n)+)", metrics_md) + if techniques_section: + for line in techniques_section.group(1).strip().split("\n"): + tech_match = re.match(r"-\s*\*\*(.+?)\*\*:\s*(.+)", line) + if tech_match: + pii_type, technique = tech_match.groups() + result["techniques_applied"][pii_type] = technique + + return result + + +# -------------------------------- Fixtures ---------------------------------------- + + +@pytest.fixture +def sample_text_en(): + """English text with various PII types.""" + return """ + John Smith works at Acme Corporation. His email is john.smith@example.com + and his phone number is +1-555-123-4567. He lives in New York City at + 123 Main Street, Apartment 4B. His SSN is 123-45-6789. + """ + + +@pytest.fixture +def sample_text_multi_person(): + """Text with multiple person names.""" + return """ + The meeting included Alice Johnson, Bob Williams, and Charlie Brown. + They discussed the project with Maria Garcia and David Wilson. + """ + + +@pytest.fixture +def sample_text_mixed_pii(): + """Text with multiple PII types for AC1 comprehensive testing.""" + return """ + Contact Information: + Name: Dr. Emily Watson + Email: emily.watson@hospital.com + Phone: +44-20-7946-0958 + Website: https://patient-portal.hospital.com/records + """ + + +@pytest.fixture +def encrypt_person_config(): + """Configuration to encrypt PERSON entities.""" + return AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_person_key", + ) + ) + ], + ) + + +@pytest.fixture +def retain_person_config(): + """Configuration to retain PERSON entities unchanged.""" + return AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig(technique=RetainConfig(type="retain", pii=[PIIEntityEnum.PERSON])) + ], + ) + + +@pytest.fixture +def mixed_technique_config(): + """Configuration with encryption and retention for AC1 testing.""" + return AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON, PIIEntityEnum.EMAIL], + key_name="test_mixed_key", + ) + ), + PseudoTechniqueConfig( + technique=RetainConfig(type="retain", pii=[PIIEntityEnum.PHONE_NUMBERS]) + ), + ], + ) + + +# ================================================================================================ +# AC1: Pseudonymisation and Retention Are Applied Correctly +# ================================================================================================ + + +def test_ac1_encrypt_configured_pii_types(sample_text_mixed_pii, encrypt_person_config): + """AC1: Test that configured PII types are encrypted correctly.""" + clear_vault_key("test_person_key") + + result_text, metrics_md = run_unstructured_op(encrypt_person_config, sample_text_mixed_pii) + metrics = parse_metrics_markdown(metrics_md) + + # Verify person name is encrypted (not in plaintext) + assert "Emily Watson" not in result_text, "Configured PERSON PII should be encrypted" + + # Verify encryption token is present + assert "{encrypt:" in result_text, "Encrypted token should be present in result" + + # Verify PII was detected and processed + assert metrics["total_pii_detected"] > 0, "System should detect PII entities" + assert "PERSON" in metrics["pii_by_type"], "PERSON type should be in detected PII" + + # Verify text structure is preserved (surrounding text intact) + assert "Contact Information:" in result_text, "Non-PII text structure should be preserved" + + +def test_ac1_retain_configured_pii_unchanged(sample_text_multi_person): + """AC1: Test that PII types marked for retention remain unchanged.""" + retain_config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig(technique=RetainConfig(type="retain", pii=[PIIEntityEnum.PERSON])) + ], + ) + + result_text, metrics_md = run_unstructured_op(retain_config, sample_text_multi_person) + metrics = parse_metrics_markdown(metrics_md) + + # Verify retained PII types remain in plaintext + assert "Alice Johnson" in result_text, "Retained PERSON PII should remain unchanged" + assert "Bob Williams" in result_text, "Retained PERSON PII should remain unchanged" + + # Verify technique applied is 'retain' + assert ( + "retain" in metrics["techniques_applied"].get("PERSON", "").lower() + ), "Retain technique should be recorded for PERSON type" + + +def test_ac1_unconfigured_pii_replaced_with_placeholders(sample_text_mixed_pii): + """AC1: Test that unconfigured PII types are replaced with placeholders.""" + encrypt_person_only = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_person_only_key", + ) + ) + ], + ) + + clear_vault_key("test_person_only_key") + + result_text, metrics_md = run_unstructured_op(encrypt_person_only, sample_text_mixed_pii) + + # Verify person is encrypted (configured) + assert "Emily Watson" not in result_text, "Configured PERSON should be encrypted" + + # Verify unconfigured PII types have placeholders + assert ( + "{{" in result_text and "}}" in result_text + ), "Unconfigured PII should be replaced with placeholders" + + # Verify original unconfigured PII values are not in result + assert ( + "emily.watson@hospital.com" not in result_text + ), "Unconfigured EMAIL should be replaced with placeholder" + + # Verify placeholder format + assert ( + "{{EMAIL}}" in result_text or "{{URL}}" in result_text + ), "Placeholders should indicate entity type" + + +def test_ac1_mixed_techniques_applied_correctly(sample_text_mixed_pii, mixed_technique_config): + """AC1: Test that multiple techniques (encrypt, retain) are applied correctly.""" + clear_vault_key("test_mixed_key") + + result_text, metrics_md = run_unstructured_op(mixed_technique_config, sample_text_mixed_pii) + metrics = parse_metrics_markdown(metrics_md) + + # Verify encrypted PII types (PERSON, EMAIL) + assert "Emily Watson" not in result_text, "Configured PERSON should be encrypted" + assert "emily.watson@hospital.com" not in result_text, "Configured EMAIL should be encrypted" + + # Verify retained PII type (PHONE_NUMBERS) + assert "+44-20-7946-0958" in result_text, "Configured PHONE_NUMBERS should be retained" + + # Verify metrics reflect different techniques + assert ( + "encrypt" in metrics["techniques_applied"].get("PERSON", "").lower() + ), "Encrypt technique should be applied to PERSON" + assert ( + "encrypt" in metrics["techniques_applied"].get("EMAIL", "").lower() + ), "Encrypt technique should be applied to EMAIL" + assert ( + "retain" in metrics["techniques_applied"].get("PHONE_NUMBERS", "").lower() + ), "Retain technique should be applied to PHONE_NUMBERS" + + +def test_ac1_multiple_instances_same_pii_type(sample_text_multi_person, encrypt_person_config): + """AC1: Test that all instances of a configured PII type are processed.""" + clear_vault_key("test_person_key") + + result_text, metrics_md = run_unstructured_op(encrypt_person_config, sample_text_multi_person) + metrics = parse_metrics_markdown(metrics_md) + + # Verify all person names are encrypted + person_names = [ + "Alice Johnson", + "Bob Williams", + "Charlie Brown", + "Maria Garcia", + "David Wilson", + ] + for name in person_names: + assert name not in result_text, f"All PERSON instances should be encrypted: {name}" + + # Verify metrics count multiple instances + assert metrics["pii_by_type"].get("PERSON", 0) >= len( + person_names + ), f"Should detect at least {len(person_names)} PERSON entities" + + +def test_ac1_empty_text_returns_empty(encrypt_person_config): + """AC1: Test that empty or null text input raises a ValueError.""" + clear_vault_key("test_person_key") + + with pytest.raises(ValueError) as exc_info: + run_unstructured_op(encrypt_person_config, "") + + assert "empty" in str(exc_info.value).lower(), "Error should indicate empty input" + + +def test_ac1_text_without_pii_remains_unchanged(): + """AC1: Test that text without any PII remains unchanged after processing.""" + no_pii_text = """ + The weather today is sunny with a high of 25 degrees Celsius. + The conference starts at 9:00 AM in Room 301. + """ + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_no_pii_key", + ) + ) + ], + ) + + clear_vault_key("test_no_pii_key") + + result_text, metrics_md = run_unstructured_op(config, no_pii_text) + metrics = parse_metrics_markdown(metrics_md) + + assert result_text.strip() == no_pii_text.strip(), "Text without PII should remain unchanged" + assert metrics["total_pii_detected"] == 0, "No PII should be detected" + + +def test_ac1_placeholder_format_indicates_entity_type(sample_text_mixed_pii): + """AC1: Test that placeholders for unconfigured PII indicate the entity type.""" + encrypt_person_only = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_placeholder_key", + ) + ) + ], + ) + + clear_vault_key("test_placeholder_key") + + result_text, metrics_md = run_unstructured_op(encrypt_person_only, sample_text_mixed_pii) + metrics = parse_metrics_markdown(metrics_md) + + # Verify placeholder format (scrubadub uses {{TYPE}} format) + placeholder_pattern = r"\{\{[A-Z_]+\}\}" + placeholders = re.findall(placeholder_pattern, result_text) + + assert ( + len(placeholders) > 0 + ), "Result should contain entity-type placeholders for unconfigured PII" + + # Verify metrics track which PII types were detected + assert len(metrics["pii_by_type"]) > 0, "Metrics should list detected PII types" + + +# ================================================================================================ +# AC2: Invalid Execution Handling +# ================================================================================================ + + +def test_ac2_graceful_abort_on_scrubadub_failure(): + """AC2: Test graceful abort when the PII detection engine (Scrubadub) fails.""" + text = "Test user John Smith with email john@example.com" + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_abort_key", + ) + ) + ], + ) + + clear_vault_key("test_abort_key") + + # Mock Scrubadub to fail at the right import path + with patch( + "field_level_pseudo_anonymisation.unstructured_ops.scrubadub.Scrubber" + ) as mock_scrubber_class: + mock_scrubber = MagicMock() + mock_scrubber.iter_filth.side_effect = RuntimeError("Scrubadub internal error") + mock_scrubber_class.return_value = mock_scrubber + + with pytest.raises(RuntimeError) as exc_info: + run_unstructured_op(config, text) + + error_msg = str(exc_info.value).lower() + assert ( + "pii" in error_msg + or "detection" in error_msg + or "scrubadub" in error_msg + or "failed" in error_msg + ), "Error message should indicate PII detection failure" + + +def test_ac2_graceful_abort_on_encryption_failure(sample_text_en): + """AC2: Test graceful abort when an encryption technique fails during execution.""" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_encrypt_fail_key", + ) + ) + ], + ) + + clear_vault_key("test_encrypt_fail_key") + + # Mock encrypt function at correct path - it's imported from techniques module + encrypt_path = ( + "field_level_pseudo_anonymisation" + ".techniques.anonymisation_pseudonymisation_techniques.encrypt" + ) + with patch(encrypt_path) as mock_encrypt: + mock_encrypt.side_effect = Exception("Encryption algorithm failure") + + with pytest.raises(RuntimeError) as exc_info: + run_unstructured_op(config, sample_text_en) + + error_msg = str(exc_info.value).lower() + assert ( + "encrypt" in error_msg or "failed" in error_msg or "technique" in error_msg + ), "Error message should indicate encryption failure" + + +def test_ac2_null_text_input_raises_error(encrypt_person_config): + """AC2: Test that a null (None) text input is rejected with an error.""" + clear_vault_key("test_person_key") + + # Dagster will raise DagsterTypeCheckDidNotPass before op executes + from dagster import DagsterTypeCheckDidNotPass + + with pytest.raises((ValueError, DagsterTypeCheckDidNotPass, TypeError)): + run_unstructured_op(encrypt_person_config, None) + + +def test_ac2_invalid_language_configuration(): + """AC2: Test that an unsupported language in the config raises a validation error.""" + # This should fail at config creation due to Pydantic validation + with pytest.raises((ValueError, TypeError)): + AnonymisePseudonymizeUnstructuredConfig( + language="invalid_lang", # Should fail Pydantic validation + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", pii=[PIIEntityEnum.PERSON], key_name="test_key" + ) + ) + ], + ) + + +def test_ac2_very_large_text_processing(): + """AC2: Test that very large text inputs are processed successfully without memory errors.""" + # Create large text with repeated PII patterns + large_text = ( + """ + John Smith works at company. Email: john.smith@example.com. + """ + * 1000 + ) # ~60KB of text with repeated PII + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON, PIIEntityEnum.EMAIL], + key_name="test_large_text_key", + ) + ) + ], + ) + + clear_vault_key("test_large_text_key") + + result_text, metrics_md = run_unstructured_op(config, large_text) + metrics = parse_metrics_markdown(metrics_md) + + # Verify processing completed + assert result_text is not None, "Large text should be processed successfully" + assert len(result_text) > 0, "Result should not be empty" + assert metrics["total_pii_detected"] > 0, "PII should be detected in large text" + + +# ================================================================================================ +# AC3: Execution Audit & Logging - Positive Scenario +# ================================================================================================ + + +def test_ac3_successful_execution_logs_timestamp_and_run_id(sample_text_en, encrypt_person_config): + """AC3: Test that successful execution context contains a run ID for logging.""" + clear_vault_key("test_person_key") + + op_config_dict = config_to_dagster_dict_unstructured(encrypt_person_config) + context = build_op_context(op_config=op_config_dict) + + # Capture run context + run_id = context.run_id + + # Execute operation + result_text, metrics = anonymize_pseudonymize_unstructured(context, text=sample_text_en) + + # Verify run identifier is available for logging + assert run_id is not None, "Run ID must be available for audit logging" + + # Verify outputs are returned (for Dagster to log) + assert result_text is not None, "Result text should be available for logging" + assert metrics is not None, "Metrics should be available for logging" + + +def test_ac3_successful_execution_logs_configuration_parameters( + sample_text_en, mixed_technique_config +): + """AC3: Test that the used configuration is accessible for logging on success.""" + clear_vault_key("test_mixed_key") + + op_config_dict = config_to_dagster_dict_unstructured(mixed_technique_config) + context = build_op_context(op_config=op_config_dict) + + result_text, metrics = anonymize_pseudonymize_unstructured(context, text=sample_text_en) + + # Verify configuration is captured and accessible + assert "used_function" in op_config_dict, "Configuration must be accessible for logging" + assert len(op_config_dict["used_function"]) == 2, "Multiple techniques should be captured" + + # Verify techniques are logged + techniques = [func["technique"] for func in op_config_dict["used_function"]] + assert any( + "encrypt" in str(tech) for tech in techniques + ), "Encrypt technique should be in configuration" + assert any( + "retain" in str(tech) for tech in techniques + ), "Retain technique should be in configuration" + + # Verify metrics contain technique information (in markdown string) + metrics_str = metrics.value + assert ( + "Techniques Applied" in metrics_str + ), "Applied techniques should be in metrics for logging" + + +def test_ac3_successful_execution_logs_no_raw_pii(sample_text_mixed_pii, encrypt_person_config): + """AC3: Test that logs and metrics from a successful run do not contain raw PII.""" + clear_vault_key("test_person_key") + + op_config_dict = config_to_dagster_dict_unstructured(encrypt_person_config) + context = build_op_context(op_config=op_config_dict) + + result_text, metrics = anonymize_pseudonymize_unstructured(context, text=sample_text_mixed_pii) + + # Verify raw PII values are not in metrics + metrics_str = metrics.value + + sensitive_values = ["Emily Watson", "emily.watson@hospital.com", "+44-20-7946-0958"] + + for pii_value in sensitive_values: + assert ( + pii_value not in metrics_str + ), f"Raw PII value should not appear in metrics: {pii_value}" + + # Verify configuration logs do not contain raw PII + config_str = str(op_config_dict) + for pii_value in sensitive_values: + assert ( + pii_value not in config_str + ), f"Raw PII value should not appear in configuration logs: {pii_value}" + + +# ================================================================================================ +# AC4: Execution Audit & Logging - Negative Scenario +# ================================================================================================ + + +def test_ac4_failed_execution_logs_error_details(): + """AC4: Negative execution should surface clear error details (encryption key failure).""" + text = "Test user John Smith" + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_fail_log_key", + ) + ) + ], + ) + clear_vault_key("test_fail_log_key") + ctx = build_op_context(op_config=config_to_dagster_dict_unstructured(config)) + + # Patch the key retrieval used inside unstructured_ops to force failure + with patch( + "field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key", + side_effect=RuntimeError("Encryption key retrieval failed"), + ): + with pytest.raises(RuntimeError) as exc_info: + # Consume the generator to trigger execution and raise the exception + list(anonymize_pseudonymize_unstructured(ctx, text=text)) + + msg = str(exc_info.value).lower() + assert "key" in msg and "failed" in msg, "Error message should mention key failure" + + +def test_ac4_failed_execution_logs_configuration_used(): + """AC4: Test that the attempted configuration is available for logging on failure.""" + text = "Test data with person John Doe" + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_config_fail_key", + ) + ) + ], + ) + + clear_vault_key("test_config_fail_key") + + op_config_dict = config_to_dagster_dict_unstructured(config) + context = build_op_context(op_config=op_config_dict) + + # Mock _initialize_scrubber to fail + with patch( + "field_level_pseudo_anonymisation.unstructured_ops._initialize_scrubber" + ) as mock_init_scrubber: + mock_init_scrubber.side_effect = Exception("Scrubber module not available") + + with pytest.raises((RuntimeError, Exception)) as exc_info: + list(anonymize_pseudonymize_unstructured(context, text=text)) + + # Verify configuration is still accessible despite failure + assert op_config_dict is not None, "Configuration must be accessible for failure audit" + assert ( + "used_function" in op_config_dict + ), "Technique configuration should be available for diagnosis" + + # Verify error was raised with proper message + error_msg = str(exc_info.value).lower() + assert ( + "pii" in error_msg + or "detection" in error_msg + or "failed" in error_msg + or "scrubber" in error_msg + or "module" in error_msg + ), "Error should indicate detection/processing failed" + + +def test_ac4_failed_execution_logs_failure_reason(): + """AC4: Test that the reason for a failure is clearly indicated in the error message.""" + text = "User: Alice Smith, Email: alice@example.com" + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.en, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON, PIIEntityEnum.EMAIL], + key_name="test_failure_reason_key", + ) + ) + ], + ) + + clear_vault_key("test_failure_reason_key") + + # Mock key retrieval function to fail + with patch( + "field_level_pseudo_anonymisation.unstructured_ops.create_get_encryption_key" + ) as mock_get_key: + mock_get_key.side_effect = RuntimeError("Vault connection timeout") + + with pytest.raises(RuntimeError) as exc_info: + run_unstructured_op(config, text) + + # Verify failure reason is in error message + error_msg = str(exc_info.value).lower() + assert ( + "encrypt" in error_msg + or "key" in error_msg + or "timeout" in error_msg + or "failed" in error_msg + ), "Error should indicate key retrieval/encryption failure" + + +# ================================================================================================ +# Additional Tests - Edge Cases and Integration +# ================================================================================================ + + +def test_multi_language_support_italian(): + """Additional test: Verify that Italian text is processed correctly.""" + italian_text = """ + Il dottor Marco Rossi lavora presso l'ospedale. + Email: marco.rossi@ospedale.it + Telefono: +39-06-12345678 + """ + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.it, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON], + key_name="test_italian_key", + ) + ) + ], + ) + + clear_vault_key("test_italian_key") + + result_text, metrics_md = run_unstructured_op(config, italian_text) + metrics = parse_metrics_markdown(metrics_md) + + # Verify processing occurred + assert result_text != italian_text, "Italian text should be processed" + assert metrics["total_pii_detected"] > 0, "PII should be detected in Italian text" + + +def test_special_characters_in_text(): + """Additional test: Verify handling of text with special Unicode characters.""" + special_text = """ + User: João da Silva 🇧🇷 + Email: joão@empresa.com.br + Message: "Hello, World!" — Testing special chars: €, £, ¥, ©, ® + """ + + config = AnonymisePseudonymizeUnstructuredConfig( + language=LanguageEnum.pt, + used_function=[ + PseudoTechniqueConfig( + technique=EncryptConfig( + type="encrypt", + pii=[PIIEntityEnum.PERSON, PIIEntityEnum.EMAIL], + key_name="test_special_chars_key", + ) + ) + ], + ) + + clear_vault_key("test_special_chars_key") + + result_text, metrics_md = run_unstructured_op(config, special_text) + + # Verify processing completed without encoding errors + assert result_text is not None, "Special characters should not cause processing failure" + assert len(result_text) > 0, "Result should not be empty" + + +def test_deterministic_encryption_within_session(sample_text_en, encrypt_person_config): + """Additional test: Verify encryption format consistency across runs.""" + clear_vault_key("test_person_key") + + result1, metrics_md1 = run_unstructured_op(encrypt_person_config, sample_text_en) + result2, metrics_md2 = run_unstructured_op(encrypt_person_config, sample_text_en) + + # Both should have encryption tokens + assert "{encrypt:" in result1, "First run should produce encrypted tokens" + assert "{encrypt:" in result2, "Second run should produce encrypted tokens" + + # Verify consistent PII detection + metrics1 = parse_metrics_markdown(metrics_md1) + metrics2 = parse_metrics_markdown(metrics_md2) + + assert ( + metrics1["total_pii_detected"] == metrics2["total_pii_detected"] + ), "PII detection should be consistent across runs" + + # Verify token format is consistent (Fernet base64 pattern) + token_pattern = r"\{encrypt:gAAAAAB[A-Za-z0-9+/=_-]+\}" + tokens1 = re.findall(token_pattern, result1) + tokens2 = re.findall(token_pattern, result2) + + assert len(tokens1) == len(tokens2), "Same number of encryption tokens should be generated" diff --git a/tests/field_level_pseudo_anonymisation/test_jobs.py b/tests/field_level_pseudo_anonymisation/test_jobs.py new file mode 100644 index 0000000..616c3d5 --- /dev/null +++ b/tests/field_level_pseudo_anonymisation/test_jobs.py @@ -0,0 +1,58 @@ +from template_code_location.field_level_pseudo_anonymisation.jobs import ( + anonymize_pseudonymize_structured_job, + anonymize_pseudonymize_structured_job_s3, + depseudonymize_structured_job, + depseudonymize_structured_job_s3, + anonymize_pseudonymize_unstructured_job_s3, + anonymize_pseudonymize_unstructured_job, + depseudonymize_unstructured_job_s3, + depseudonymize_unstructured_job +) + + +def test_anonymize_pseudonymize_structured_job_is_callable(): + """Test anonymize_pseudonymize_structured_job is a valid Dagster job""" + assert callable(anonymize_pseudonymize_structured_job) + assert hasattr(anonymize_pseudonymize_structured_job, 'execute_in_process') + + +def test_anonymize_pseudonymize_structured_job_s3_is_callable(): + """Test anonymize_pseudonymize_structured_job_s3 is a valid Dagster job""" + assert callable(anonymize_pseudonymize_structured_job_s3) + assert hasattr(anonymize_pseudonymize_structured_job_s3, 'execute_in_process') + + +def test_depseudonymize_structured_job_is_callable(): + """Test depseudonymize_structured_job is a valid Dagster job""" + assert callable(depseudonymize_structured_job) + assert hasattr(depseudonymize_structured_job, 'execute_in_process') + + +def test_depseudonymize_structured_job_s3_is_callable(): + """Test depseudonymize_structured_job_s3 is a valid Dagster job""" + assert callable(depseudonymize_structured_job_s3) + assert hasattr(depseudonymize_structured_job_s3, 'execute_in_process') + + +def test_anonymize_pseudonymize_unstructured_job_is_callable(): + """Test anonymize_pseudonymize_unstructured_job is a valid Dagster job""" + assert callable(anonymize_pseudonymize_unstructured_job) + assert hasattr(anonymize_pseudonymize_unstructured_job, 'execute_in_process') + + +def test_anonymize_pseudonymize_unstructured_job_s3_is_callable(): + """Test anonymize_pseudonymize_unstructured_job_s3 is a valid Dagster job""" + assert callable(anonymize_pseudonymize_unstructured_job_s3) + assert hasattr(anonymize_pseudonymize_unstructured_job_s3, 'execute_in_process') + + +def test_depseudonymize_unstructured_job_is_callable(): + """Test depseudonymize_unstructured_job is a valid Dagster job""" + assert callable(depseudonymize_unstructured_job) + assert hasattr(depseudonymize_unstructured_job, 'execute_in_process') + + +def test_depseudonymize_unstructured_job_s3_is_callable(): + """Test depseudonymize_unstructured_job_s3 is a valid Dagster job""" + assert callable(depseudonymize_unstructured_job_s3) + assert hasattr(depseudonymize_unstructured_job_s3, 'execute_in_process')