701 lines
27 KiB
Python
701 lines
27 KiB
Python
"""Unit tests for data processing operations."""
|
|
|
|
import pytest
|
|
import pandas as pd
|
|
from template_code_location.data_processing.ops import (
|
|
remove_duplicates,
|
|
fill_missing_values,
|
|
standardize_categorical_values,
|
|
correct_typos,
|
|
normalize_datetime,
|
|
normalize_numeric_min_max,
|
|
normalize_coordinates,
|
|
add_global_aggregations
|
|
)
|
|
from template_code_location.data_processing.config_models import (
|
|
FillMissingConfiguration,
|
|
ColumnsSelectConfiguration,
|
|
SpellCheckConfiguration,
|
|
AggregationConfiguration,
|
|
CoordinatesNormalizationConfiguration
|
|
)
|
|
|
|
|
|
class TestRemoveDuplicates:
|
|
"""Tests for the remove_duplicates operation."""
|
|
|
|
def test_remove_duplicates_basic(self, mock_context, sample_dataframe):
|
|
"""Test basic duplicate removal."""
|
|
result = remove_duplicates(mock_context, sample_dataframe)
|
|
|
|
# Should have 3 unique rows (john doe appears 3x, jane smith 1x, bob johnson 1x)
|
|
assert result.shape[0] == 3
|
|
assert len(result) < len(sample_dataframe)
|
|
|
|
def test_remove_duplicates_no_duplicates(self, mock_context):
|
|
"""Test remove_duplicates when there are no duplicates."""
|
|
df = pd.DataFrame({
|
|
'A': [1, 2, 3],
|
|
'B': ['x', 'y', 'z']
|
|
})
|
|
result = remove_duplicates(mock_context, df)
|
|
|
|
assert result.shape[0] == 3
|
|
pd.testing.assert_frame_equal(result, df)
|
|
|
|
def test_remove_duplicates_all_duplicates(self, mock_context):
|
|
"""Test remove_duplicates when all rows are identical."""
|
|
df = pd.DataFrame({
|
|
'A': [1, 1, 1],
|
|
'B': ['x', 'x', 'x']
|
|
})
|
|
result = remove_duplicates(mock_context, df)
|
|
|
|
assert result.shape[0] == 1
|
|
|
|
def test_remove_duplicates_empty_dataframe(self, mock_context, empty_dataframe):
|
|
"""Test remove_duplicates with empty DataFrame."""
|
|
result = remove_duplicates(mock_context, empty_dataframe)
|
|
|
|
assert result.shape[0] == 0
|
|
assert result.shape[1] == 0
|
|
|
|
def test_remove_duplicates_preserves_data_types(self, mock_context):
|
|
"""Test that remove_duplicates preserves data types."""
|
|
df = pd.DataFrame({
|
|
'int_col': [1, 2, 1],
|
|
'str_col': ['a', 'b', 'a'],
|
|
'float_col': [1.5, 2.5, 1.5]
|
|
})
|
|
result = remove_duplicates(mock_context, df)
|
|
|
|
assert result['int_col'].dtype == df['int_col'].dtype
|
|
assert result['str_col'].dtype == df['str_col'].dtype
|
|
assert result['float_col'].dtype == df['float_col'].dtype
|
|
|
|
|
|
class TestFillMissingValues:
|
|
"""Tests for the fill_missing_values operation."""
|
|
|
|
def test_fill_missing_values_basic(self, mock_context, dataframe_with_missing_values):
|
|
"""Test basic missing value filling."""
|
|
config = FillMissingConfiguration(fill_map={'Column1': '0', 'Column2': 'N/A'})
|
|
result = fill_missing_values(mock_context, config, dataframe_with_missing_values)
|
|
|
|
# Check that no NaN values remain
|
|
assert result['Column1'].isna().sum() == 0
|
|
assert result['Column2'].isna().sum() == 0
|
|
|
|
def test_fill_missing_values_with_different_values(self, mock_context):
|
|
"""Test filling with different replacement values."""
|
|
df = pd.DataFrame({
|
|
'A': [1, None, 3],
|
|
'B': [None, 'b', 'c']
|
|
})
|
|
config = FillMissingConfiguration(fill_map={'A': '-1', 'B': 'UNKNOWN'})
|
|
result = fill_missing_values(mock_context, config, df)
|
|
|
|
assert result.loc[1, 'A'] == '-1'
|
|
assert result.loc[0, 'B'] == 'UNKNOWN'
|
|
|
|
def test_fill_missing_values_partial_columns(self, mock_context):
|
|
"""Test filling only specified columns."""
|
|
df = pd.DataFrame({
|
|
'A': [1, None, 3],
|
|
'B': [None, 'b', 'c']
|
|
})
|
|
config = FillMissingConfiguration(fill_map={'A': '999'})
|
|
result = fill_missing_values(mock_context, config, df)
|
|
|
|
assert result.loc[1, 'A'] == '999'
|
|
assert pd.isna(result.loc[0, 'B']) # B should still have NaN
|
|
|
|
def test_fill_missing_values_no_missing(self, mock_context):
|
|
"""Test when there are no missing values."""
|
|
df = pd.DataFrame({
|
|
'A': ['1', '2', '3'],
|
|
'B': ['a', 'b', 'c']
|
|
})
|
|
config = FillMissingConfiguration(fill_map={'A': '0'})
|
|
result = fill_missing_values(mock_context, config, df)
|
|
|
|
pd.testing.assert_frame_equal(result, df)
|
|
|
|
def test_fill_missing_values_empty_dataframe(self, mock_context, empty_dataframe):
|
|
"""Test with empty DataFrame."""
|
|
config = FillMissingConfiguration(fill_map={})
|
|
result = fill_missing_values(mock_context, config, empty_dataframe)
|
|
|
|
assert result.shape[0] == 0
|
|
|
|
|
|
class TestStandardizeCategoricalValues:
|
|
"""Tests for the standardize_categorical_values operation."""
|
|
|
|
def test_standardize_categorical_basic(self, mock_context, sample_dataframe):
|
|
"""Test basic categorical standardization."""
|
|
config = ColumnsSelectConfiguration(columns=['Name', 'City', 'Status'])
|
|
result = standardize_categorical_values(mock_context, config, sample_dataframe)
|
|
|
|
# Check that values are lowercase and stripped
|
|
assert result['Name'].iloc[0] == 'john doe'
|
|
assert result['City'].iloc[1] == 'los angeles'
|
|
assert result['Status'].iloc[1] == 'inactive'
|
|
|
|
def test_standardize_categorical_single_column(self, mock_context):
|
|
"""Test standardization on a single column."""
|
|
df = pd.DataFrame({
|
|
'City': [' NEW YORK ', 'LOS ANGELES', ' chicago ']
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['City'])
|
|
result = standardize_categorical_values(mock_context, config, df)
|
|
|
|
assert result['City'].iloc[0] == 'new york'
|
|
assert result['City'].iloc[1] == 'los angeles'
|
|
assert result['City'].iloc[2] == 'chicago'
|
|
|
|
def test_standardize_categorical_missing_column(self, mock_context, sample_dataframe):
|
|
"""Test with non-existent column (should skip)."""
|
|
config = ColumnsSelectConfiguration(columns=['NonExistent', 'Name'])
|
|
result = standardize_categorical_values(mock_context, config, sample_dataframe)
|
|
|
|
# Should process 'Name' column without error
|
|
assert result['Name'].iloc[0] == 'john doe'
|
|
|
|
def test_standardize_categorical_with_missing_values(self, mock_context):
|
|
"""Test standardization with missing values."""
|
|
df = pd.DataFrame({
|
|
'Category': [' ACTIVE ', None, ' pending ']
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['Category'])
|
|
result = standardize_categorical_values(mock_context, config, df)
|
|
|
|
assert result['Category'].iloc[0] == 'active'
|
|
assert result['Category'].iloc[1] == ''
|
|
assert result['Category'].iloc[2] == 'pending'
|
|
|
|
def test_standardize_categorical_empty_dataframe(self, mock_context, empty_dataframe):
|
|
"""Test with empty DataFrame."""
|
|
config = ColumnsSelectConfiguration(columns=['A', 'B'])
|
|
result = standardize_categorical_values(mock_context, config, empty_dataframe)
|
|
|
|
assert result.shape[0] == 0
|
|
|
|
def test_standardize_categorical_numeric_columns(self, mock_context):
|
|
"""Test that numeric columns are converted to strings."""
|
|
df = pd.DataFrame({
|
|
'NumCol': [1, 2, 3]
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['NumCol'])
|
|
result = standardize_categorical_values(mock_context, config, df)
|
|
|
|
assert result['NumCol'].iloc[0] == '1'
|
|
assert isinstance(result['NumCol'].iloc[0], str)
|
|
|
|
|
|
class TestCorrectTypos:
|
|
"""Tests for the correct_typos operation."""
|
|
|
|
def test_correct_typos_basic(self, mock_context):
|
|
"""Test basic typo correction."""
|
|
df = pd.DataFrame({
|
|
'Name': ['jon', 'jayne', 'bob']
|
|
})
|
|
config = SpellCheckConfiguration(columns=['Name'], language='en')
|
|
result = correct_typos(mock_context, config, df)
|
|
|
|
# Result should have corrections applied
|
|
assert result.shape[0] == 3
|
|
|
|
def test_correct_typos_missing_column(self, mock_context):
|
|
"""Test with non-existent column (should skip)."""
|
|
df = pd.DataFrame({
|
|
'Name': ['jon', 'jayne']
|
|
})
|
|
config = SpellCheckConfiguration(columns=['NonExistent'], language='en')
|
|
result = correct_typos(mock_context, config, df)
|
|
|
|
# Should not raise error, just skip
|
|
pd.testing.assert_frame_equal(result, df)
|
|
|
|
def test_correct_typos_with_missing_values(self, mock_context):
|
|
"""Test typo correction with missing values."""
|
|
df = pd.DataFrame({
|
|
'Text': ['helo', '', 'wrld']
|
|
})
|
|
config = SpellCheckConfiguration(columns=['Text'], language='en')
|
|
result = correct_typos(mock_context, config, df)
|
|
|
|
# Empty strings should be preserved
|
|
assert result.loc[1, 'Text'] == ''
|
|
|
|
def test_correct_typos_empty_dataframe(self, mock_context, empty_dataframe):
|
|
"""Test with empty DataFrame."""
|
|
config = SpellCheckConfiguration(columns=['A'], language='en')
|
|
result = correct_typos(mock_context, config, empty_dataframe)
|
|
|
|
assert result.shape[0] == 0
|
|
|
|
def test_correct_typos_different_languages(self, mock_context):
|
|
"""Test typo correction with different languages."""
|
|
df = pd.DataFrame({
|
|
'Text': ['ciao', 'mondo']
|
|
})
|
|
|
|
for lang in ['en', 'es', 'it']:
|
|
config = SpellCheckConfiguration(columns=['Text'], language=lang)
|
|
result = correct_typos(mock_context, config, df)
|
|
|
|
# Should process without error
|
|
assert result.shape[0] == 2
|
|
|
|
def test_correct_typos_numeric_values(self, mock_context):
|
|
"""Test typo correction on numeric values converted to strings."""
|
|
df = pd.DataFrame({
|
|
'Values': [123, 456, 789]
|
|
})
|
|
config = SpellCheckConfiguration(columns=['Values'], language='en')
|
|
result = correct_typos(mock_context, config, df)
|
|
|
|
# Numeric values should be converted to string and processed
|
|
assert result.shape[0] == 3
|
|
|
|
class TestNormalizeDatetime:
|
|
"""Tests for the normalize_datetime operation."""
|
|
|
|
def test_normalize_datetime_basic(self, mock_context):
|
|
"""Test basic datetime normalization to ISO format."""
|
|
df = pd.DataFrame({
|
|
'date_col': ['2023-01-01 10:00:00', '2023-12-31T23:59:59']
|
|
})
|
|
|
|
config = ColumnsSelectConfiguration(columns=['date_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'date_col_iso' in result.columns
|
|
assert result['date_col_iso'].iloc[0] == '2023-01-01T10:00:00Z'
|
|
assert result['date_col_iso'].iloc[1] == '2023-12-31T23:59:59Z'
|
|
|
|
def test_normalize_datetime_missing_column(self, mock_context, sample_dataframe):
|
|
"""Test behavior when a configured column is missing in the DataFrame."""
|
|
config = ColumnsSelectConfiguration(columns=['non_existent_column'])
|
|
|
|
result = normalize_datetime(mock_context, config, sample_dataframe.copy())
|
|
|
|
pd.testing.assert_frame_equal(result, sample_dataframe)
|
|
|
|
def test_normalize_datetime_unparseable_values(self, mock_context):
|
|
"""Test column with values that cannot be parsed as dates."""
|
|
df = pd.DataFrame({
|
|
'invalid_col': ['not-a-date', 'completely-random-text']
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['invalid_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'invalid_col_iso' not in result.columns
|
|
|
|
def test_normalize_datetime_mixed_and_nulls(self, mock_context):
|
|
"""Test column with mixed valid dates, invalid dates, and NaNs."""
|
|
df = pd.DataFrame({
|
|
'mixed_col': ['2023-05-01', None, 'invalid-date']
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['mixed_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'mixed_col_iso' in result.columns
|
|
assert result['mixed_col_iso'].iloc[0] == '2023-05-01T00:00:00Z'
|
|
|
|
assert result['mixed_col_iso'].iloc[1] == ""
|
|
assert result['mixed_col_iso'].iloc[2] == ""
|
|
|
|
def test_normalize_datetime_empty_dataframe(self, mock_context, empty_dataframe):
|
|
"""Test with an empty DataFrame."""
|
|
config = ColumnsSelectConfiguration(columns=['some_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, empty_dataframe)
|
|
|
|
assert result.empty
|
|
|
|
def test_normalize_datetime_epoch_only(self, mock_context, capsys):
|
|
"""If parsing a column yields only the Unix epoch date, it should be skipped."""
|
|
df = pd.DataFrame({
|
|
'weird_col': ['0', 0, '0000', '']
|
|
})
|
|
|
|
config = ColumnsSelectConfiguration(columns=['weird_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'weird_col_iso' not in result.columns
|
|
|
|
captured = capsys.readouterr()
|
|
assert "all normalized values are '1970-01-01'" in captured.err
|
|
|
|
def test_normalize_datetime_all_1970_skipped(self, mock_context, capsys):
|
|
"""If all formatted values are '1970-01-01', the column should be skipped with a warning."""
|
|
df = pd.DataFrame({
|
|
'ts_col': ['1970-01-01 05:30:00', '1970-01-01 12:00:00']
|
|
})
|
|
|
|
config = ColumnsSelectConfiguration(columns=['ts_col'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'ts_col_iso' not in result.columns
|
|
|
|
captured = capsys.readouterr()
|
|
assert "all normalized values are '1970-01-01'" in captured.err
|
|
|
|
def test_normalize_datetime_integer_age_column_skipped(self, mock_context, capsys):
|
|
"""If an integer column like 'age' is passed, all values become 1970-01-01 and should be skipped."""
|
|
df = pd.DataFrame({
|
|
'age': [66, 45, 40, 43, 20, 26, 69, 21, 46]
|
|
})
|
|
|
|
config = ColumnsSelectConfiguration(columns=['age'])
|
|
|
|
result = normalize_datetime(mock_context, config, df.copy())
|
|
|
|
assert 'age_iso' not in result.columns
|
|
|
|
captured = capsys.readouterr()
|
|
assert "all normalized values are '1970-01-01'" in captured.err
|
|
|
|
class TestNormalizeNumericMinMax:
|
|
"""Tests for the normalize_numeric_min_max operation."""
|
|
|
|
def test_normalize_numeric_basic(self, mock_context):
|
|
"""Test standard min-max normalization between 0 and 1."""
|
|
df = pd.DataFrame({
|
|
'score': [10, 20, 30, 40, 50]
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['score'])
|
|
|
|
result = normalize_numeric_min_max(mock_context, config, df.copy())
|
|
|
|
assert 'score_norm' in result.columns
|
|
assert result['score_norm'].min() == 0.0
|
|
assert result['score_norm'].max() == 1.0
|
|
|
|
assert result['score_norm'].iloc[2] == 0.5
|
|
|
|
def test_normalize_numeric_missing_column(self, mock_context):
|
|
"""Test skipping of non-existent columns."""
|
|
df = pd.DataFrame({'existing': [1, 2, 3]})
|
|
config = ColumnsSelectConfiguration(columns=['missing_col'])
|
|
|
|
result = normalize_numeric_min_max(mock_context, config, df.copy())
|
|
|
|
assert 'missing_col_norm' not in result.columns
|
|
|
|
def test_normalize_numeric_constant_values(self, mock_context):
|
|
"""Test skipping when min == max to avoid division by zero."""
|
|
df = pd.DataFrame({
|
|
'constant': [10, 10, 10]
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['constant'])
|
|
|
|
result = normalize_numeric_min_max(mock_context, config, df.copy())
|
|
|
|
assert 'constant_norm' not in result.columns
|
|
|
|
def test_normalize_numeric_with_nans(self, mock_context):
|
|
"""Test normalization with NaN values (pandas min/max ignore NaNs by default)."""
|
|
df = pd.DataFrame({
|
|
'with_nans': [10, None, 50]
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['with_nans'])
|
|
|
|
result = normalize_numeric_min_max(mock_context, config, df.copy())
|
|
|
|
assert 'with_nans_norm' in result.columns
|
|
assert result['with_nans_norm'].iloc[0] == 0.0
|
|
assert result['with_nans_norm'].iloc[2] == 1.0
|
|
assert pd.isna(result['with_nans_norm'].iloc[1])
|
|
|
|
def test_normalize_numeric_multiple_columns(self, mock_context):
|
|
"""Test processing multiple columns in one call."""
|
|
df = pd.DataFrame({
|
|
'A': [1, 2],
|
|
'B': [10, 20]
|
|
})
|
|
config = ColumnsSelectConfiguration(columns=['A', 'B'])
|
|
|
|
result = normalize_numeric_min_max(mock_context, config, df.copy())
|
|
|
|
assert 'A_norm' in result.columns
|
|
assert 'B_norm' in result.columns
|
|
|
|
class TestNormalizeCoordinates:
|
|
"""Tests for the normalize_coordinates operation."""
|
|
|
|
def test_normalize_coordinates_basic(self, mock_context):
|
|
"""Test rounding and basic coordinate normalization."""
|
|
df = pd.DataFrame({
|
|
'lat': [45.123456, 46.0],
|
|
'lon': [9.123456, 10.0]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon')
|
|
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert result['lat'].iloc[0] == 45.1235
|
|
assert result['lon'].iloc[0] == 9.1235
|
|
|
|
assert len(result) == 2
|
|
|
|
def test_normalize_coordinates_filtering(self, mock_context):
|
|
"""Test filtering of out-of-range coordinates."""
|
|
df = pd.DataFrame({
|
|
'lat': [45.0, 100.0, -91.0, 0.0], # 100 e -91 sono out of range
|
|
'lon': [9.0, 0.0, 0.0, 200.0] # 200 è out of range
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon')
|
|
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 1
|
|
assert result['lat'].iloc[0] == 45.0
|
|
|
|
def test_normalize_coordinates_invalid_types(self, mock_context):
|
|
"""Test conversion of strings to numeric and handling of NaNs."""
|
|
df = pd.DataFrame({
|
|
'lat': ["45.5", "invalid", None],
|
|
'lon': ["9.5", "10.0", "11.0"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon')
|
|
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 1
|
|
assert isinstance(result['lat'].iloc[0], float)
|
|
|
|
def test_normalize_coordinates_empty_df(self, mock_context, empty_dataframe):
|
|
"""Test with an empty DataFrame."""
|
|
|
|
df = pd.DataFrame(columns=['lat', 'lon'])
|
|
config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon')
|
|
|
|
result = normalize_coordinates(mock_context, config, df)
|
|
|
|
assert len(result) == 0
|
|
assert result.empty
|
|
|
|
def test_normalize_coordinates_default_config(self, mock_context):
|
|
"""Test that normalize_coordinates uses default 'lat'/'lon' columns when no config is provided."""
|
|
df = pd.DataFrame({
|
|
'lat': [45.123456, 46.0],
|
|
'lon': [9.123456, 10.0]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration()
|
|
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert result['lat'].iloc[0] == 45.1235
|
|
assert result['lon'].iloc[0] == 9.1235
|
|
assert len(result) == 2
|
|
|
|
def test_normalize_coordinates_null_config_values(self, mock_context):
|
|
"""Test that null lat/lon column names fall back to defaults ('lat'/'lon')."""
|
|
df = pd.DataFrame({
|
|
'lat': [45.123456, 46.0],
|
|
'lon': [9.123456, 10.0]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(latColumn=None, lonColumn=None)
|
|
|
|
assert config.latColumn == "lat"
|
|
assert config.lonColumn == "lon"
|
|
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert result['lat'].iloc[0] == 45.1235
|
|
assert result['lon'].iloc[0] == 9.1235
|
|
assert len(result) == 2
|
|
|
|
def test_normalize_coordinates_dms_degree_symbol(self, mock_context):
|
|
"""Test DMS parsing with degree/minute/second symbols like 40°26'46\"N."""
|
|
df = pd.DataFrame({
|
|
'lat': ["40°26'46\"N", "51°30'26\"N"],
|
|
'lon': ["79°58'56\"W", "0°7'39\"W"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 2
|
|
# 40°26'46"N ≈ 40.4461
|
|
assert abs(result['lat'].iloc[0] - 40.4461) < 0.001
|
|
# 79°58'56"W ≈ -79.9822
|
|
assert abs(result['lon'].iloc[0] - (-79.9822)) < 0.001
|
|
|
|
def test_normalize_coordinates_dms_spaced_format(self, mock_context):
|
|
"""Test DMS parsing with space-separated format like '40 26 46 N'."""
|
|
df = pd.DataFrame({
|
|
'lat': ["40 26 46 N"],
|
|
'lon': ["79 58 56 W"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 1
|
|
assert abs(result['lat'].iloc[0] - 40.4461) < 0.001
|
|
assert abs(result['lon'].iloc[0] - (-79.9822)) < 0.001
|
|
|
|
def test_normalize_coordinates_dms_already_decimal(self, mock_context):
|
|
"""Test that string columns with decimal values are auto-parsed correctly."""
|
|
df = pd.DataFrame({
|
|
'lat': ["45.5", "46.0"],
|
|
'lon': ["9.5", "10.0"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 2
|
|
assert result['lat'].iloc[0] == 45.5
|
|
assert result['lon'].iloc[0] == 9.5
|
|
|
|
def test_normalize_coordinates_dms_mixed_valid_invalid(self, mock_context):
|
|
"""Test auto-detection with a mix of valid DMS, valid decimal, and unparseable values."""
|
|
df = pd.DataFrame({
|
|
'lat': ["40°26'46\"N", "not_a_coord", "51.5"],
|
|
'lon': ["79°58'56\"W", "10.0", "0.1"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
# Row with "not_a_coord" for lat should be dropped (NaN lat)
|
|
assert len(result) == 2
|
|
|
|
def test_normalize_coordinates_dms_out_of_range(self, mock_context):
|
|
"""Test that DMS-parsed coordinates outside valid range are filtered out."""
|
|
df = pd.DataFrame({
|
|
'lat': ["91°0'0\"N", "45°0'0\"N"],
|
|
'lon': ["0°0'0\"E", "9°0'0\"E"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
# First row has lat=91° which is out of [-90, 90]
|
|
assert len(result) == 1
|
|
assert abs(result['lat'].iloc[0] - 45.0) < 0.001
|
|
|
|
def test_normalize_coordinates_dms_south_and_east(self, mock_context):
|
|
"""Test DMS parsing with south latitude and east longitude."""
|
|
df = pd.DataFrame({
|
|
'lat': ["33°51'54\"S"],
|
|
'lon': ["151°12'36\"E"]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(
|
|
latColumn='lat', lonColumn='lon'
|
|
)
|
|
result = normalize_coordinates(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 1
|
|
# 33°51'54"S ≈ -33.865
|
|
assert result['lat'].iloc[0] < 0
|
|
assert abs(result['lat'].iloc[0] - (-33.865)) < 0.001
|
|
# 151°12'36"E ≈ 151.21
|
|
assert result['lon'].iloc[0] > 0
|
|
assert abs(result['lon'].iloc[0] - 151.21) < 0.01
|
|
|
|
def test_normalize_coordinates_autodetect_numeric_vs_dms(self, mock_context):
|
|
"""Test that numeric columns are coerced directly while string columns are parsed as DMS."""
|
|
# Numeric columns — should go through pd.to_numeric path
|
|
df_numeric = pd.DataFrame({
|
|
'lat': [45.123456, 46.0],
|
|
'lon': [9.123456, 10.0]
|
|
})
|
|
config = CoordinatesNormalizationConfiguration(latColumn='lat', lonColumn='lon')
|
|
result_numeric = normalize_coordinates(mock_context, config, df_numeric.copy())
|
|
|
|
assert result_numeric['lat'].iloc[0] == 45.1235
|
|
assert len(result_numeric) == 2
|
|
|
|
# String DMS columns — should go through _parse_dms_to_decimal path
|
|
df_dms = pd.DataFrame({
|
|
'lat': ["40°26'46\"N"],
|
|
'lon': ["79°58'56\"W"]
|
|
})
|
|
result_dms = normalize_coordinates(mock_context, config, df_dms.copy())
|
|
|
|
assert len(result_dms) == 1
|
|
assert abs(result_dms['lat'].iloc[0] - 40.4461) < 0.001
|
|
|
|
class TestAddGlobalAggregations:
|
|
"""Tests for the add_global_aggregations operation."""
|
|
|
|
def test_add_global_aggregations_success(self, mock_context):
|
|
"""Test a successful group by and aggregation."""
|
|
df = pd.DataFrame({
|
|
'category': ['A', 'A', 'B'],
|
|
'value': [10, 20, 100],
|
|
'ignored_str': ['x', 'y', 'z']
|
|
})
|
|
|
|
config = AggregationConfiguration(
|
|
columns=['category'],
|
|
operation='sum'
|
|
)
|
|
|
|
result = add_global_aggregations(mock_context, config, df.copy())
|
|
|
|
assert len(result) == 2
|
|
assert result.loc[result['category'] == 'A', 'value'].values[0] == 30
|
|
assert result.loc[result['category'] == 'B', 'value'].values[0] == 100
|
|
assert 'ignored_str' not in result.columns
|
|
mock_context.log.info.assert_called()
|
|
|
|
def test_add_global_aggregations_missing_column(self, mock_context):
|
|
"""Test skipping a column that does not exist in the dataframe."""
|
|
df = pd.DataFrame({'value': [1, 2, 3]})
|
|
config = AggregationConfiguration(
|
|
columns=['missing_col'],
|
|
operation='count'
|
|
)
|
|
|
|
result = add_global_aggregations(mock_context, config, df.copy())
|
|
|
|
mock_context.log.warning.assert_any_call("Column 'missing_col' not found, skipping aggregation.")
|
|
assert len(result) == 1
|
|
|
|
def test_add_global_aggregations_unsupported_op(self, mock_context):
|
|
"""Test the warning when an unsupported operation is provided."""
|
|
df = pd.DataFrame({'category': ['A'], 'value': [1]})
|
|
|
|
config = AggregationConfiguration(
|
|
columns=['category'],
|
|
operation='unsupported'
|
|
)
|
|
|
|
with pytest.raises(Exception):
|
|
add_global_aggregations(mock_context, config, df.copy())
|
|
|
|
mock_context.log.warning.assert_any_call("Unsupported aggregation 'unsupported'")
|
|
|
|
def test_add_global_aggregations_only_numeric_kept(self, mock_context):
|
|
"""Verify that non-numeric and non-grouping columns are dropped."""
|
|
df = pd.DataFrame({
|
|
'group': ['A', 'A'],
|
|
'num': [1, 2],
|
|
'text': ['hello', 'world']
|
|
})
|
|
config = AggregationConfiguration(columns=['group'], operation='mean')
|
|
|
|
result = add_global_aggregations(mock_context, config, df.copy())
|
|
|
|
assert 'text' not in result.columns
|
|
assert 'num' in result.columns
|
|
assert 'group' in result.columns
|