4e0b216410
- Rename src/template-code-location to src/template_code_location - Copy data-processing jobs/ops/config_models - Copy dataframe-level-anonymisation jobs/ops/utils/config_models - Copy field-level-pseudo-anonymisation jobs/ops/techniques/config_models - Update all imports to template_code_location namespace - Merge all jobs into unified repository.py with sensors/resources/loggers - Update pyproject.toml with all dependencies - Update Dockerfile for consolidated image
55 lines
1.3 KiB
TOML
55 lines
1.3 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68.0", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "template-code-location"
|
|
version = "0.1.0"
|
|
description = "Consolidated code location for all data services workflows"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
# Dagster core
|
|
"dagster>=1.8.13",
|
|
"dagster-webserver>=1.8.13",
|
|
"dagster-postgres>=0.24.13",
|
|
# Data processing
|
|
"pandas>=2.1.4",
|
|
"pyarrow>=23.0",
|
|
"numpy>=2.4",
|
|
"lxml>=6.0",
|
|
"xmltodict>=1.0",
|
|
"rdflib>=7.6",
|
|
"openpyxl",
|
|
"xlrd>=2.0.1",
|
|
"tabulate==0.8.10",
|
|
"pyspellchecker>=0.8.4",
|
|
"PyGeodesy>=24.6.11",
|
|
# Validation
|
|
"great_expectations>=1.16",
|
|
"pandera>=0.31",
|
|
"pydantic>=2.6.0,<3.0.0",
|
|
# Scraping
|
|
"scrapy>=2.15",
|
|
"BeautifulSoup4>=4.14",
|
|
# Anonymisation libraries
|
|
"pycanon==1.0.1.post2",
|
|
"anjana>=1.0.0",
|
|
# Field-level pseudo-anonymisation
|
|
"scrubadub",
|
|
"scrubadub_spacy",
|
|
"hvac",
|
|
"cryptography",
|
|
# Util services (git dependency)
|
|
"util-services @ git+https://code.europa.eu/simpl/simpl-open/development/data-services/util-services.git@v0.4.1",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=8.0.0",
|
|
"pytest-cov>=7.0.0",
|
|
"pytest-mock>=3.0.0"
|
|
]
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["src"]
|