From 0701bdb9fdc1513bd13dcc9c47eb6d67606cd5bc Mon Sep 17 00:00:00 2001 From: Adrian Lodin Date: Fri, 29 May 2026 17:42:46 +0300 Subject: [PATCH 1/8] [SIMPL-28035](https://jira.simplprogramme.eu/browse/SIMPL-28035) Update... --- documents/Development Guide.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/documents/Development Guide.md b/documents/Development Guide.md index aa4870b..7bf1d6f 100644 --- a/documents/Development Guide.md +++ b/documents/Development Guide.md @@ -121,10 +121,13 @@ If jobs or obs are updated, check out their related `tests` and update them acco If jobs names are updated, make sure to update the import and definitions. ### 2.3.3 Workflow deletion -Before deleting an existing workflow, first check its status (if is needed or referenced) in the **Dagster UI**: -- Navigate to the "Runs". Identify a given workflow using the "Filter" button or navigation "Newer", "Older" buttons. -- Check details and status: open workflow details by clicking on the target, or just check the status value. -- Check logs: open workflow logs by clicking on the uuid. +Before deleting an existing workflow, first check if is needed or referenced in: +- **Asset Orchestrator** + - Perform an `GET/workflowDefinitions` api call, to get the workflow details. +- **Dagster UI**: + - Navigate to the "Runs". Identify a given workflow using the "Filter" button or navigation "Newer", "Older" buttons. + - Check details and status: open workflow details by clicking on the target, or just check the status value. + - Check logs: open workflow logs by clicking on the uuid. Here are two practical examples how to delete a workflow: From dc0774599dbb8bfed9c20019a822539c79ace61c Mon Sep 17 00:00:00 2001 From: Adrian LODIN Date: Fri, 29 May 2026 18:45:16 +0300 Subject: [PATCH 2/8] [SIMPL-28035](https://jira.simplprogramme.eu/browse/SIMPL-28035) Update project version. Changelog: other --- pipeline.variables.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline.variables.sh b/pipeline.variables.sh index 4a3f9c4..789e7b0 100644 --- a/pipeline.variables.sh +++ b/pipeline.variables.sh @@ -1 +1 @@ -PROJECT_VERSION_NUMBER="0.1.0" \ No newline at end of file +PROJECT_VERSION_NUMBER="0.2.1" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 64a6268..821e836 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "template-code-location" -version = "0.2.0" +version = "0.2.1" description = "Consolidated code location for all data services workflows" requires-python = ">=3.12" dependencies = [ From e168eebff3c3b51aeb52cb779e6c6abd31a88d0a Mon Sep 17 00:00:00 2001 From: Adrian LODIN Date: Wed, 3 Jun 2026 12:44:13 +0300 Subject: [PATCH 3/8] [SIMPL-28035](https://jira.simplprogramme.eu/browse/SIMPL-28035) Update project version. Changelog: other --- pipeline.variables.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline.variables.sh b/pipeline.variables.sh index 789e7b0..cd522de 100644 --- a/pipeline.variables.sh +++ b/pipeline.variables.sh @@ -1 +1 @@ -PROJECT_VERSION_NUMBER="0.2.1" \ No newline at end of file +PROJECT_VERSION_NUMBER="0.2.0" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 821e836..64a6268 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "template-code-location" -version = "0.2.1" +version = "0.2.0" description = "Consolidated code location for all data services workflows" requires-python = ">=3.12" dependencies = [ From 63025e83c880d8cf7aacd552128f94d18fa02e2b Mon Sep 17 00:00:00 2001 From: Geordi Parrillo Date: Wed, 3 Jun 2026 12:58:58 +0200 Subject: [PATCH 4/8] [SIMPL-27884](https://jira.simplprogramm.eu/browse/SIMPL-27884 Fix CHANGELOG.md Changelog: fixed --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6546fd9..1a4ddd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ ## 0.1.0 (2026-05-29) -No changes. +### added (3 changes) + +- [feat(SIMPL-24642): consolidate all code locations into template-code-location](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/4e0b216410fc0d0879b583047fc7ba1f0390c612) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/4)) +- [init](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/0ed949a4e86b4e7676deba638b55f5d924b6d8b3) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/2)) +- [Feature/simpl 24306](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/2125933d3dfb36f71ac6853d7aace92f8b2e8eeb) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/1)) From b19182d1d7b579f19fa2e76b072a40d91ff4ae73 Mon Sep 17 00:00:00 2001 From: ILay Date: Wed, 3 Jun 2026 13:38:00 +0200 Subject: [PATCH 5/8] Add initial README.md with project overview and usage instructions --- README.md | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/README.md b/README.md index e69de29..b2faf7e 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,169 @@ +# Template Code Location + +> Purpose: `template-code-location` is the predefined Application Workflow Template for Simpl-Open providers. It offers a standard Dagster code location that providers can download, extend, and package when building new application workflows, reducing onboarding effort and avoiding configuration drift. + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [What This Template Includes](#what-this-template-includes) +3. [Repository Layout](#repository-layout) +4. [Prerequisites](#prerequisites) +5. [Quick Start](#quick-start) +6. [Development Guide](#development-guide) +7. [Testing](#testing) +8. [Packaging and Deployment](#packaging-and-deployment) +9. [Documentation](#documentation) +10. [Contributing](#contributing) + +--- + +## Overview + +This repository provides a standardized baseline for Dagster code locations in the Simpl-Open ecosystem. + +It is intended for providers who need a ready-made starting point for new application workflows. Instead of creating a code location from scratch, a provider can use this template to: + +- start from an agreed repository structure +- reuse the standard Dagster packaging model +- follow the same conventions for jobs, ops, resources, and definitions +- integrate existing Simpl-Open workflow packages as dependencies +- add custom workflow logic with minimal setup effort + +The template is intentionally simple. It includes a small example job and ops that demonstrate the expected pattern for creating new workflows, while the main Dagster entrypoint already shows how to register both local jobs and jobs imported from external packages. + +## What This Template Includes + +- A working Dagster code location package under `src/template_code_location` +- A repository entrypoint in `src/template_code_location/repository.py` +- Example local ops in `src/template_code_location/ops/ops.py` +- Example local job in `src/template_code_location/jobs/jobs.py` +- Preconfigured dependencies on shared Simpl-Open workflow packages through `pyproject.toml` +- A Dockerfile for container packaging +- Supporting guidance in the `documents/` folder + +This template is not meant to contain business-specific logic by default. Providers are expected to replace or extend the sample workflow with their own application-specific jobs and operations. + +## Repository Layout + +```text +template-code-location/ +|- src/ +| \- template_code_location/ +| |- __init__.py +| |- repository.py +| |- jobs/ +| | |- __init__.py +| | \- jobs.py +| \- ops/ +| |- __init__.py +| \- ops.py +|- documents/ +|- Dockerfile +|- pyproject.toml +|- uv.lock +\- README.md +``` + +Key files: + +- `src/template_code_location/repository.py`: Dagster `Definitions` entrypoint used to register jobs, sensors, resources, and loggers +- `src/template_code_location/jobs/jobs.py`: place for custom workflow definitions +- `src/template_code_location/ops/ops.py`: place for custom operational steps used by local jobs +- `pyproject.toml`: package metadata and Git-based dependencies on shared Simpl-Open packages +- [documents/Development Guide.md](documents/Development%20Guide.md): workflow lifecycle guidance for creating, editing, and deleting workflows + +## Prerequisites + +Before using this template, make sure the following are available: + +- Python 3.12+ +- `uv` package manager +- Access to the Simpl-Open Git repositories referenced by `pyproject.toml` +- A local or shared Dagster environment for validation +- Docker, if you plan to build the image locally + +Depending on the workflows you build, you may also need access to supporting platform services such as object storage, PostgreSQL, Vault, or Kubernetes. + +## Quick Start + +### 1. Install dependencies + +```powershell +uv sync --dev +``` + +### 2. Start the Dagster code server locally + +```powershell +uv run dagster code-server start -h 0.0.0.0 -p 4000 -f src/template_code_location/repository.py +``` + +This starts the code location using the repository entrypoint already defined in the template. + +### 3. Review the sample workflow + +The template ships with: + +- `fetch_data` and `process_data` ops in `src/template_code_location/ops/ops.py` +- `data_processing_job` in `src/template_code_location/jobs/jobs.py` + +These are example building blocks only. Providers should rename, replace, or extend them to match the target application workflow. + +## Development Guide + +Workflow creation, update, deletion, and the expected way to work with external code locations are covered in [documents/Development Guide.md](documents/Development%20Guide.md). + +Use that guide as the primary reference when customizing this template for a provider-specific workflow. + +## Testing + +Custom workflows should always be validated before publication. + +Recommended checks: + +- add unit tests for local jobs and ops +- run `pytest` locally after every workflow change +- verify that the Dagster code server can load `src/template_code_location/repository.py` without import or definition errors + +Example command: + +```powershell +uv run pytest +``` + +At the moment this template does not include a `tests/` directory, so providers should add one as part of implementing their own workflows. + +## Packaging and Deployment + +This repository is designed to be packaged as a Docker image and promoted through the standard Simpl-Open delivery flow. + +High-level process: + +1. Implement and validate the workflow locally. +2. Commit the changes to a feature branch. +3. Open a pull request for review. +4. Build and publish the container image through CI/CD. +5. Update the deployment configuration to use the new image. +6. Verify the new code location in Dagster. + +Use immutable image tags such as semantic versions or commit-based identifiers rather than `latest`. + +## Documentation + +Additional guidance is available in the repository documents: + +- [documents/Development Guide.md](documents/Development%20Guide.md): workflow creation, edit, deletion, and deployment process +- [documents/Output Separation and Non-Overwrite Principles.md](documents/Output%20Separation%20and%20Non-Overwrite%20Principles.md): output handling expectations for workflow implementations + +## Contributing + +When extending this template: + +- keep the repository structure stable so onboarding stays predictable +- preserve the separation between ops, jobs, and repository definitions +- avoid hardcoded credentials and environment-specific values +- update tests and documentation together with workflow changes + +This repository should remain a reusable baseline for providers, not a dumping ground for unrelated or one-off implementations. From 5760f9424af425645427598f38a1258c380d999a Mon Sep 17 00:00:00 2001 From: ILay Date: Wed, 3 Jun 2026 15:05:43 +0200 Subject: [PATCH 6/8] [SIMPL-28408] Added readme. Changelog: added --- README.md | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b2faf7e..bbc476a 100644 --- a/README.md +++ b/README.md @@ -49,21 +49,17 @@ This template is not meant to contain business-specific logic by default. Provid ```text template-code-location/ -|- src/ -| \- template_code_location/ -| |- __init__.py -| |- repository.py -| |- jobs/ -| | |- __init__.py -| | \- jobs.py -| \- ops/ -| |- __init__.py -| \- ops.py -|- documents/ -|- Dockerfile -|- pyproject.toml -|- uv.lock -\- README.md +├── src/ +│ └── template_code_location/ +│ ├── repository.py # Unified entry point (all jobs/sensors/resources) +│ ├── jobs/ # Custom jobs specific to this code location +│ │ └── jobs.py +│ └── ops/ # Custom ops specific to this code location +│ └── ops.py +├── tests/ # Unit & integration tests +├── Dockerfile +├── pyproject.toml # Dependencies & external package sources +└── README.md ``` Key files: From 56c81705856c6d12bc43a0698450165f1cfb009d Mon Sep 17 00:00:00 2001 From: ILay Date: Wed, 3 Jun 2026 15:28:09 +0200 Subject: [PATCH 7/8] Remove CHANGELOG.md from MR develop to main --- CHANGELOG.md | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 1a4ddd3..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -## 0.1.0 (2026-05-29) - -### added (3 changes) - -- [feat(SIMPL-24642): consolidate all code locations into template-code-location](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/4e0b216410fc0d0879b583047fc7ba1f0390c612) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/4)) -- [init](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/0ed949a4e86b4e7676deba638b55f5d924b6d8b3) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/2)) -- [Feature/simpl 24306](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/commit/2125933d3dfb36f71ac6853d7aace92f8b2e8eeb) ([merge request](https://code.europa.eu/simpl/simpl-open/development/data-services/template-code-location/-/merge_requests/1)) - From 7c67371bb0fda929372056b81cd92fe258d214de Mon Sep 17 00:00:00 2001 From: ILay Date: Wed, 3 Jun 2026 16:26:01 +0200 Subject: [PATCH 8/8] add empty changelog --- CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29