ogdc-runner 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ogdc_runner-0.3.0/.dockerignore +10 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/workflows/publish.yml +6 -6
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/workflows/test.yml +11 -24
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.pre-commit-config.yaml +5 -4
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/CHANGELOG.md +12 -0
- ogdc_runner-0.3.0/Dockerfile +19 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/PKG-INFO +23 -7
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/README.md +10 -5
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/CONTRIBUTING.md +59 -65
- ogdc_runner-0.3.0/docs/architecture/argo.md +58 -0
- ogdc_runner-0.3.0/docs/architecture/index.md +110 -0
- ogdc_runner-0.3.0/docs/architecture/service.md +92 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/conf.py +2 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/index.md +1 -1
- ogdc_runner-0.3.0/docs/recipes.md +261 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/user_guide.md +29 -3
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/noxfile.py +44 -6
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/pyproject.toml +54 -13
- ogdc_runner-0.3.0/src/ogdc_runner/__init__.py +19 -0
- ogdc_runner-0.3.0/src/ogdc_runner/__main__.py +334 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/argo.py +151 -12
- ogdc_runner-0.3.0/src/ogdc_runner/constants.py +13 -0
- ogdc_runner-0.3.0/src/ogdc_runner/dataone/__init__.py +7 -0
- ogdc_runner-0.3.0/src/ogdc_runner/dataone/resolver.py +205 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/exceptions.py +30 -0
- ogdc_runner-0.3.0/src/ogdc_runner/inputs.py +118 -0
- ogdc_runner-0.3.0/src/ogdc_runner/models/__init__.py +25 -0
- ogdc_runner-0.3.0/src/ogdc_runner/models/base.py +13 -0
- ogdc_runner-0.3.0/src/ogdc_runner/models/parallel_config.py +82 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/models/recipe_config.py +169 -18
- ogdc_runner-0.3.0/src/ogdc_runner/parallel.py +135 -0
- ogdc_runner-0.3.0/src/ogdc_runner/partitioning.py +144 -0
- ogdc_runner-0.3.0/src/ogdc_runner/publish.py +373 -0
- ogdc_runner-0.3.0/src/ogdc_runner/recipe.py +150 -0
- ogdc_runner-0.3.0/src/ogdc_runner/scripts/README.md +12 -0
- ogdc_runner-0.3.0/src/ogdc_runner/scripts/__init__.py +1 -0
- ogdc_runner-0.3.0/src/ogdc_runner/scripts/partition_processing.sh +74 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/__init__.py +1 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/auth.py +147 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/auth_routes.py +156 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/db.py +52 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/main.py +62 -0
- ogdc_runner-0.3.0/src/ogdc_runner/service/user.py +104 -0
- ogdc_runner-0.3.0/src/ogdc_runner/workflow/shell.py +339 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/workflow/viz_workflow.py +18 -10
- ogdc_runner-0.3.0/tests/conftest.py +77 -0
- ogdc_runner-0.3.0/tests/integration/__init__.py +0 -0
- ogdc_runner-0.3.0/tests/integration/conftest.py +31 -0
- ogdc_runner-0.3.0/tests/integration/test_cli.py +38 -0
- ogdc_runner-0.3.0/tests/integration/test_dataone_input_recipe.py +31 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/integration/test_ogdc_recipe.py +1 -5
- ogdc_runner-0.3.0/tests/integration/test_ogdc_viz_recipe.py +29 -0
- ogdc_runner-0.3.0/tests/integration/test_temporary_output.py +45 -0
- ogdc_runner-0.3.0/tests/integration/test_validate_all_recipes.py +14 -0
- ogdc_runner-0.3.0/tests/test_parallel_shell_recipe_dir/meta.yml +19 -0
- ogdc_runner-0.3.0/tests/test_parallel_shell_recipe_dir/recipe.sh +8 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_shell_workflow_recipe_dir/meta.yml +2 -0
- ogdc_runner-0.3.0/tests/test_temp_output_recipe_dir/meta.yml +9 -0
- ogdc_runner-0.3.0/tests/test_temp_output_recipe_dir/recipe.sh +2 -0
- ogdc_runner-0.3.0/tests/test_viz_workflow_recipe_dir/meta.yml +10 -0
- ogdc_runner-0.3.0/tests/test_viz_workflow_recipe_dir/test_ice_basins.gpkg +0 -0
- ogdc_runner-0.3.0/tests/unit/__init__.py +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_argo.py +62 -4
- ogdc_runner-0.3.0/tests/unit/test_cli.py +14 -0
- ogdc_runner-0.3.0/tests/unit/test_dataone_input_model.py +294 -0
- ogdc_runner-0.3.0/tests/unit/test_dataone_inputs.py +173 -0
- ogdc_runner-0.3.0/tests/unit/test_dataone_resolver.py +186 -0
- ogdc_runner-0.3.0/tests/unit/test_models.py +160 -0
- ogdc_runner-0.3.0/tests/unit/test_parallel.py +60 -0
- ogdc_runner-0.3.0/tests/unit/test_parallel_recipe.py +28 -0
- ogdc_runner-0.3.0/tests/unit/test_partitioning.py +57 -0
- ogdc_runner-0.3.0/tests/unit/test_recipe.py +48 -0
- ogdc_runner-0.3.0/tests/unit/test_service.py +143 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_viz_workflow.py +3 -3
- ogdc_runner-0.2.0/Dockerfile +0 -7
- ogdc_runner-0.2.0/docs/architecture.md +0 -47
- ogdc_runner-0.2.0/docs/recipes.md +0 -121
- ogdc_runner-0.2.0/src/ogdc_runner/__init__.py +0 -11
- ogdc_runner-0.2.0/src/ogdc_runner/__main__.py +0 -80
- ogdc_runner-0.2.0/src/ogdc_runner/_version.py +0 -34
- ogdc_runner-0.2.0/src/ogdc_runner/constants.py +0 -5
- ogdc_runner-0.2.0/src/ogdc_runner/inputs.py +0 -61
- ogdc_runner-0.2.0/src/ogdc_runner/publish.py +0 -168
- ogdc_runner-0.2.0/src/ogdc_runner/recipe.py +0 -47
- ogdc_runner-0.2.0/src/ogdc_runner/workflow/shell.py +0 -98
- ogdc_runner-0.2.0/tests/conftest.py +0 -18
- ogdc_runner-0.2.0/tests/test_viz_workflow_recipe_dir/meta.yml +0 -16
- ogdc_runner-0.2.0/tests/unit/test_models.py +0 -60
- ogdc_runner-0.2.0/tests/unit/test_recipe.py +0 -11
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.git_archival.txt +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.gitattributes +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/dependabot.yml +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.gitignore +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.readthedocs.yaml +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/LICENSE +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/.gitignore +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/Makefile +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/make.bat +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/_version.pyi +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/api.py +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/py.typed +0 -0
- {ogdc_runner-0.2.0/src/ogdc_runner/models → ogdc_runner-0.3.0/src/ogdc_runner/workflow}/__init__.py +0 -0
- {ogdc_runner-0.2.0/src/ogdc_runner/workflow → ogdc_runner-0.3.0/tests}/__init__.py +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_shell_workflow_recipe_dir/recipe.sh +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_viz_workflow_recipe_dir/config.json +0 -0
- {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_package.py +0 -0
|
@@ -35,7 +35,7 @@ jobs:
|
|
|
35
35
|
runs-on: "ubuntu-latest"
|
|
36
36
|
|
|
37
37
|
steps:
|
|
38
|
-
- uses: "actions/checkout@
|
|
38
|
+
- uses: "actions/checkout@v6"
|
|
39
39
|
with:
|
|
40
40
|
fetch-depth: 0
|
|
41
41
|
|
|
@@ -49,14 +49,14 @@ jobs:
|
|
|
49
49
|
runs-on: "ubuntu-latest"
|
|
50
50
|
steps:
|
|
51
51
|
- name: "Check out repository"
|
|
52
|
-
uses: "actions/checkout@
|
|
52
|
+
uses: "actions/checkout@v6"
|
|
53
53
|
|
|
54
54
|
- name: "Build container image"
|
|
55
55
|
run: |
|
|
56
56
|
docker build --tag "ghcr.io/qgreenland-net/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" .
|
|
57
57
|
docker save "ghcr.io/qgreenland-net/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -o docker-image.tar
|
|
58
58
|
- name: "Upload image tarball as artifact"
|
|
59
|
-
uses: "actions/upload-artifact@
|
|
59
|
+
uses: "actions/upload-artifact@v7"
|
|
60
60
|
with:
|
|
61
61
|
name: "docker-image-tar"
|
|
62
62
|
path: "docker-image.tar"
|
|
@@ -75,7 +75,7 @@ jobs:
|
|
|
75
75
|
runs-on: "ubuntu-latest"
|
|
76
76
|
|
|
77
77
|
steps:
|
|
78
|
-
- uses: "actions/download-artifact@
|
|
78
|
+
- uses: "actions/download-artifact@v8"
|
|
79
79
|
with:
|
|
80
80
|
name: "Packages"
|
|
81
81
|
path: "dist"
|
|
@@ -94,7 +94,7 @@ jobs:
|
|
|
94
94
|
needs: ["build-python", "build-docker"]
|
|
95
95
|
steps:
|
|
96
96
|
- name: "Download image tarball"
|
|
97
|
-
uses: "actions/download-artifact@
|
|
97
|
+
uses: "actions/download-artifact@v8"
|
|
98
98
|
with:
|
|
99
99
|
name: "docker-image-tar"
|
|
100
100
|
|
|
@@ -125,7 +125,7 @@ jobs:
|
|
|
125
125
|
runs-on: "ubuntu-latest"
|
|
126
126
|
steps:
|
|
127
127
|
- name: "Checkout"
|
|
128
|
-
uses: "actions/checkout@
|
|
128
|
+
uses: "actions/checkout@v6"
|
|
129
129
|
- name: "Release"
|
|
130
130
|
uses: "softprops/action-gh-release@v2"
|
|
131
131
|
with:
|
|
@@ -17,23 +17,6 @@ env:
|
|
|
17
17
|
FORCE_COLOR: 3
|
|
18
18
|
|
|
19
19
|
jobs:
|
|
20
|
-
typecheck:
|
|
21
|
-
name: "Typecheck"
|
|
22
|
-
runs-on: "ubuntu-latest"
|
|
23
|
-
steps:
|
|
24
|
-
- uses: "actions/checkout@v5"
|
|
25
|
-
with:
|
|
26
|
-
fetch-depth: 0
|
|
27
|
-
- uses: "actions/setup-python@v6"
|
|
28
|
-
with:
|
|
29
|
-
python-version: "3.11"
|
|
30
|
-
|
|
31
|
-
- name: "Install package"
|
|
32
|
-
run: "python -m pip install .[test]"
|
|
33
|
-
|
|
34
|
-
- name: "Typecheck package"
|
|
35
|
-
run: "mypy"
|
|
36
|
-
|
|
37
20
|
test:
|
|
38
21
|
name: "Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }}"
|
|
39
22
|
runs-on: "${{ matrix.runs-on }}"
|
|
@@ -44,7 +27,7 @@ jobs:
|
|
|
44
27
|
runs-on: ["ubuntu-latest", "macos-latest", "windows-latest"]
|
|
45
28
|
|
|
46
29
|
steps:
|
|
47
|
-
- uses: "actions/checkout@
|
|
30
|
+
- uses: "actions/checkout@v6"
|
|
48
31
|
with:
|
|
49
32
|
fetch-depth: 0
|
|
50
33
|
|
|
@@ -53,14 +36,18 @@ jobs:
|
|
|
53
36
|
python-version: "${{ matrix.python-version }}"
|
|
54
37
|
allow-prereleases: true
|
|
55
38
|
|
|
39
|
+
# The MacOS runner does not have the required postgresql binaries that are
|
|
40
|
+
# required to `pip install` psycopg2, which is required for the API to
|
|
41
|
+
# connect to the OGDC postgresql DB.
|
|
42
|
+
- name: "Install Postgresql on macOS"
|
|
43
|
+
if: runner.os == 'macOS'
|
|
44
|
+
run: |
|
|
45
|
+
brew update
|
|
46
|
+
brew install postgresql
|
|
47
|
+
|
|
56
48
|
- name: "Install package"
|
|
57
49
|
run: "python -m pip install .[test]"
|
|
58
50
|
|
|
59
51
|
- name: "Test package"
|
|
60
52
|
# prettier-ignore
|
|
61
|
-
run: "
|
|
62
|
-
|
|
63
|
-
- name: "Upload coverage report"
|
|
64
|
-
uses: "codecov/codecov-action@v5.5.1"
|
|
65
|
-
with:
|
|
66
|
-
token: "${{ secrets.CODECOV_TOKEN }}"
|
|
53
|
+
run: "nox -s test_ci"
|
|
@@ -34,14 +34,15 @@ repos:
|
|
|
34
34
|
- id: rst-inline-touching-normal
|
|
35
35
|
|
|
36
36
|
- repo: https://github.com/rbubley/mirrors-prettier
|
|
37
|
-
rev: "v3.
|
|
37
|
+
rev: "v3.8.1"
|
|
38
38
|
hooks:
|
|
39
39
|
- id: prettier
|
|
40
40
|
types_or: [yaml, markdown, html, css, scss, javascript, json]
|
|
41
41
|
args: [--prose-wrap=always]
|
|
42
|
+
exclude: ^README\.md$
|
|
42
43
|
|
|
43
44
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
44
|
-
rev: "v0.
|
|
45
|
+
rev: "v0.15.4"
|
|
45
46
|
hooks:
|
|
46
47
|
- id: ruff
|
|
47
48
|
args: ["--fix", "--show-fixes"]
|
|
@@ -58,13 +59,13 @@ repos:
|
|
|
58
59
|
- id: shellcheck
|
|
59
60
|
|
|
60
61
|
- repo: https://github.com/abravalheri/validate-pyproject
|
|
61
|
-
rev: "v0.
|
|
62
|
+
rev: "v0.25"
|
|
62
63
|
hooks:
|
|
63
64
|
- id: validate-pyproject
|
|
64
65
|
additional_dependencies: ["validate-pyproject-schema-store[all]"]
|
|
65
66
|
|
|
66
67
|
- repo: https://github.com/python-jsonschema/check-jsonschema
|
|
67
|
-
rev: "0.
|
|
68
|
+
rev: "0.37.0"
|
|
68
69
|
hooks:
|
|
69
70
|
- id: check-dependabot
|
|
70
71
|
- id: check-github-workflows
|
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
# v0.3.0
|
|
2
|
+
|
|
3
|
+
- Create service interface for ogdc-runner.
|
|
4
|
+
- Refactor URL validation to use Pydantic model validators.
|
|
5
|
+
- Add recipe ID truncation to avoid Kubernetes name length limits.
|
|
6
|
+
- Add `temporary` output type and support retrieval of those outputs from s3
|
|
7
|
+
storage via the `ogdc-runner get-output` CLI.
|
|
8
|
+
- Add DataONE input type for fetching datasets directly from DataONE
|
|
9
|
+
repositories
|
|
10
|
+
- Add `parallel-shell` recipe type and related support for executing shell
|
|
11
|
+
recipes in parallel.
|
|
12
|
+
|
|
1
13
|
# v0.2.0
|
|
2
14
|
|
|
3
15
|
- Add support for both `local` (via rancher-desktop) and `dev` environments
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Docker image for:
|
|
2
|
+
# * Running gdal/ogr and fetch (wget) commands for ogdc recipes.
|
|
3
|
+
# * Provides the OGDC service interface (default CMD).
|
|
4
|
+
# WARNING: this image *MUST NOT* be based on busybox/alpine linux due to a known
|
|
5
|
+
# networking issue in non-local environments. For context, see:
|
|
6
|
+
# https://github.com/QGreenland-Net/ogdc-helm/issues/31
|
|
7
|
+
FROM ghcr.io/osgeo/gdal:ubuntu-full-3.12.0
|
|
8
|
+
# We use and wget to fetch data from remote sources.
|
|
9
|
+
# libpq-dev is required for connecting to postgresql db
|
|
10
|
+
RUN apt update && apt install -y wget rsync pip python3-venv libpq-dev zip netcdf-bin libnetcdf-dev
|
|
11
|
+
|
|
12
|
+
WORKDIR /ogdc_runner/
|
|
13
|
+
COPY . /ogdc_runner
|
|
14
|
+
RUN python -m venv .venv && . ./.venv/bin/activate && pip install --editable .
|
|
15
|
+
|
|
16
|
+
EXPOSE 8000
|
|
17
|
+
|
|
18
|
+
# Default to running the production fastapi server
|
|
19
|
+
CMD . ./.venv/bin/activate && fastapi run --port 8000 --host 0.0.0.0 src/ogdc_runner/service/main.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ogdc-runner
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Defines OGDC recipe API(s) and submits recipes to OGDC for execution
|
|
5
5
|
Project-URL: Homepage, https://github.com/qgreenland-net/ogdc-runner
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/qgreenland-net/ogdc-runner/issues
|
|
@@ -40,18 +40,27 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
40
40
|
Classifier: Topic :: Scientific/Engineering
|
|
41
41
|
Classifier: Typing :: Typed
|
|
42
42
|
Requires-Python: >=3.11
|
|
43
|
+
Requires-Dist: boto3>=1.42
|
|
43
44
|
Requires-Dist: click>=8
|
|
45
|
+
Requires-Dist: dataone-libclient>=3.5.0
|
|
46
|
+
Requires-Dist: fastapi[standard]
|
|
44
47
|
Requires-Dist: fsspec
|
|
45
48
|
Requires-Dist: hera>=5.18.0
|
|
46
49
|
Requires-Dist: loguru
|
|
50
|
+
Requires-Dist: psycopg2-binary
|
|
51
|
+
Requires-Dist: pwdlib[argon2]
|
|
47
52
|
Requires-Dist: pydantic>=2.7.1
|
|
48
|
-
Requires-Dist:
|
|
53
|
+
Requires-Dist: pyjwt
|
|
54
|
+
Requires-Dist: pyyaml>=6.0
|
|
55
|
+
Requires-Dist: requests
|
|
56
|
+
Requires-Dist: sqlmodel
|
|
49
57
|
Provides-Extra: dev
|
|
50
58
|
Requires-Dist: mypy~=1.10.0; extra == 'dev'
|
|
51
59
|
Requires-Dist: nox; extra == 'dev'
|
|
52
60
|
Requires-Dist: pytest-cov>=3; extra == 'dev'
|
|
53
61
|
Requires-Dist: pytest>=6; extra == 'dev'
|
|
54
62
|
Requires-Dist: types-pyyaml; extra == 'dev'
|
|
63
|
+
Requires-Dist: types-requests; extra == 'dev'
|
|
55
64
|
Provides-Extra: docs
|
|
56
65
|
Requires-Dist: autodoc-pydantic; extra == 'docs'
|
|
57
66
|
Requires-Dist: furo>=2023.08.17; extra == 'docs'
|
|
@@ -62,21 +71,21 @@ Requires-Dist: sphinx>=7.0; extra == 'docs'
|
|
|
62
71
|
Requires-Dist: sphinxcontrib-mermaid; extra == 'docs'
|
|
63
72
|
Provides-Extra: test
|
|
64
73
|
Requires-Dist: mypy~=1.10.0; extra == 'test'
|
|
74
|
+
Requires-Dist: nox; extra == 'test'
|
|
65
75
|
Requires-Dist: pytest-cov>=3; extra == 'test'
|
|
66
76
|
Requires-Dist: pytest-order; extra == 'test'
|
|
67
77
|
Requires-Dist: pytest>=6; extra == 'test'
|
|
68
78
|
Requires-Dist: types-pyyaml; extra == 'test'
|
|
79
|
+
Requires-Dist: types-requests; extra == 'test'
|
|
69
80
|
Description-Content-Type: text/markdown
|
|
70
81
|
|
|
71
82
|
# ogdc-runner
|
|
72
83
|
|
|
73
84
|
<!-- SPHINX-START -->
|
|
74
85
|
|
|
75
|
-
[![Actions Status][actions-badge]][actions-link]
|
|
76
86
|
[![Documentation Status][rtd-badge]][rtd-link]
|
|
77
87
|
|
|
78
88
|
[![PyPI version][pypi-version]][pypi-link]
|
|
79
|
-
[![Conda-Forge][conda-badge]][conda-link]
|
|
80
89
|
[![PyPI platforms][pypi-platforms]][pypi-link]
|
|
81
90
|
|
|
82
91
|
[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
|
|
@@ -87,11 +96,18 @@ OGDC for execution.
|
|
|
87
96
|
See the [ogdc-runner ReadTheDocs](https://ogdc-runner.readthedocs.io/) for
|
|
88
97
|
detailed documentation on how to use and contribute to the `ogdc-runner`.
|
|
89
98
|
|
|
99
|
+
## Citation
|
|
100
|
+
|
|
101
|
+
Cite this software as:
|
|
102
|
+
|
|
103
|
+
> Trey Stafford, Robyn Marowitz, Rushiraj Nenuji, Matthew B. Jones, Twila
|
|
104
|
+
> Moon. 2025. Open Geospatial Data Cloud (OGDC) Runner: Client tool and API for
|
|
105
|
+
> scalable and repeatable geospatial data transformation recipes. (version
|
|
106
|
+
> 0.2.0). Arctic Data Center.
|
|
107
|
+
> [doi:10.18739/A2MS3K382](https://doi.org/doi:10.18739/A2MS3K382)
|
|
108
|
+
|
|
90
109
|
<!-- prettier-ignore-start -->
|
|
91
|
-
[actions-badge]: https://github.com/qgreenland-net/ogdc-runner/workflows/CI/badge.svg
|
|
92
110
|
[actions-link]: https://github.com/qgreenland-net/ogdc-runner/actions
|
|
93
|
-
[conda-badge]: https://img.shields.io/conda/vn/conda-forge/ogdc-runner
|
|
94
|
-
[conda-link]: https://github.com/conda-forge/ogdc-runner-feedstock
|
|
95
111
|
[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
|
|
96
112
|
[github-discussions-link]: https://github.com/qgreenland-net/ogdc-runner/discussions
|
|
97
113
|
[pypi-link]: https://pypi.org/project/ogdc-runner/
|
|
@@ -2,11 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
<!-- SPHINX-START -->
|
|
4
4
|
|
|
5
|
-
[![Actions Status][actions-badge]][actions-link]
|
|
6
5
|
[![Documentation Status][rtd-badge]][rtd-link]
|
|
7
6
|
|
|
8
7
|
[![PyPI version][pypi-version]][pypi-link]
|
|
9
|
-
[![Conda-Forge][conda-badge]][conda-link]
|
|
10
8
|
[![PyPI platforms][pypi-platforms]][pypi-link]
|
|
11
9
|
|
|
12
10
|
[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
|
|
@@ -17,11 +15,18 @@ OGDC for execution.
|
|
|
17
15
|
See the [ogdc-runner ReadTheDocs](https://ogdc-runner.readthedocs.io/) for
|
|
18
16
|
detailed documentation on how to use and contribute to the `ogdc-runner`.
|
|
19
17
|
|
|
18
|
+
## Citation
|
|
19
|
+
|
|
20
|
+
Cite this software as:
|
|
21
|
+
|
|
22
|
+
> Trey Stafford, Robyn Marowitz, Rushiraj Nenuji, Matthew B. Jones, Twila
|
|
23
|
+
> Moon. 2025. Open Geospatial Data Cloud (OGDC) Runner: Client tool and API for
|
|
24
|
+
> scalable and repeatable geospatial data transformation recipes. (version
|
|
25
|
+
> 0.2.0). Arctic Data Center.
|
|
26
|
+
> [doi:10.18739/A2MS3K382](https://doi.org/doi:10.18739/A2MS3K382)
|
|
27
|
+
|
|
20
28
|
<!-- prettier-ignore-start -->
|
|
21
|
-
[actions-badge]: https://github.com/qgreenland-net/ogdc-runner/workflows/CI/badge.svg
|
|
22
29
|
[actions-link]: https://github.com/qgreenland-net/ogdc-runner/actions
|
|
23
|
-
[conda-badge]: https://img.shields.io/conda/vn/conda-forge/ogdc-runner
|
|
24
|
-
[conda-link]: https://github.com/conda-forge/ogdc-runner-feedstock
|
|
25
30
|
[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
|
|
26
31
|
[github-discussions-link]: https://github.com/qgreenland-net/ogdc-runner/discussions
|
|
27
32
|
[pypi-link]: https://pypi.org/project/ogdc-runner/
|
|
@@ -9,11 +9,21 @@ description of best practices for developing scientific packages.
|
|
|
9
9
|
Add yourself as an author in `pyproject.toml`
|
|
10
10
|
```
|
|
11
11
|
|
|
12
|
+
## Familiarize yourself with the project
|
|
13
|
+
|
|
14
|
+
- Be sure to look over the [Architecture](./architecture/index.md) docs to get
|
|
15
|
+
an understanding of the `ogdc-runner` before contributing any code.
|
|
16
|
+
|
|
17
|
+
- The `ogdc-runner` is one component of the QGreenland-Net Open Geospatial Data
|
|
18
|
+
Cloud (OGDC). See the [QGreenland-Net](https://qgreenland-net.github.io/)
|
|
19
|
+
webpage for more information about how this project fits into that larger
|
|
20
|
+
effort. In particular, check out the
|
|
21
|
+
[QGreenland-Net Contributing docs](https://qgreenland-net.github.io/contributing/)!
|
|
22
|
+
|
|
12
23
|
## Setting up a local development environment
|
|
13
24
|
|
|
14
25
|
First, ensure you have [ogdc-helm](https://github.com/QGreenland-Net/ogdc-helm)
|
|
15
|
-
setup for local development with `rancher-desktop
|
|
16
|
-
expected to be forwarded for access via localhost.
|
|
26
|
+
setup for local development with `rancher-desktop` using `skaffold`.
|
|
17
27
|
|
|
18
28
|
Now you can set up a python development environment for `ogdc-runner` by
|
|
19
29
|
running:
|
|
@@ -24,33 +34,10 @@ source ./.venv/bin/activate
|
|
|
24
34
|
pip install -v --editable ".[dev]"
|
|
25
35
|
```
|
|
26
36
|
|
|
27
|
-
###
|
|
28
|
-
|
|
29
|
-
The `ogdc-runner` supports using a local `ogdc-runner` image for development
|
|
30
|
-
purposes (e.g., you want to change and test something about the image without
|
|
31
|
-
needing to release it to the GHCR).
|
|
32
|
-
|
|
33
|
-
First, build a local image:
|
|
34
|
-
|
|
35
|
-
```{note}
|
|
36
|
-
The docker image must be built in the `rancher-desktop` context so
|
|
37
|
-
that it is available to the Argo deployment on the developer's local machine.
|
|
38
|
-
Check that you have the correct context selected with `docker context ls`.
|
|
39
|
-
```
|
|
40
|
-
|
|
41
|
-
```{warning}
|
|
42
|
-
The `ogdc-runner` docker image (and any others intended to be run on the k8s
|
|
43
|
-
cluster as part of the OGDC) **MUST NOT** be based on busybox/alpine Linux due
|
|
44
|
-
to a known networking issue in non-local environments. For context, see:
|
|
45
|
-
<https://github.com/QGreenland-Net/ogdc-helm/issues/31>
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
```
|
|
49
|
-
docker build . -t ogdc-runner
|
|
50
|
-
```
|
|
37
|
+
### Required environment variables
|
|
51
38
|
|
|
52
|
-
|
|
53
|
-
|
|
39
|
+
Set the `ENVIRONMENT` envvar to `local`. This will tell `ogdc-runner` to operate
|
|
40
|
+
in "local" development mode:
|
|
54
41
|
|
|
55
42
|
```
|
|
56
43
|
export ENVIRONMENT=local
|
|
@@ -80,17 +67,14 @@ be re-pulled, ensuring that the latest `main` branch image is being used.
|
|
|
80
67
|
|
|
81
68
|
## Testing, linting, rendering docs with Nox
|
|
82
69
|
|
|
83
|
-
|
|
84
|
-
`pipx run nox` to run it without installing, or `pipx install nox`. If you don't
|
|
85
|
-
have pipx, then you can install with `pip install pipx`. If you use macOS, use
|
|
86
|
-
`brew install pipx nox`. To use:
|
|
70
|
+
To run all tests, simply run `nox`:
|
|
87
71
|
|
|
88
72
|
```console
|
|
89
73
|
nox
|
|
90
74
|
```
|
|
91
75
|
|
|
92
|
-
This will
|
|
93
|
-
|
|
76
|
+
This will run the typechecker, unit, and integration tests (requiring
|
|
77
|
+
`ogdc-helm` to be deployed locally).
|
|
94
78
|
|
|
95
79
|
### Running specific tasks with Nox
|
|
96
80
|
|
|
@@ -98,6 +82,13 @@ ones that are not installed.
|
|
|
98
82
|
nox -s {job-name}
|
|
99
83
|
```
|
|
100
84
|
|
|
85
|
+
For example, to run only the tests run in CI, which are fast and do not require
|
|
86
|
+
a locally deployed OGDC stack:
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
nox -s test_ci
|
|
90
|
+
```
|
|
91
|
+
|
|
101
92
|
To view available jobs:
|
|
102
93
|
|
|
103
94
|
```console
|
|
@@ -107,6 +98,28 @@ nox -l
|
|
|
107
98
|
Nox handles everything for you, including setting up an temporary virtual
|
|
108
99
|
environment for each run.
|
|
109
100
|
|
|
101
|
+
### Testing
|
|
102
|
+
|
|
103
|
+
Use `nox` to run all tests (with pytest):
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
nox
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Building docs
|
|
110
|
+
|
|
111
|
+
You can build the docs using:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
nox -s docs
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
You can see a preview with:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
nox -s docs -- --serve
|
|
121
|
+
```
|
|
122
|
+
|
|
110
123
|
### Reusing Nox virtual environments
|
|
111
124
|
|
|
112
125
|
**By default, Nox deletes and recreates virtual environments for every run.**
|
|
@@ -131,36 +144,6 @@ pre-commit install # install Git pre-commit hook from .pre-commit-config.yml
|
|
|
131
144
|
You can also/alternatively run `pre-commit run` (will run for changed files
|
|
132
145
|
only) or `pre-commit run --all-files` to check even without installing the hook.
|
|
133
146
|
|
|
134
|
-
## Testing
|
|
135
|
-
|
|
136
|
-
Use pytest to run the unit checks:
|
|
137
|
-
|
|
138
|
-
```bash
|
|
139
|
-
pytest
|
|
140
|
-
```
|
|
141
|
-
|
|
142
|
-
### Coverage
|
|
143
|
-
|
|
144
|
-
Use pytest-cov to generate coverage reports:
|
|
145
|
-
|
|
146
|
-
```bash
|
|
147
|
-
pytest --cov=ogdc-runner
|
|
148
|
-
```
|
|
149
|
-
|
|
150
|
-
## Building docs
|
|
151
|
-
|
|
152
|
-
You can build the docs using:
|
|
153
|
-
|
|
154
|
-
```bash
|
|
155
|
-
nox -s docs
|
|
156
|
-
```
|
|
157
|
-
|
|
158
|
-
You can see a preview with:
|
|
159
|
-
|
|
160
|
-
```bash
|
|
161
|
-
nox -s docs -- --serve
|
|
162
|
-
```
|
|
163
|
-
|
|
164
147
|
## Continuous Integration
|
|
165
148
|
|
|
166
149
|
This project uses [GitHub Actions](https://docs.github.com/en/actions) to
|
|
@@ -189,7 +172,18 @@ This project uses [semantic versioning](https://semver.org/).
|
|
|
189
172
|
> 3. PATCH version when you make backward compatible bug fixes
|
|
190
173
|
|
|
191
174
|
Decide what the version will be for your release, and ensure that the CHANGELOG
|
|
192
|
-
contains an entry for the
|
|
175
|
+
contains an entry for the `## NEXT_VERSION`.
|
|
176
|
+
|
|
177
|
+
**Bump the Version**
|
|
178
|
+
|
|
179
|
+
Use bump-my-version to automatically update the version number in all configured
|
|
180
|
+
files (e.g., pyproject.toml, CHANGELOG.md).
|
|
181
|
+
|
|
182
|
+
Choose the appropriate part to bump:
|
|
183
|
+
|
|
184
|
+
- PATCH release: `bump-my-version bump patch`
|
|
185
|
+
- MINOR release: `bump-my-version bump minor`
|
|
186
|
+
- MAJOR release: `bump-my-version bump major`
|
|
193
187
|
|
|
194
188
|
Once `main` is ready for a release (feature branches are merged and the
|
|
195
189
|
CHANGELOG is up-to-date), tag the latest commit with the version to be released
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Argo Workflows
|
|
2
|
+
|
|
3
|
+
[OGDC recipes](../recipes.md) are executed via one or more
|
|
4
|
+
[argo-workflows](https://argo-workflows.readthedocs.io/). The
|
|
5
|
+
[OGDC Service API](./service.md) is responsible for translating user recipes
|
|
6
|
+
into argo workflows that are submitted and executed by Argo.
|
|
7
|
+
|
|
8
|
+
This document includes details of how Argo is setup and used by the OGDC.
|
|
9
|
+
|
|
10
|
+
## argo-workflows server configuration
|
|
11
|
+
|
|
12
|
+
The configuration for argo-workflows is defined in the
|
|
13
|
+
[ogdc-helm](https://github.com/qgreenland-net/ogdc-helm) repository. Refer to
|
|
14
|
+
the configurations there to see the specifics of how argo is setup for
|
|
15
|
+
deployment.
|
|
16
|
+
|
|
17
|
+
## Argo Python API (Hera)
|
|
18
|
+
|
|
19
|
+
The OGDC uses the [hera](https://hera.readthedocs.io/en/stable/) python package
|
|
20
|
+
to interact with the argo-workflows API. Boilerplate setup and configuration of
|
|
21
|
+
argo via hera can be found in {mod}`ogdc_runner.argo`.
|
|
22
|
+
|
|
23
|
+
## Workflow artifacts
|
|
24
|
+
|
|
25
|
+
Argo
|
|
26
|
+
[artifacts](https://argo-workflows.readthedocs.io/en/latest/walk-through/artifacts/)
|
|
27
|
+
are configured to use an S3-compatible
|
|
28
|
+
[artifact repository](https://argo-workflows.readthedocs.io/en/latest/configure-artifact-repository/).
|
|
29
|
+
|
|
30
|
+
Artifacts are used to store intermediate workflow outputs and the final output
|
|
31
|
+
of recipes with a `temporary` output type.
|
|
32
|
+
|
|
33
|
+
Artifacts will be automatically garbage collected on workflow deletion. See the
|
|
34
|
+
{ref}`workflow-persistence` section below for details on automatic workflow
|
|
35
|
+
deletion.
|
|
36
|
+
|
|
37
|
+
<!-- prettier-ignore-start -->
|
|
38
|
+
(workflow-persistence)=
|
|
39
|
+
## Workflow persistence
|
|
40
|
+
<!-- prettier-ignore-end -->
|
|
41
|
+
|
|
42
|
+
Successful Argo workflows are retained for 1 day. Workflows associated with
|
|
43
|
+
recipes with the `temporary` output type are retained for 7 days to allow for
|
|
44
|
+
sufficient time to retrieve final outputs.
|
|
45
|
+
|
|
46
|
+
Successful workflows with the `ogdc/persist-workflow-in-archive: true` label
|
|
47
|
+
will be archived in the OGDC's postgresql database for long-term storage. These
|
|
48
|
+
archived workflows can be used for metrics and data provenance purposes.
|
|
49
|
+
|
|
50
|
+
Non-successful workflows are not automatically cleaned up or archived. They are
|
|
51
|
+
retained for inspection/debugging and should be cleaned up manually once the
|
|
52
|
+
issue leading to failure is resolved.
|
|
53
|
+
|
|
54
|
+
To ensure consistent behavior of OGDC-submitted argo workflows (e.g., setting
|
|
55
|
+
the archival label and TTL for successful workflows), the
|
|
56
|
+
{class}`ogdc_runner.argo.OgdcWorkflow` context manager has been defined to wrap
|
|
57
|
+
the behavior of [hera's](https://hera.readthedocs.io/en/stable/)
|
|
58
|
+
[Workflow](https://hera.readthedocs.io/en/stable/api/workflows/workflow_classes/workflow/#hera.workflows.workflow.Workflow).
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
```{toctree}
|
|
4
|
+
:maxdepth: 1
|
|
5
|
+
:hidden:
|
|
6
|
+
|
|
7
|
+
service.md
|
|
8
|
+
argo.md
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
```{mermaid}
|
|
12
|
+
graph LR
|
|
13
|
+
|
|
14
|
+
%% Definitions
|
|
15
|
+
subgraph ADC_K8S[ADC k8s]
|
|
16
|
+
subgraph OGDC[OGDC]
|
|
17
|
+
OGDC_RUNNER_SERVICE[ogdc-runner service]
|
|
18
|
+
ARGO[Argo Workflows]
|
|
19
|
+
DB[(PostgreSQL Database)]
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
subgraph OGDC_RUNNER_CLI[ogdc-runner CLI]
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
%% Relationships
|
|
27
|
+
OGDC_RUNNER_CLI -->|"submit|status"| OGDC_RUNNER_SERVICE
|
|
28
|
+
|
|
29
|
+
OGDC_RUNNER_SERVICE -->|"create/query Argo workflow(s) required to execute recipe instructions"|ARGO
|
|
30
|
+
OGDC_RUNNER_SERVICE -->|"create/query Users"|DB
|
|
31
|
+
|
|
32
|
+
%% Style
|
|
33
|
+
style OGDC_RUNNER_SERVICE stroke:#ff6666,stroke-width:2px;
|
|
34
|
+
style OGDC_RUNNER_CLI stroke:#ff6666,stroke-width:2px;
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
[Please view our main documentation site for context](https://qgreenland-net.github.io).
|
|
38
|
+
|
|
39
|
+
This component:
|
|
40
|
+
|
|
41
|
+
- Defines and documents the recipe API(s). See [OGDC Recipes](../recipes.md).
|
|
42
|
+
- Defines a CLI published as the `ogdc-runner` pypi Python package that accepts
|
|
43
|
+
a recipe as input and submits it to the service layer.
|
|
44
|
+
- Defines a service layer published as the `ghcr.io/qgreenland-net/ogdc-runner`
|
|
45
|
+
docker image that translates submitted OGDC recipes into Argo Workflows for
|
|
46
|
+
execution. See [OGDC Service API](./service.md) for more details.
|
|
47
|
+
|
|
48
|
+
## Parallel Execution
|
|
49
|
+
|
|
50
|
+
The ogdc-runner supports parallel execution of workflow tasks via Argo's DAG
|
|
51
|
+
(Directed Acyclic Graph) model. When a recipe enables parallel execution, the
|
|
52
|
+
runner:
|
|
53
|
+
|
|
54
|
+
1. **Partitions** input data based on the configured strategy (e.g., grouping
|
|
55
|
+
files)
|
|
56
|
+
2. **Creates** independent Argo tasks for each partition
|
|
57
|
+
3. **Orchestrates** parallel execution with configurable maximum parallelism
|
|
58
|
+
|
|
59
|
+
The {class}`ogdc_runner.parallel.ParallelExecutionOrchestrator` class manages
|
|
60
|
+
this process, creating Argo Container templates and DAG tasks with proper
|
|
61
|
+
dependencies and parameters. Argo handles task scheduling and resource
|
|
62
|
+
allocation, distributing work across available cluster resources.
|
|
63
|
+
|
|
64
|
+
Key modules:
|
|
65
|
+
|
|
66
|
+
- {mod}`ogdc_runner.parallel`: Orchestration logic for parallel task creation
|
|
67
|
+
- {mod}`ogdc_runner.partitioning`: Partitioning strategies for dividing work
|
|
68
|
+
- {mod}`ogdc_runner.models.parallel_config`: Configuration models for parallel
|
|
69
|
+
execution
|
|
70
|
+
|
|
71
|
+
## Aspirational
|
|
72
|
+
|
|
73
|
+
Original aspirational architecture diagram of the `ogdc-runner` as it relates to
|
|
74
|
+
the rest of the OGDC.
|
|
75
|
+
|
|
76
|
+
```{warning}
|
|
77
|
+
This diagram does not reflect the actual, current implementation. This remains for reference only.
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```{mermaid}
|
|
81
|
+
graph LR
|
|
82
|
+
|
|
83
|
+
%% Definitions
|
|
84
|
+
subgraph ADC_K8S[ADC k8s]
|
|
85
|
+
subgraph GHA_SELFHOSTED[GHA self-hosted runner]
|
|
86
|
+
OGDC_RUNNER[ogdc-runner]
|
|
87
|
+
end
|
|
88
|
+
OGDC[OGDC]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
subgraph RECIPE_REPO[Recipe repo]
|
|
92
|
+
GHA[GitHub Actions]
|
|
93
|
+
RECIPE[Recipe]
|
|
94
|
+
SECRET[Secret token]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
%% Relationships
|
|
100
|
+
OGDC_RUNNER -->|submit| OGDC
|
|
101
|
+
GHA_SELFHOSTED -->|status| GHA
|
|
102
|
+
GHA -->|trigger| GHA_SELFHOSTED
|
|
103
|
+
|
|
104
|
+
SECRET -->|read| GHA
|
|
105
|
+
RECIPE -->|on change| GHA
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
%% Style
|
|
109
|
+
style OGDC_RUNNER stroke:#ff6666,stroke-width:2px;
|
|
110
|
+
```
|