ogdc-runner 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. ogdc_runner-0.3.0/.dockerignore +10 -0
  2. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/workflows/publish.yml +6 -6
  3. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/workflows/test.yml +11 -24
  4. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.pre-commit-config.yaml +5 -4
  5. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/CHANGELOG.md +12 -0
  6. ogdc_runner-0.3.0/Dockerfile +19 -0
  7. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/PKG-INFO +23 -7
  8. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/README.md +10 -5
  9. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/CONTRIBUTING.md +59 -65
  10. ogdc_runner-0.3.0/docs/architecture/argo.md +58 -0
  11. ogdc_runner-0.3.0/docs/architecture/index.md +110 -0
  12. ogdc_runner-0.3.0/docs/architecture/service.md +92 -0
  13. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/conf.py +2 -0
  14. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/index.md +1 -1
  15. ogdc_runner-0.3.0/docs/recipes.md +261 -0
  16. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/user_guide.md +29 -3
  17. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/noxfile.py +44 -6
  18. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/pyproject.toml +54 -13
  19. ogdc_runner-0.3.0/src/ogdc_runner/__init__.py +19 -0
  20. ogdc_runner-0.3.0/src/ogdc_runner/__main__.py +334 -0
  21. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/argo.py +151 -12
  22. ogdc_runner-0.3.0/src/ogdc_runner/constants.py +13 -0
  23. ogdc_runner-0.3.0/src/ogdc_runner/dataone/__init__.py +7 -0
  24. ogdc_runner-0.3.0/src/ogdc_runner/dataone/resolver.py +205 -0
  25. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/exceptions.py +30 -0
  26. ogdc_runner-0.3.0/src/ogdc_runner/inputs.py +118 -0
  27. ogdc_runner-0.3.0/src/ogdc_runner/models/__init__.py +25 -0
  28. ogdc_runner-0.3.0/src/ogdc_runner/models/base.py +13 -0
  29. ogdc_runner-0.3.0/src/ogdc_runner/models/parallel_config.py +82 -0
  30. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/models/recipe_config.py +169 -18
  31. ogdc_runner-0.3.0/src/ogdc_runner/parallel.py +135 -0
  32. ogdc_runner-0.3.0/src/ogdc_runner/partitioning.py +144 -0
  33. ogdc_runner-0.3.0/src/ogdc_runner/publish.py +373 -0
  34. ogdc_runner-0.3.0/src/ogdc_runner/recipe.py +150 -0
  35. ogdc_runner-0.3.0/src/ogdc_runner/scripts/README.md +12 -0
  36. ogdc_runner-0.3.0/src/ogdc_runner/scripts/__init__.py +1 -0
  37. ogdc_runner-0.3.0/src/ogdc_runner/scripts/partition_processing.sh +74 -0
  38. ogdc_runner-0.3.0/src/ogdc_runner/service/__init__.py +1 -0
  39. ogdc_runner-0.3.0/src/ogdc_runner/service/auth.py +147 -0
  40. ogdc_runner-0.3.0/src/ogdc_runner/service/auth_routes.py +156 -0
  41. ogdc_runner-0.3.0/src/ogdc_runner/service/db.py +52 -0
  42. ogdc_runner-0.3.0/src/ogdc_runner/service/main.py +62 -0
  43. ogdc_runner-0.3.0/src/ogdc_runner/service/user.py +104 -0
  44. ogdc_runner-0.3.0/src/ogdc_runner/workflow/shell.py +339 -0
  45. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/workflow/viz_workflow.py +18 -10
  46. ogdc_runner-0.3.0/tests/conftest.py +77 -0
  47. ogdc_runner-0.3.0/tests/integration/__init__.py +0 -0
  48. ogdc_runner-0.3.0/tests/integration/conftest.py +31 -0
  49. ogdc_runner-0.3.0/tests/integration/test_cli.py +38 -0
  50. ogdc_runner-0.3.0/tests/integration/test_dataone_input_recipe.py +31 -0
  51. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/integration/test_ogdc_recipe.py +1 -5
  52. ogdc_runner-0.3.0/tests/integration/test_ogdc_viz_recipe.py +29 -0
  53. ogdc_runner-0.3.0/tests/integration/test_temporary_output.py +45 -0
  54. ogdc_runner-0.3.0/tests/integration/test_validate_all_recipes.py +14 -0
  55. ogdc_runner-0.3.0/tests/test_parallel_shell_recipe_dir/meta.yml +19 -0
  56. ogdc_runner-0.3.0/tests/test_parallel_shell_recipe_dir/recipe.sh +8 -0
  57. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_shell_workflow_recipe_dir/meta.yml +2 -0
  58. ogdc_runner-0.3.0/tests/test_temp_output_recipe_dir/meta.yml +9 -0
  59. ogdc_runner-0.3.0/tests/test_temp_output_recipe_dir/recipe.sh +2 -0
  60. ogdc_runner-0.3.0/tests/test_viz_workflow_recipe_dir/meta.yml +10 -0
  61. ogdc_runner-0.3.0/tests/test_viz_workflow_recipe_dir/test_ice_basins.gpkg +0 -0
  62. ogdc_runner-0.3.0/tests/unit/__init__.py +0 -0
  63. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_argo.py +62 -4
  64. ogdc_runner-0.3.0/tests/unit/test_cli.py +14 -0
  65. ogdc_runner-0.3.0/tests/unit/test_dataone_input_model.py +294 -0
  66. ogdc_runner-0.3.0/tests/unit/test_dataone_inputs.py +173 -0
  67. ogdc_runner-0.3.0/tests/unit/test_dataone_resolver.py +186 -0
  68. ogdc_runner-0.3.0/tests/unit/test_models.py +160 -0
  69. ogdc_runner-0.3.0/tests/unit/test_parallel.py +60 -0
  70. ogdc_runner-0.3.0/tests/unit/test_parallel_recipe.py +28 -0
  71. ogdc_runner-0.3.0/tests/unit/test_partitioning.py +57 -0
  72. ogdc_runner-0.3.0/tests/unit/test_recipe.py +48 -0
  73. ogdc_runner-0.3.0/tests/unit/test_service.py +143 -0
  74. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_viz_workflow.py +3 -3
  75. ogdc_runner-0.2.0/Dockerfile +0 -7
  76. ogdc_runner-0.2.0/docs/architecture.md +0 -47
  77. ogdc_runner-0.2.0/docs/recipes.md +0 -121
  78. ogdc_runner-0.2.0/src/ogdc_runner/__init__.py +0 -11
  79. ogdc_runner-0.2.0/src/ogdc_runner/__main__.py +0 -80
  80. ogdc_runner-0.2.0/src/ogdc_runner/_version.py +0 -34
  81. ogdc_runner-0.2.0/src/ogdc_runner/constants.py +0 -5
  82. ogdc_runner-0.2.0/src/ogdc_runner/inputs.py +0 -61
  83. ogdc_runner-0.2.0/src/ogdc_runner/publish.py +0 -168
  84. ogdc_runner-0.2.0/src/ogdc_runner/recipe.py +0 -47
  85. ogdc_runner-0.2.0/src/ogdc_runner/workflow/shell.py +0 -98
  86. ogdc_runner-0.2.0/tests/conftest.py +0 -18
  87. ogdc_runner-0.2.0/tests/test_viz_workflow_recipe_dir/meta.yml +0 -16
  88. ogdc_runner-0.2.0/tests/unit/test_models.py +0 -60
  89. ogdc_runner-0.2.0/tests/unit/test_recipe.py +0 -11
  90. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.git_archival.txt +0 -0
  91. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.gitattributes +0 -0
  92. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.github/dependabot.yml +0 -0
  93. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.gitignore +0 -0
  94. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/.readthedocs.yaml +0 -0
  95. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/LICENSE +0 -0
  96. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/.gitignore +0 -0
  97. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/Makefile +0 -0
  98. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/docs/make.bat +0 -0
  99. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/_version.pyi +0 -0
  100. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/api.py +0 -0
  101. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/src/ogdc_runner/py.typed +0 -0
  102. {ogdc_runner-0.2.0/src/ogdc_runner/models → ogdc_runner-0.3.0/src/ogdc_runner/workflow}/__init__.py +0 -0
  103. {ogdc_runner-0.2.0/src/ogdc_runner/workflow → ogdc_runner-0.3.0/tests}/__init__.py +0 -0
  104. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_shell_workflow_recipe_dir/recipe.sh +0 -0
  105. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/test_viz_workflow_recipe_dir/config.json +0 -0
  106. {ogdc_runner-0.2.0 → ogdc_runner-0.3.0}/tests/unit/test_package.py +0 -0
@@ -0,0 +1,10 @@
1
+ # Ignore everything by default
2
+ **
3
+
4
+ # Allow only build-required files
5
+
6
+ !Dockerfile*
7
+ !src/
8
+ !pyproject.toml
9
+ !LICENSE
10
+ !README.md
@@ -35,7 +35,7 @@ jobs:
35
35
  runs-on: "ubuntu-latest"
36
36
 
37
37
  steps:
38
- - uses: "actions/checkout@v5"
38
+ - uses: "actions/checkout@v6"
39
39
  with:
40
40
  fetch-depth: 0
41
41
 
@@ -49,14 +49,14 @@ jobs:
49
49
  runs-on: "ubuntu-latest"
50
50
  steps:
51
51
  - name: "Check out repository"
52
- uses: "actions/checkout@v5"
52
+ uses: "actions/checkout@v6"
53
53
 
54
54
  - name: "Build container image"
55
55
  run: |
56
56
  docker build --tag "ghcr.io/qgreenland-net/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" .
57
57
  docker save "ghcr.io/qgreenland-net/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -o docker-image.tar
58
58
  - name: "Upload image tarball as artifact"
59
- uses: "actions/upload-artifact@v5"
59
+ uses: "actions/upload-artifact@v7"
60
60
  with:
61
61
  name: "docker-image-tar"
62
62
  path: "docker-image.tar"
@@ -75,7 +75,7 @@ jobs:
75
75
  runs-on: "ubuntu-latest"
76
76
 
77
77
  steps:
78
- - uses: "actions/download-artifact@v6"
78
+ - uses: "actions/download-artifact@v8"
79
79
  with:
80
80
  name: "Packages"
81
81
  path: "dist"
@@ -94,7 +94,7 @@ jobs:
94
94
  needs: ["build-python", "build-docker"]
95
95
  steps:
96
96
  - name: "Download image tarball"
97
- uses: "actions/download-artifact@v6"
97
+ uses: "actions/download-artifact@v8"
98
98
  with:
99
99
  name: "docker-image-tar"
100
100
 
@@ -125,7 +125,7 @@ jobs:
125
125
  runs-on: "ubuntu-latest"
126
126
  steps:
127
127
  - name: "Checkout"
128
- uses: "actions/checkout@v5"
128
+ uses: "actions/checkout@v6"
129
129
  - name: "Release"
130
130
  uses: "softprops/action-gh-release@v2"
131
131
  with:
@@ -17,23 +17,6 @@ env:
17
17
  FORCE_COLOR: 3
18
18
 
19
19
  jobs:
20
- typecheck:
21
- name: "Typecheck"
22
- runs-on: "ubuntu-latest"
23
- steps:
24
- - uses: "actions/checkout@v5"
25
- with:
26
- fetch-depth: 0
27
- - uses: "actions/setup-python@v6"
28
- with:
29
- python-version: "3.11"
30
-
31
- - name: "Install package"
32
- run: "python -m pip install .[test]"
33
-
34
- - name: "Typecheck package"
35
- run: "mypy"
36
-
37
20
  test:
38
21
  name: "Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }}"
39
22
  runs-on: "${{ matrix.runs-on }}"
@@ -44,7 +27,7 @@ jobs:
44
27
  runs-on: ["ubuntu-latest", "macos-latest", "windows-latest"]
45
28
 
46
29
  steps:
47
- - uses: "actions/checkout@v5"
30
+ - uses: "actions/checkout@v6"
48
31
  with:
49
32
  fetch-depth: 0
50
33
 
@@ -53,14 +36,18 @@ jobs:
53
36
  python-version: "${{ matrix.python-version }}"
54
37
  allow-prereleases: true
55
38
 
39
+ # The MacOS runner does not have the required postgresql binaries that are
40
+ # required to `pip install` psycopg2, which is required for the API to
41
+ # connect to the OGDC postgresql DB.
42
+ - name: "Install Postgresql on macOS"
43
+ if: runner.os == 'macOS'
44
+ run: |
45
+ brew update
46
+ brew install postgresql
47
+
56
48
  - name: "Install package"
57
49
  run: "python -m pip install .[test]"
58
50
 
59
51
  - name: "Test package"
60
52
  # prettier-ignore
61
- run: "python -m pytest -ra --durations=20 --cov --cov-report=xml --cov-report=term tests/unit"
62
-
63
- - name: "Upload coverage report"
64
- uses: "codecov/codecov-action@v5.5.1"
65
- with:
66
- token: "${{ secrets.CODECOV_TOKEN }}"
53
+ run: "nox -s test_ci"
@@ -34,14 +34,15 @@ repos:
34
34
  - id: rst-inline-touching-normal
35
35
 
36
36
  - repo: https://github.com/rbubley/mirrors-prettier
37
- rev: "v3.6.2"
37
+ rev: "v3.8.1"
38
38
  hooks:
39
39
  - id: prettier
40
40
  types_or: [yaml, markdown, html, css, scss, javascript, json]
41
41
  args: [--prose-wrap=always]
42
+ exclude: ^README\.md$
42
43
 
43
44
  - repo: https://github.com/astral-sh/ruff-pre-commit
44
- rev: "v0.14.3"
45
+ rev: "v0.15.4"
45
46
  hooks:
46
47
  - id: ruff
47
48
  args: ["--fix", "--show-fixes"]
@@ -58,13 +59,13 @@ repos:
58
59
  - id: shellcheck
59
60
 
60
61
  - repo: https://github.com/abravalheri/validate-pyproject
61
- rev: "v0.24.1"
62
+ rev: "v0.25"
62
63
  hooks:
63
64
  - id: validate-pyproject
64
65
  additional_dependencies: ["validate-pyproject-schema-store[all]"]
65
66
 
66
67
  - repo: https://github.com/python-jsonschema/check-jsonschema
67
- rev: "0.34.1"
68
+ rev: "0.37.0"
68
69
  hooks:
69
70
  - id: check-dependabot
70
71
  - id: check-github-workflows
@@ -1,3 +1,15 @@
1
+ # v0.3.0
2
+
3
+ - Create service interface for ogdc-runner.
4
+ - Refactor URL validation to use Pydantic model validators.
5
+ - Add recipe ID truncation to avoid Kubernetes name length limits.
6
+ - Add `temporary` output type and support retrieval of those outputs from s3
7
+ storage via the `ogdc-runner get-output` CLI.
8
+ - Add DataONE input type for fetching datasets directly from DataONE
9
+ repositories
10
+ - Add `parallel-shell` recipe type and related support for executing shell
11
+ recipes in parallel.
12
+
1
13
  # v0.2.0
2
14
 
3
15
  - Add support for both `local` (via rancher-desktop) and `dev` environments
@@ -0,0 +1,19 @@
1
+ # Docker image for:
2
+ # * Running gdal/ogr and fetch (wget) commands for ogdc recipes.
3
+ # * Provides the OGDC service interface (default CMD).
4
+ # WARNING: this image *MUST NOT* be based on busybox/alpine linux due to a known
5
+ # networking issue in non-local environments. For context, see:
6
+ # https://github.com/QGreenland-Net/ogdc-helm/issues/31
7
+ FROM ghcr.io/osgeo/gdal:ubuntu-full-3.12.0
8
+ # We use and wget to fetch data from remote sources.
9
+ # libpq-dev is required for connecting to postgresql db
10
+ RUN apt update && apt install -y wget rsync pip python3-venv libpq-dev zip netcdf-bin libnetcdf-dev
11
+
12
+ WORKDIR /ogdc_runner/
13
+ COPY . /ogdc_runner
14
+ RUN python -m venv .venv && . ./.venv/bin/activate && pip install --editable .
15
+
16
+ EXPOSE 8000
17
+
18
+ # Default to running the production fastapi server
19
+ CMD . ./.venv/bin/activate && fastapi run --port 8000 --host 0.0.0.0 src/ogdc_runner/service/main.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ogdc-runner
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Defines OGDC recipe API(s) and submits recipes to OGDC for execution
5
5
  Project-URL: Homepage, https://github.com/qgreenland-net/ogdc-runner
6
6
  Project-URL: Bug Tracker, https://github.com/qgreenland-net/ogdc-runner/issues
@@ -40,18 +40,27 @@ Classifier: Programming Language :: Python :: 3.12
40
40
  Classifier: Topic :: Scientific/Engineering
41
41
  Classifier: Typing :: Typed
42
42
  Requires-Python: >=3.11
43
+ Requires-Dist: boto3>=1.42
43
44
  Requires-Dist: click>=8
45
+ Requires-Dist: dataone-libclient>=3.5.0
46
+ Requires-Dist: fastapi[standard]
44
47
  Requires-Dist: fsspec
45
48
  Requires-Dist: hera>=5.18.0
46
49
  Requires-Dist: loguru
50
+ Requires-Dist: psycopg2-binary
51
+ Requires-Dist: pwdlib[argon2]
47
52
  Requires-Dist: pydantic>=2.7.1
48
- Requires-Dist: pyyaml
53
+ Requires-Dist: pyjwt
54
+ Requires-Dist: pyyaml>=6.0
55
+ Requires-Dist: requests
56
+ Requires-Dist: sqlmodel
49
57
  Provides-Extra: dev
50
58
  Requires-Dist: mypy~=1.10.0; extra == 'dev'
51
59
  Requires-Dist: nox; extra == 'dev'
52
60
  Requires-Dist: pytest-cov>=3; extra == 'dev'
53
61
  Requires-Dist: pytest>=6; extra == 'dev'
54
62
  Requires-Dist: types-pyyaml; extra == 'dev'
63
+ Requires-Dist: types-requests; extra == 'dev'
55
64
  Provides-Extra: docs
56
65
  Requires-Dist: autodoc-pydantic; extra == 'docs'
57
66
  Requires-Dist: furo>=2023.08.17; extra == 'docs'
@@ -62,21 +71,21 @@ Requires-Dist: sphinx>=7.0; extra == 'docs'
62
71
  Requires-Dist: sphinxcontrib-mermaid; extra == 'docs'
63
72
  Provides-Extra: test
64
73
  Requires-Dist: mypy~=1.10.0; extra == 'test'
74
+ Requires-Dist: nox; extra == 'test'
65
75
  Requires-Dist: pytest-cov>=3; extra == 'test'
66
76
  Requires-Dist: pytest-order; extra == 'test'
67
77
  Requires-Dist: pytest>=6; extra == 'test'
68
78
  Requires-Dist: types-pyyaml; extra == 'test'
79
+ Requires-Dist: types-requests; extra == 'test'
69
80
  Description-Content-Type: text/markdown
70
81
 
71
82
  # ogdc-runner
72
83
 
73
84
  <!-- SPHINX-START -->
74
85
 
75
- [![Actions Status][actions-badge]][actions-link]
76
86
  [![Documentation Status][rtd-badge]][rtd-link]
77
87
 
78
88
  [![PyPI version][pypi-version]][pypi-link]
79
- [![Conda-Forge][conda-badge]][conda-link]
80
89
  [![PyPI platforms][pypi-platforms]][pypi-link]
81
90
 
82
91
  [![GitHub Discussion][github-discussions-badge]][github-discussions-link]
@@ -87,11 +96,18 @@ OGDC for execution.
87
96
  See the [ogdc-runner ReadTheDocs](https://ogdc-runner.readthedocs.io/) for
88
97
  detailed documentation on how to use and contribute to the `ogdc-runner`.
89
98
 
99
+ ## Citation
100
+
101
+ Cite this software as:
102
+
103
+ > Trey Stafford, Robyn Marowitz, Rushiraj Nenuji, Matthew B. Jones, Twila
104
+ > Moon. 2025. Open Geospatial Data Cloud (OGDC) Runner: Client tool and API for
105
+ > scalable and repeatable geospatial data transformation recipes. (version
106
+ > 0.2.0). Arctic Data Center.
107
+ > [doi:10.18739/A2MS3K382](https://doi.org/doi:10.18739/A2MS3K382)
108
+
90
109
  <!-- prettier-ignore-start -->
91
- [actions-badge]: https://github.com/qgreenland-net/ogdc-runner/workflows/CI/badge.svg
92
110
  [actions-link]: https://github.com/qgreenland-net/ogdc-runner/actions
93
- [conda-badge]: https://img.shields.io/conda/vn/conda-forge/ogdc-runner
94
- [conda-link]: https://github.com/conda-forge/ogdc-runner-feedstock
95
111
  [github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
96
112
  [github-discussions-link]: https://github.com/qgreenland-net/ogdc-runner/discussions
97
113
  [pypi-link]: https://pypi.org/project/ogdc-runner/
@@ -2,11 +2,9 @@
2
2
 
3
3
  <!-- SPHINX-START -->
4
4
 
5
- [![Actions Status][actions-badge]][actions-link]
6
5
  [![Documentation Status][rtd-badge]][rtd-link]
7
6
 
8
7
  [![PyPI version][pypi-version]][pypi-link]
9
- [![Conda-Forge][conda-badge]][conda-link]
10
8
  [![PyPI platforms][pypi-platforms]][pypi-link]
11
9
 
12
10
  [![GitHub Discussion][github-discussions-badge]][github-discussions-link]
@@ -17,11 +15,18 @@ OGDC for execution.
17
15
  See the [ogdc-runner ReadTheDocs](https://ogdc-runner.readthedocs.io/) for
18
16
  detailed documentation on how to use and contribute to the `ogdc-runner`.
19
17
 
18
+ ## Citation
19
+
20
+ Cite this software as:
21
+
22
+ > Trey Stafford, Robyn Marowitz, Rushiraj Nenuji, Matthew B. Jones, Twila
23
+ > Moon. 2025. Open Geospatial Data Cloud (OGDC) Runner: Client tool and API for
24
+ > scalable and repeatable geospatial data transformation recipes. (version
25
+ > 0.2.0). Arctic Data Center.
26
+ > [doi:10.18739/A2MS3K382](https://doi.org/doi:10.18739/A2MS3K382)
27
+
20
28
  <!-- prettier-ignore-start -->
21
- [actions-badge]: https://github.com/qgreenland-net/ogdc-runner/workflows/CI/badge.svg
22
29
  [actions-link]: https://github.com/qgreenland-net/ogdc-runner/actions
23
- [conda-badge]: https://img.shields.io/conda/vn/conda-forge/ogdc-runner
24
- [conda-link]: https://github.com/conda-forge/ogdc-runner-feedstock
25
30
  [github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
26
31
  [github-discussions-link]: https://github.com/qgreenland-net/ogdc-runner/discussions
27
32
  [pypi-link]: https://pypi.org/project/ogdc-runner/
@@ -9,11 +9,21 @@ description of best practices for developing scientific packages.
9
9
  Add yourself as an author in `pyproject.toml`
10
10
  ```
11
11
 
12
+ ## Familiarize yourself with the project
13
+
14
+ - Be sure to look over the [Architecture](./architecture/index.md) docs to get
15
+ an understanding of the `ogdc-runner` before contributing any code.
16
+
17
+ - The `ogdc-runner` is one component of the QGreenland-Net Open Geospatial Data
18
+ Cloud (OGDC). See the [QGreenland-Net](https://qgreenland-net.github.io/)
19
+ webpage for more information about how this project fits into that larger
20
+ effort. In particular, check out the
21
+ [QGreenland-Net Contributing docs](https://qgreenland-net.github.io/contributing/)!
22
+
12
23
  ## Setting up a local development environment
13
24
 
14
25
  First, ensure you have [ogdc-helm](https://github.com/QGreenland-Net/ogdc-helm)
15
- setup for local development with `rancher-desktop`. The argo server ports are
16
- expected to be forwarded for access via localhost.
26
+ setup for local development with `rancher-desktop` using `skaffold`.
17
27
 
18
28
  Now you can set up a python development environment for `ogdc-runner` by
19
29
  running:
@@ -24,33 +34,10 @@ source ./.venv/bin/activate
24
34
  pip install -v --editable ".[dev]"
25
35
  ```
26
36
 
27
- ### Using a local docker image for workflow execution
28
-
29
- The `ogdc-runner` supports using a local `ogdc-runner` image for development
30
- purposes (e.g., you want to change and test something about the image without
31
- needing to release it to the GHCR).
32
-
33
- First, build a local image:
34
-
35
- ```{note}
36
- The docker image must be built in the `rancher-desktop` context so
37
- that it is available to the Argo deployment on the developer's local machine.
38
- Check that you have the correct context selected with `docker context ls`.
39
- ```
40
-
41
- ```{warning}
42
- The `ogdc-runner` docker image (and any others intended to be run on the k8s
43
- cluster as part of the OGDC) **MUST NOT** be based on busybox/alpine Linux due
44
- to a known networking issue in non-local environments. For context, see:
45
- <https://github.com/QGreenland-Net/ogdc-helm/issues/31>
46
- ```
47
-
48
- ```
49
- docker build . -t ogdc-runner
50
- ```
37
+ ### Required environment variables
51
38
 
52
- Next, set the `ENVIRONMENT` envvar to `local`. This will tell `ogdc-runner` to
53
- use the locally built image instead of the one hosted on the GHCR:
39
+ Set the `ENVIRONMENT` envvar to `local`. This will tell `ogdc-runner` to operate
40
+ in "local" development mode:
54
41
 
55
42
  ```
56
43
  export ENVIRONMENT=local
@@ -80,17 +67,14 @@ be re-pulled, ensuring that the latest `main` branch image is being used.
80
67
 
81
68
  ## Testing, linting, rendering docs with Nox
82
69
 
83
- The fastest way to start is to use Nox. If you don't have Nox, you can use
84
- `pipx run nox` to run it without installing, or `pipx install nox`. If you don't
85
- have pipx, then you can install with `pip install pipx`. If you use macOS, use
86
- `brew install pipx nox`. To use:
70
+ To run all tests, simply run `nox`:
87
71
 
88
72
  ```console
89
73
  nox
90
74
  ```
91
75
 
92
- This will test using every installed version of Python on your system, skipping
93
- ones that are not installed.
76
+ This will run the typechecker, unit, and integration tests (requiring
77
+ `ogdc-helm` to be deployed locally).
94
78
 
95
79
  ### Running specific tasks with Nox
96
80
 
@@ -98,6 +82,13 @@ ones that are not installed.
98
82
  nox -s {job-name}
99
83
  ```
100
84
 
85
+ For example, to run only the tests run in CI, which are fast and do not require
86
+ a locally deployed OGDC stack:
87
+
88
+ ```
89
+ nox -s test_ci
90
+ ```
91
+
101
92
  To view available jobs:
102
93
 
103
94
  ```console
@@ -107,6 +98,28 @@ nox -l
107
98
  Nox handles everything for you, including setting up an temporary virtual
108
99
  environment for each run.
109
100
 
101
+ ### Testing
102
+
103
+ Use `nox` to run all tests (with pytest):
104
+
105
+ ```bash
106
+ nox
107
+ ```
108
+
109
+ ### Building docs
110
+
111
+ You can build the docs using:
112
+
113
+ ```bash
114
+ nox -s docs
115
+ ```
116
+
117
+ You can see a preview with:
118
+
119
+ ```bash
120
+ nox -s docs -- --serve
121
+ ```
122
+
110
123
  ### Reusing Nox virtual environments
111
124
 
112
125
  **By default, Nox deletes and recreates virtual environments for every run.**
@@ -131,36 +144,6 @@ pre-commit install # install Git pre-commit hook from .pre-commit-config.yml
131
144
  You can also/alternatively run `pre-commit run` (will run for changed files
132
145
  only) or `pre-commit run --all-files` to check even without installing the hook.
133
146
 
134
- ## Testing
135
-
136
- Use pytest to run the unit checks:
137
-
138
- ```bash
139
- pytest
140
- ```
141
-
142
- ### Coverage
143
-
144
- Use pytest-cov to generate coverage reports:
145
-
146
- ```bash
147
- pytest --cov=ogdc-runner
148
- ```
149
-
150
- ## Building docs
151
-
152
- You can build the docs using:
153
-
154
- ```bash
155
- nox -s docs
156
- ```
157
-
158
- You can see a preview with:
159
-
160
- ```bash
161
- nox -s docs -- --serve
162
- ```
163
-
164
147
  ## Continuous Integration
165
148
 
166
149
  This project uses [GitHub Actions](https://docs.github.com/en/actions) to
@@ -189,7 +172,18 @@ This project uses [semantic versioning](https://semver.org/).
189
172
  > 3. PATCH version when you make backward compatible bug fixes
190
173
 
191
174
  Decide what the version will be for your release, and ensure that the CHANGELOG
192
- contains an entry for the planned release.
175
+ contains an entry for the `## NEXT_VERSION`.
176
+
177
+ **Bump the Version**
178
+
179
+ Use bump-my-version to automatically update the version number in all configured
180
+ files (e.g., pyproject.toml, CHANGELOG.md).
181
+
182
+ Choose the appropriate part to bump:
183
+
184
+ - PATCH release: `bump-my-version bump patch`
185
+ - MINOR release: `bump-my-version bump minor`
186
+ - MAJOR release: `bump-my-version bump major`
193
187
 
194
188
  Once `main` is ready for a release (feature branches are merged and the
195
189
  CHANGELOG is up-to-date), tag the latest commit with the version to be released
@@ -0,0 +1,58 @@
1
+ # Argo Workflows
2
+
3
+ [OGDC recipes](../recipes.md) are executed via one or more
4
+ [argo-workflows](https://argo-workflows.readthedocs.io/). The
5
+ [OGDC Service API](./service.md) is responsible for translating user recipes
6
+ into argo workflows that are submitted and executed by Argo.
7
+
8
+ This document includes details of how Argo is setup and used by the OGDC.
9
+
10
+ ## argo-workflows server configuration
11
+
12
+ The configuration for argo-workflows is defined in the
13
+ [ogdc-helm](https://github.com/qgreenland-net/ogdc-helm) repository. Refer to
14
+ the configurations there to see the specifics of how argo is setup for
15
+ deployment.
16
+
17
+ ## Argo Python API (Hera)
18
+
19
+ The OGDC uses the [hera](https://hera.readthedocs.io/en/stable/) python package
20
+ to interact with the argo-workflows API. Boilerplate setup and configuration of
21
+ argo via hera can be found in {mod}`ogdc_runner.argo`.
22
+
23
+ ## Workflow artifacts
24
+
25
+ Argo
26
+ [artifacts](https://argo-workflows.readthedocs.io/en/latest/walk-through/artifacts/)
27
+ are configured to use an S3-compatible
28
+ [artifact repository](https://argo-workflows.readthedocs.io/en/latest/configure-artifact-repository/).
29
+
30
+ Artifacts are used to store intermediate workflow outputs and the final output
31
+ of recipes with a `temporary` output type.
32
+
33
+ Artifacts will be automatically garbage collected on workflow deletion. See the
34
+ {ref}`workflow-persistence` section below for details on automatic workflow
35
+ deletion.
36
+
37
+ <!-- prettier-ignore-start -->
38
+ (workflow-persistence)=
39
+ ## Workflow persistence
40
+ <!-- prettier-ignore-end -->
41
+
42
+ Successful Argo workflows are retained for 1 day. Workflows associated with
43
+ recipes with the `temporary` output type are retained for 7 days to allow for
44
+ sufficient time to retrieve final outputs.
45
+
46
+ Successful workflows with the `ogdc/persist-workflow-in-archive: true` label
47
+ will be archived in the OGDC's postgresql database for long-term storage. These
48
+ archived workflows can be used for metrics and data provenance purposes.
49
+
50
+ Non-successful workflows are not automatically cleaned up or archived. They are
51
+ retained for inspection/debugging and should be cleaned up manually once the
52
+ issue leading to failure is resolved.
53
+
54
+ To ensure consistent behavior of OGDC-submitted argo workflows (e.g., setting
55
+ the archival label and TTL for successful workflows), the
56
+ {class}`ogdc_runner.argo.OgdcWorkflow` context manager has been defined to wrap
57
+ the behavior of [hera's](https://hera.readthedocs.io/en/stable/)
58
+ [Workflow](https://hera.readthedocs.io/en/stable/api/workflows/workflow_classes/workflow/#hera.workflows.workflow.Workflow).
@@ -0,0 +1,110 @@
1
+ # Architecture
2
+
3
+ ```{toctree}
4
+ :maxdepth: 1
5
+ :hidden:
6
+
7
+ service.md
8
+ argo.md
9
+ ```
10
+
11
+ ```{mermaid}
12
+ graph LR
13
+
14
+ %% Definitions
15
+ subgraph ADC_K8S[ADC k8s]
16
+ subgraph OGDC[OGDC]
17
+ OGDC_RUNNER_SERVICE[ogdc-runner service]
18
+ ARGO[Argo Workflows]
19
+ DB[(PostgreSQL Database)]
20
+ end
21
+ end
22
+
23
+ subgraph OGDC_RUNNER_CLI[ogdc-runner CLI]
24
+ end
25
+
26
+ %% Relationships
27
+ OGDC_RUNNER_CLI -->|"submit|status"| OGDC_RUNNER_SERVICE
28
+
29
+ OGDC_RUNNER_SERVICE -->|"create/query Argo workflow(s) required to execute recipe instructions"|ARGO
30
+ OGDC_RUNNER_SERVICE -->|"create/query Users"|DB
31
+
32
+ %% Style
33
+ style OGDC_RUNNER_SERVICE stroke:#ff6666,stroke-width:2px;
34
+ style OGDC_RUNNER_CLI stroke:#ff6666,stroke-width:2px;
35
+ ```
36
+
37
+ [Please view our main documentation site for context](https://qgreenland-net.github.io).
38
+
39
+ This component:
40
+
41
+ - Defines and documents the recipe API(s). See [OGDC Recipes](../recipes.md).
42
+ - Defines a CLI published as the `ogdc-runner` pypi Python package that accepts
43
+ a recipe as input and submits it to the service layer.
44
+ - Defines a service layer published as the `ghcr.io/qgreenland-net/ogdc-runner`
45
+ docker image that translates submitted OGDC recipes into Argo Workflows for
46
+ execution. See [OGDC Service API](./service.md) for more details.
47
+
48
+ ## Parallel Execution
49
+
50
+ The ogdc-runner supports parallel execution of workflow tasks via Argo's DAG
51
+ (Directed Acyclic Graph) model. When a recipe enables parallel execution, the
52
+ runner:
53
+
54
+ 1. **Partitions** input data based on the configured strategy (e.g., grouping
55
+ files)
56
+ 2. **Creates** independent Argo tasks for each partition
57
+ 3. **Orchestrates** parallel execution with configurable maximum parallelism
58
+
59
+ The {class}`ogdc_runner.parallel.ParallelExecutionOrchestrator` class manages
60
+ this process, creating Argo Container templates and DAG tasks with proper
61
+ dependencies and parameters. Argo handles task scheduling and resource
62
+ allocation, distributing work across available cluster resources.
63
+
64
+ Key modules:
65
+
66
+ - {mod}`ogdc_runner.parallel`: Orchestration logic for parallel task creation
67
+ - {mod}`ogdc_runner.partitioning`: Partitioning strategies for dividing work
68
+ - {mod}`ogdc_runner.models.parallel_config`: Configuration models for parallel
69
+ execution
70
+
71
+ ## Aspirational
72
+
73
+ Original aspirational architecture diagram of the `ogdc-runner` as it relates to
74
+ the rest of the OGDC.
75
+
76
+ ```{warning}
77
+ This diagram does not reflect the actual, current implementation. This remains for reference only.
78
+ ```
79
+
80
+ ```{mermaid}
81
+ graph LR
82
+
83
+ %% Definitions
84
+ subgraph ADC_K8S[ADC k8s]
85
+ subgraph GHA_SELFHOSTED[GHA self-hosted runner]
86
+ OGDC_RUNNER[ogdc-runner]
87
+ end
88
+ OGDC[OGDC]
89
+ end
90
+
91
+ subgraph RECIPE_REPO[Recipe repo]
92
+ GHA[GitHub Actions]
93
+ RECIPE[Recipe]
94
+ SECRET[Secret token]
95
+ end
96
+
97
+
98
+
99
+ %% Relationships
100
+ OGDC_RUNNER -->|submit| OGDC
101
+ GHA_SELFHOSTED -->|status| GHA
102
+ GHA -->|trigger| GHA_SELFHOSTED
103
+
104
+ SECRET -->|read| GHA
105
+ RECIPE -->|on change| GHA
106
+
107
+
108
+ %% Style
109
+ style OGDC_RUNNER stroke:#ff6666,stroke-width:2px;
110
+ ```