ogdc-runner 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ogdc_runner/__init__.py +11 -0
- ogdc_runner/__main__.py +74 -0
- ogdc_runner/_version.py +34 -0
- ogdc_runner/_version.pyi +4 -0
- ogdc_runner/api.py +51 -0
- ogdc_runner/argo.py +199 -0
- ogdc_runner/constants.py +11 -0
- ogdc_runner/exceptions.py +25 -0
- ogdc_runner/inputs.py +61 -0
- ogdc_runner/models/__init__.py +0 -0
- ogdc_runner/models/recipe_config.py +85 -0
- ogdc_runner/publish.py +168 -0
- ogdc_runner/py.typed +0 -0
- ogdc_runner/recipe/__init__.py +24 -0
- ogdc_runner/recipe/shell.py +119 -0
- ogdc_runner/recipe/viz_workflow.py +286 -0
- ogdc_runner-0.1.0.dist-info/METADATA +102 -0
- ogdc_runner-0.1.0.dist-info/RECORD +21 -0
- ogdc_runner-0.1.0.dist-info/WHEEL +4 -0
- ogdc_runner-0.1.0.dist-info/entry_points.txt +2 -0
- ogdc_runner-0.1.0.dist-info/licenses/LICENSE +19 -0
ogdc_runner/__init__.py
ADDED
ogdc_runner/__main__.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from pydantic import ValidationError
|
|
7
|
+
|
|
8
|
+
from ogdc_runner.api import submit_ogdc_recipe
|
|
9
|
+
from ogdc_runner.argo import get_workflow_status
|
|
10
|
+
from ogdc_runner.recipe import get_recipe_config
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.group
|
|
14
|
+
def cli() -> None:
|
|
15
|
+
"""A tool for submitting data transformation recipes to OGDC for execution."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@cli.command
|
|
19
|
+
@click.argument(
|
|
20
|
+
"recipe_path",
|
|
21
|
+
required=True,
|
|
22
|
+
metavar="RECIPE-PATH",
|
|
23
|
+
type=str,
|
|
24
|
+
)
|
|
25
|
+
@click.option(
|
|
26
|
+
"--wait",
|
|
27
|
+
is_flag=True,
|
|
28
|
+
default=False,
|
|
29
|
+
help="Wait for recipe execution to complete.",
|
|
30
|
+
)
|
|
31
|
+
@click.option(
|
|
32
|
+
"--overwrite",
|
|
33
|
+
is_flag=True,
|
|
34
|
+
default=False,
|
|
35
|
+
help="Overwrite existing outputs of the given recipe if it has already run before.",
|
|
36
|
+
)
|
|
37
|
+
def submit(recipe_path: str, wait: bool, overwrite: bool) -> None:
|
|
38
|
+
"""
|
|
39
|
+
Submit a recipe to OGDC for execution.
|
|
40
|
+
|
|
41
|
+
RECIPE-PATH: Path to the recipe file. Use either a local path (e.g., '/ogdc-recipes/recipes/seal-tags')
|
|
42
|
+
or an fsspec-compatible GitHub string (e.g., 'github://qgreenland-net:ogdc-recipes@main/recipes/seal-tags').
|
|
43
|
+
"""
|
|
44
|
+
submit_ogdc_recipe(recipe_dir=recipe_path, wait=wait, overwrite=overwrite)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@cli.command
|
|
48
|
+
@click.argument(
|
|
49
|
+
"workflow_name",
|
|
50
|
+
required=True,
|
|
51
|
+
type=str,
|
|
52
|
+
)
|
|
53
|
+
def check_workflow_status(workflow_name: str) -> None:
|
|
54
|
+
"""Check an argo workflow's status."""
|
|
55
|
+
status = get_workflow_status(workflow_name)
|
|
56
|
+
print(f"Workflow {workflow_name} has status {status}.")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@cli.command
|
|
60
|
+
@click.argument(
|
|
61
|
+
"recipe_path",
|
|
62
|
+
required=True,
|
|
63
|
+
metavar="RECIPE-PATH",
|
|
64
|
+
type=str,
|
|
65
|
+
)
|
|
66
|
+
def validate_recipe(recipe_path: str) -> None:
|
|
67
|
+
"""Validate an OGDC recipe directory."""
|
|
68
|
+
try:
|
|
69
|
+
get_recipe_config(recipe_path)
|
|
70
|
+
print(f"Recipe {recipe_path} is valid.")
|
|
71
|
+
except ValidationError as err:
|
|
72
|
+
print(f"Recipe {recipe_path} is invalid.")
|
|
73
|
+
print(err)
|
|
74
|
+
sys.exit(1)
|
ogdc_runner/_version.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
TYPE_CHECKING = False
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
20
|
+
else:
|
|
21
|
+
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
23
|
+
|
|
24
|
+
version: str
|
|
25
|
+
__version__: str
|
|
26
|
+
__version_tuple__: VERSION_TUPLE
|
|
27
|
+
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
30
|
+
|
|
31
|
+
__version__ = version = '0.1.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|
ogdc_runner/_version.pyi
ADDED
ogdc_runner/api.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""API for interacting with the OGDC."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from ogdc_runner.exceptions import OgdcDataAlreadyPublished
|
|
6
|
+
from ogdc_runner.publish import data_already_published
|
|
7
|
+
from ogdc_runner.recipe import get_recipe_config
|
|
8
|
+
from ogdc_runner.recipe.shell import make_and_submit_shell_workflow
|
|
9
|
+
from ogdc_runner.recipe.viz_workflow import submit_viz_workflow_recipe
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def submit_ogdc_recipe(
|
|
13
|
+
*,
|
|
14
|
+
recipe_dir: str,
|
|
15
|
+
wait: bool,
|
|
16
|
+
overwrite: bool,
|
|
17
|
+
) -> str:
|
|
18
|
+
"""Submit an OGDC recipe for processing via argo workflows.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
recipe_dir: Path to the recipe directory
|
|
22
|
+
wait: Whether to wait for the workflow to complete
|
|
23
|
+
overwrite: Whether to overwrite existing published data
|
|
24
|
+
|
|
25
|
+
Returns the name of the OGDC shell recipe submitted to Argo.
|
|
26
|
+
"""
|
|
27
|
+
# Get the recipe configuration
|
|
28
|
+
recipe_config = get_recipe_config(recipe_dir)
|
|
29
|
+
|
|
30
|
+
# Check if the user-submitted workflow has already been published
|
|
31
|
+
if data_already_published(
|
|
32
|
+
recipe_config=recipe_config,
|
|
33
|
+
overwrite=overwrite,
|
|
34
|
+
):
|
|
35
|
+
err_msg = f"Data for recipe {recipe_config.id} have already been published."
|
|
36
|
+
raise OgdcDataAlreadyPublished(err_msg)
|
|
37
|
+
|
|
38
|
+
# Check if the recipe is a visualization workflow
|
|
39
|
+
if recipe_config.id == "viz-workflow":
|
|
40
|
+
return submit_viz_workflow_recipe(
|
|
41
|
+
recipe_dir=recipe_dir,
|
|
42
|
+
wait=wait,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# We currently expect all recipes to be "shell"
|
|
46
|
+
shell_recipe_workflow_name = make_and_submit_shell_workflow(
|
|
47
|
+
recipe_config=recipe_config,
|
|
48
|
+
wait=wait,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
return shell_recipe_workflow_name
|
ogdc_runner/argo.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
from hera.shared import global_config
|
|
7
|
+
from hera.workflows import (
|
|
8
|
+
Container,
|
|
9
|
+
Workflow,
|
|
10
|
+
WorkflowsService,
|
|
11
|
+
models,
|
|
12
|
+
)
|
|
13
|
+
from loguru import logger
|
|
14
|
+
|
|
15
|
+
from ogdc_runner.exceptions import OgdcWorkflowExecutionError
|
|
16
|
+
|
|
17
|
+
OGDC_WORKFLOW_PVC = models.Volume(
|
|
18
|
+
name="workflow-volume",
|
|
19
|
+
persistent_volume_claim=models.PersistentVolumeClaimVolumeSource(
|
|
20
|
+
claim_name="cephfs-qgnet-ogdc-workflow-pvc",
|
|
21
|
+
),
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ArgoConfig:
|
|
26
|
+
"""Configuration for Argo workflows."""
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
namespace: str,
|
|
31
|
+
service_account_name: str,
|
|
32
|
+
workflows_service_url: str,
|
|
33
|
+
runner_image: str,
|
|
34
|
+
runner_image_tag: str,
|
|
35
|
+
image_pull_policy: str,
|
|
36
|
+
):
|
|
37
|
+
self.namespace = namespace
|
|
38
|
+
self.service_account_name = service_account_name
|
|
39
|
+
self.workflows_service_url = workflows_service_url
|
|
40
|
+
self.runner_image = runner_image
|
|
41
|
+
self.runner_image_tag = runner_image_tag
|
|
42
|
+
self.image_pull_policy = image_pull_policy
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def full_image_path(self) -> str:
|
|
46
|
+
"""Return the full image path with tag."""
|
|
47
|
+
return f"{self.runner_image}:{self.runner_image_tag}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ArgoManager:
|
|
51
|
+
"""Manager for Argo workflow configurations and services."""
|
|
52
|
+
|
|
53
|
+
def __init__(self) -> None:
|
|
54
|
+
self._config = self._initialize_config()
|
|
55
|
+
self._workflow_service = self._setup_workflow_service()
|
|
56
|
+
self._apply_global_config()
|
|
57
|
+
|
|
58
|
+
def _initialize_config(self) -> ArgoConfig:
|
|
59
|
+
"""Initialize Argo configuration from environment variables with defaults."""
|
|
60
|
+
is_dev_environment = os.environ.get("ENVIRONMENT") == "dev"
|
|
61
|
+
|
|
62
|
+
# Default runner image configuration
|
|
63
|
+
runner_image = (
|
|
64
|
+
"ogdc-runner"
|
|
65
|
+
if is_dev_environment
|
|
66
|
+
else "ghcr.io/qgreenland-net/ogdc-runner"
|
|
67
|
+
)
|
|
68
|
+
runner_image_tag = os.environ.get("OGDC_RUNNER_IMAGE_TAG", "latest")
|
|
69
|
+
image_pull_policy = "Never" if is_dev_environment else "IfNotPresent"
|
|
70
|
+
|
|
71
|
+
return ArgoConfig(
|
|
72
|
+
namespace=os.environ.get("ARGO_NAMESPACE", "qgnet"),
|
|
73
|
+
service_account_name=os.environ.get(
|
|
74
|
+
"ARGO_SERVICE_ACCOUNT_NAME", "argo-workflow"
|
|
75
|
+
),
|
|
76
|
+
workflows_service_url=os.environ.get(
|
|
77
|
+
"ARGO_WORKFLOWS_SERVICE_URL", "http://localhost:2746"
|
|
78
|
+
),
|
|
79
|
+
runner_image=runner_image,
|
|
80
|
+
runner_image_tag=runner_image_tag,
|
|
81
|
+
image_pull_policy=image_pull_policy,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def _setup_workflow_service(self) -> WorkflowsService:
|
|
85
|
+
"""Set up and return the Argo WorkflowsService with namespace set."""
|
|
86
|
+
return WorkflowsService(
|
|
87
|
+
host=self._config.workflows_service_url, namespace=self._config.namespace
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
def _apply_global_config(self) -> None:
|
|
91
|
+
"""Apply the current configuration to Hera's global config."""
|
|
92
|
+
global_config.namespace = self._config.namespace
|
|
93
|
+
global_config.service_account_name = self._config.service_account_name
|
|
94
|
+
global_config.image = self._config.full_image_path
|
|
95
|
+
|
|
96
|
+
global_config.set_class_defaults(
|
|
97
|
+
Container,
|
|
98
|
+
image_pull_policy=self._config.image_pull_policy,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
global_config.set_class_defaults(
|
|
102
|
+
Workflow,
|
|
103
|
+
# Setup artifact garbage collection
|
|
104
|
+
artifact_gc=models.ArtifactGC(
|
|
105
|
+
strategy="OnWorkflowDeletion",
|
|
106
|
+
service_account_name=self._config.service_account_name,
|
|
107
|
+
),
|
|
108
|
+
# Setup default OGDC workflow pvc
|
|
109
|
+
volumes=[OGDC_WORKFLOW_PVC],
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def workflow_service(self) -> WorkflowsService:
|
|
114
|
+
"""Get the current workflow service."""
|
|
115
|
+
return self._workflow_service
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def config(self) -> ArgoConfig:
|
|
119
|
+
"""Get the current configuration."""
|
|
120
|
+
return self._config
|
|
121
|
+
|
|
122
|
+
def update_image(
|
|
123
|
+
self,
|
|
124
|
+
image: str | None = None,
|
|
125
|
+
tag: str | None = None,
|
|
126
|
+
pull_policy: str | None = None,
|
|
127
|
+
) -> None:
|
|
128
|
+
"""
|
|
129
|
+
Update the runner image configuration and re-apply global config.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
image: New image path (without tag)
|
|
133
|
+
tag: New image tag
|
|
134
|
+
pull_policy: New image pull policy
|
|
135
|
+
"""
|
|
136
|
+
if image is not None:
|
|
137
|
+
self._config.runner_image = image
|
|
138
|
+
|
|
139
|
+
if tag is not None:
|
|
140
|
+
self._config.runner_image_tag = tag
|
|
141
|
+
|
|
142
|
+
if pull_policy is not None:
|
|
143
|
+
self._config.image_pull_policy = pull_policy
|
|
144
|
+
|
|
145
|
+
# Re-apply global config with updated values
|
|
146
|
+
self._apply_global_config()
|
|
147
|
+
logger.info(
|
|
148
|
+
f"Updated runner image to {self._config.full_image_path} with pull policy {self._config.image_pull_policy}"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Initialize the ArgoManager as a singleton
|
|
153
|
+
ARGO_MANAGER: ArgoManager = ArgoManager()
|
|
154
|
+
ARGO_WORKFLOW_SERVICE: WorkflowsService = ARGO_MANAGER.workflow_service
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def get_workflow_status(workflow_name: str) -> str | None:
|
|
158
|
+
"""Return the given workflow's status (e.g., `'Succeeded'`)"""
|
|
159
|
+
workflow = ARGO_WORKFLOW_SERVICE.get_workflow(name=workflow_name)
|
|
160
|
+
|
|
161
|
+
status: str | None = workflow.status.phase # type: ignore[union-attr]
|
|
162
|
+
|
|
163
|
+
return status
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def wait_for_workflow_completion(workflow_name: str) -> None:
|
|
167
|
+
while True:
|
|
168
|
+
status = get_workflow_status(workflow_name)
|
|
169
|
+
if status:
|
|
170
|
+
logger.info(f"Workflow status: {status}")
|
|
171
|
+
# Terminal states
|
|
172
|
+
if status == "Failed":
|
|
173
|
+
raise OgdcWorkflowExecutionError(
|
|
174
|
+
f"Workflow with name {workflow_name} failed."
|
|
175
|
+
)
|
|
176
|
+
if status == "Succeeded":
|
|
177
|
+
return
|
|
178
|
+
time.sleep(5)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def submit_workflow(workflow: Workflow, *, wait: bool = False) -> str:
|
|
182
|
+
"""Submit the given workflow and return its name as a str."""
|
|
183
|
+
workflow.create()
|
|
184
|
+
|
|
185
|
+
workflow_name = workflow.name
|
|
186
|
+
|
|
187
|
+
# mypy seems to think that the workflow name might be `None`. I have not
|
|
188
|
+
# encountered this case, but maybe it would indicate a problem we should be
|
|
189
|
+
# aware of?
|
|
190
|
+
if workflow_name is None:
|
|
191
|
+
err_msg = "Problem with submitting workflow."
|
|
192
|
+
raise OgdcWorkflowExecutionError(err_msg)
|
|
193
|
+
|
|
194
|
+
logger.success(f"Successfully submitted workflow with name {workflow_name}")
|
|
195
|
+
|
|
196
|
+
if wait:
|
|
197
|
+
wait_for_workflow_completion(workflow_name)
|
|
198
|
+
|
|
199
|
+
return workflow_name
|
ogdc_runner/constants.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# .txt? To make sure it's clear the file can't be executed directly
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
SHELL_RECIPE_FILENAME = "recipe.sh"
|
|
5
|
+
# .yaml and .yml?
|
|
6
|
+
RECIPE_CONFIG_FILENAME = "meta.yml"
|
|
7
|
+
|
|
8
|
+
# filename for driving the viz workflow
|
|
9
|
+
VIZ_RECIPE_FILENAME = "config.json"
|
|
10
|
+
|
|
11
|
+
VIZ_RECIPE_BATCH_SIZE = 250
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class OgdcRunnerException(Exception):
|
|
5
|
+
"""Base exception for `ogdc-runner`."""
|
|
6
|
+
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OgdcDataAlreadyPublished(OgdcRunnerException):
|
|
11
|
+
"""Exception that is raised when data for a recipe have already been published."""
|
|
12
|
+
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class OgdcWorkflowExecutionError(OgdcRunnerException):
|
|
17
|
+
"""Exception for unexpected failures resulting from argo workflow executions."""
|
|
18
|
+
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class OgdcInvalidRecipeDir(OgdcRunnerException):
|
|
23
|
+
"""Exception that is raised when recipe directory is not available/valid."""
|
|
24
|
+
|
|
25
|
+
pass
|
ogdc_runner/inputs.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Code for accessing input data of OGDC recipes"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from hera.workflows import (
|
|
6
|
+
Artifact,
|
|
7
|
+
Container,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
from ogdc_runner.exceptions import OgdcWorkflowExecutionError
|
|
11
|
+
from ogdc_runner.models.recipe_config import RecipeConfig
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_fetch_input_template(
|
|
15
|
+
recipe_config: RecipeConfig,
|
|
16
|
+
) -> Container:
|
|
17
|
+
"""Creates a container template that fetches multiple inputs from URLs or file paths.
|
|
18
|
+
|
|
19
|
+
Supports:
|
|
20
|
+
- HTTP/HTTPS URLs
|
|
21
|
+
- File paths (including PVC paths)
|
|
22
|
+
"""
|
|
23
|
+
# Create commands to fetch each input
|
|
24
|
+
fetch_commands = []
|
|
25
|
+
|
|
26
|
+
for param in recipe_config.input.params:
|
|
27
|
+
# Check if the parameter is a URL
|
|
28
|
+
if param.type == "url":
|
|
29
|
+
# It's a URL, use wget
|
|
30
|
+
fetch_commands.append(
|
|
31
|
+
f"wget --content-disposition -P /output_dir/ {param.value}"
|
|
32
|
+
)
|
|
33
|
+
elif param.type == "file_system":
|
|
34
|
+
filename = str(param.value).split("/")[-1]
|
|
35
|
+
fetch_commands.append(f"cp {param.value} /output_dir/{filename}")
|
|
36
|
+
elif param.type == "pvc_mount":
|
|
37
|
+
# TODO: support PVC paths as input.
|
|
38
|
+
# Because it is a PVC, we expect it to be mounted to the first
|
|
39
|
+
# step's container, so no move should be necessary.
|
|
40
|
+
err_msg = "PVC mounts are not yet supported"
|
|
41
|
+
raise NotImplementedError(err_msg)
|
|
42
|
+
else:
|
|
43
|
+
raise OgdcWorkflowExecutionError(
|
|
44
|
+
f"Unsupported input type: {param.type} for parameter {param.value}"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# Join all commands with && for sequential execution
|
|
48
|
+
combined_command = " && ".join(fetch_commands)
|
|
49
|
+
if not combined_command:
|
|
50
|
+
combined_command = "echo 'No input files to fetch'"
|
|
51
|
+
|
|
52
|
+
template = Container(
|
|
53
|
+
name=f"{recipe_config.id}-fetch-template-",
|
|
54
|
+
command=["sh", "-c"],
|
|
55
|
+
args=[
|
|
56
|
+
f"mkdir -p /output_dir/ && {combined_command}",
|
|
57
|
+
],
|
|
58
|
+
outputs=[Artifact(name="output-dir", path="/output_dir/")],
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
return template
|
|
File without changes
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from pydantic import AnyUrl, BaseModel, Field, computed_field, field_validator
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# Input parameter with type and value
|
|
10
|
+
class InputParam(BaseModel):
|
|
11
|
+
value: AnyUrl | str
|
|
12
|
+
type: Literal["url", "pvc_mount", "file_system"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Create a model for the recipe input
|
|
16
|
+
class RecipeInput(BaseModel):
|
|
17
|
+
params: list[InputParam]
|
|
18
|
+
|
|
19
|
+
@field_validator("params")
|
|
20
|
+
def validate_params(cls, params: list[InputParam]) -> list[InputParam]:
|
|
21
|
+
"""Ensure there's at least one input parameter."""
|
|
22
|
+
if not params:
|
|
23
|
+
error_msg = "At least one input parameter is required"
|
|
24
|
+
raise ValueError(error_msg)
|
|
25
|
+
return params
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class RecipeOutput(BaseModel):
|
|
29
|
+
dataone_id: str = "TODO"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RecipeMeta(BaseModel):
|
|
33
|
+
"""Model for a recipe's metadata (`meta.yaml`)."""
|
|
34
|
+
|
|
35
|
+
# Allow alphanumeric characters, `.`, ` ` (space), and `,`.
|
|
36
|
+
# The name is used to create an ID for the recipe that must be k8s-compliant
|
|
37
|
+
# (lower-case, alphanumeric characters, `.`, and `,`).
|
|
38
|
+
name: str = Field(..., pattern=r"^[a-zA-Z0-9 .-]+$")
|
|
39
|
+
|
|
40
|
+
# Type of recipe, e.g., "shell", "visualization", etc.
|
|
41
|
+
type: Literal["shell", "visualization"]
|
|
42
|
+
|
|
43
|
+
input: RecipeInput
|
|
44
|
+
output: RecipeOutput = RecipeOutput()
|
|
45
|
+
|
|
46
|
+
# Optional Docker image (supports both local and hosted images)
|
|
47
|
+
# Examples: "my-local-image", "ghcr.io/owner/image:latest"
|
|
48
|
+
image: str | None = Field(
|
|
49
|
+
default=None, description="Docker image with optional tag"
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class RecipeConfig(RecipeMeta):
|
|
54
|
+
"""Model for a recipe's configuration.
|
|
55
|
+
|
|
56
|
+
This includes the data in `meta.yaml`, plus some internal metadata/config
|
|
57
|
+
that is generated dynamically at runtime (e.g., `recipe_directory`).
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
# ffspec-compatible recipe directory string.
|
|
61
|
+
# This is where the rest of the config was set from.
|
|
62
|
+
recipe_directory: str
|
|
63
|
+
|
|
64
|
+
@computed_field # type: ignore[misc]
|
|
65
|
+
@cached_property
|
|
66
|
+
def id(self) -> str:
|
|
67
|
+
k8s_name = self.name.lower().replace(" ", "-")
|
|
68
|
+
|
|
69
|
+
return k8s_name
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class RecipeImage(BaseModel):
|
|
73
|
+
"""
|
|
74
|
+
Image configuration for the recipe.
|
|
75
|
+
|
|
76
|
+
Supports both local and hosted Docker images.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
image: str = Field(..., description="Docker image name")
|
|
80
|
+
tag: str = Field(default="latest", description="Docker image tag")
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def full_image_path(self) -> str:
|
|
84
|
+
"""Return the full image path including tag."""
|
|
85
|
+
return f"{self.image}:{self.tag}"
|
ogdc_runner/publish.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Code for interacting with and publishing outputs of OGDC recipes"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from hera.workflows import (
|
|
6
|
+
Artifact,
|
|
7
|
+
Container,
|
|
8
|
+
Parameter,
|
|
9
|
+
Steps,
|
|
10
|
+
Workflow,
|
|
11
|
+
models,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from ogdc_runner.argo import ARGO_WORKFLOW_SERVICE, OGDC_WORKFLOW_PVC, submit_workflow
|
|
15
|
+
from ogdc_runner.exceptions import OgdcWorkflowExecutionError
|
|
16
|
+
from ogdc_runner.models.recipe_config import RecipeConfig
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def make_publish_template(
|
|
20
|
+
recipe_id: str,
|
|
21
|
+
) -> Container:
|
|
22
|
+
"""Creates a container template that will move final output data into the
|
|
23
|
+
OGDC data storage volume under a subpath named for the recipe_id."""
|
|
24
|
+
template = Container(
|
|
25
|
+
name="publish-data-",
|
|
26
|
+
command=["sh", "-c"],
|
|
27
|
+
args=[
|
|
28
|
+
"rsync --progress /input_dir/* /output_dir/",
|
|
29
|
+
],
|
|
30
|
+
inputs=[Artifact(name="input-dir", path="/input_dir/")],
|
|
31
|
+
volume_mounts=[
|
|
32
|
+
models.VolumeMount(
|
|
33
|
+
name=OGDC_WORKFLOW_PVC.name,
|
|
34
|
+
mount_path="/output_dir/",
|
|
35
|
+
sub_path=recipe_id,
|
|
36
|
+
)
|
|
37
|
+
],
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
return template
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def remove_existing_published_data(
|
|
44
|
+
*,
|
|
45
|
+
recipe_config: RecipeConfig,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Executes an argo workflow that removes published data for a recipe if it
|
|
48
|
+
exists."""
|
|
49
|
+
with Workflow(
|
|
50
|
+
generate_name=f"{recipe_config.id}-remove-existing-data-",
|
|
51
|
+
entrypoint="steps",
|
|
52
|
+
workflows_service=ARGO_WORKFLOW_SERVICE,
|
|
53
|
+
) as w:
|
|
54
|
+
overwrite_template = Container(
|
|
55
|
+
name="overwrite-already-published-",
|
|
56
|
+
command=["sh", "-c"],
|
|
57
|
+
args=[
|
|
58
|
+
f"rm -rf /mnt/{recipe_config.id}",
|
|
59
|
+
],
|
|
60
|
+
volume_mounts=[
|
|
61
|
+
models.VolumeMount(
|
|
62
|
+
name=OGDC_WORKFLOW_PVC.name,
|
|
63
|
+
mount_path="/mnt/",
|
|
64
|
+
),
|
|
65
|
+
],
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
with Steps(name="steps"):
|
|
69
|
+
overwrite_template()
|
|
70
|
+
|
|
71
|
+
workflow_name = submit_workflow(workflow=w, wait=True)
|
|
72
|
+
|
|
73
|
+
# Cleanup this workflow, it is no longer needed
|
|
74
|
+
ARGO_WORKFLOW_SERVICE.delete_workflow(workflow_name)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def check_for_existing_published_data(
|
|
78
|
+
*,
|
|
79
|
+
recipe_config: RecipeConfig,
|
|
80
|
+
) -> bool:
|
|
81
|
+
"""Execute argo workflow that checks if the given recipe has published data.
|
|
82
|
+
|
|
83
|
+
Returns `True` if data have already been published for the given recipe,
|
|
84
|
+
otherwise `False`.
|
|
85
|
+
"""
|
|
86
|
+
with Workflow(
|
|
87
|
+
generate_name=f"{recipe_config.id}-check-published-",
|
|
88
|
+
entrypoint="steps",
|
|
89
|
+
workflows_service=ARGO_WORKFLOW_SERVICE,
|
|
90
|
+
) as w:
|
|
91
|
+
check_dir_template = Container(
|
|
92
|
+
name="check-already-published",
|
|
93
|
+
command=["sh", "-c"],
|
|
94
|
+
# Check for the existence of the recipe-specific subpath. If it
|
|
95
|
+
# exists, write out a file with "yes". Otherwise write out a file
|
|
96
|
+
# with "no". This file becomes an argo parameter that we can check
|
|
97
|
+
# later.
|
|
98
|
+
args=[
|
|
99
|
+
f'test -d /mnt/{recipe_config.id} && echo "yes" > /tmp/published.txt || echo "no" > /tmp/published.txt',
|
|
100
|
+
],
|
|
101
|
+
outputs=[
|
|
102
|
+
Parameter(
|
|
103
|
+
name="data-published",
|
|
104
|
+
value_from=models.ValueFrom(path="/tmp/published.txt"),
|
|
105
|
+
),
|
|
106
|
+
],
|
|
107
|
+
volume_mounts=[
|
|
108
|
+
models.VolumeMount(
|
|
109
|
+
name=OGDC_WORKFLOW_PVC.name,
|
|
110
|
+
mount_path="/mnt/",
|
|
111
|
+
),
|
|
112
|
+
],
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
with Steps(name="steps"):
|
|
116
|
+
check_dir_template()
|
|
117
|
+
|
|
118
|
+
# wait for the workflow to complete.
|
|
119
|
+
workflow_name = submit_workflow(workflow=w, wait=True)
|
|
120
|
+
|
|
121
|
+
# If overwrite is not True, we need to check the result of the
|
|
122
|
+
# `check-already-published` step to see if the data have been published or
|
|
123
|
+
# not.
|
|
124
|
+
# Check the result. Get an updated instance of the workflow, with the latest
|
|
125
|
+
# states for all notdes. Then, iterate through the nodes and find the
|
|
126
|
+
# template we define above ("check-already-published") and extract its
|
|
127
|
+
# output parameter.
|
|
128
|
+
completed_workflow = ARGO_WORKFLOW_SERVICE.get_workflow(name=workflow_name)
|
|
129
|
+
result = None
|
|
130
|
+
for node in completed_workflow.status.nodes.values(): # type: ignore[union-attr]
|
|
131
|
+
if node.template_name == "check-already-published":
|
|
132
|
+
result = node.outputs.parameters[0].value # type: ignore[union-attr, index]
|
|
133
|
+
if not result:
|
|
134
|
+
err_msg = "Failed to check if data have been published"
|
|
135
|
+
raise OgdcWorkflowExecutionError(err_msg)
|
|
136
|
+
|
|
137
|
+
assert result in ("yes", "no")
|
|
138
|
+
|
|
139
|
+
# Cleanup this workflow, it is no longer needed
|
|
140
|
+
ARGO_WORKFLOW_SERVICE.delete_workflow(workflow_name)
|
|
141
|
+
|
|
142
|
+
return result == "yes"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def data_already_published(
|
|
146
|
+
*,
|
|
147
|
+
recipe_config: RecipeConfig,
|
|
148
|
+
overwrite: bool,
|
|
149
|
+
) -> bool:
|
|
150
|
+
"""Check for the existence of published data for the given
|
|
151
|
+
recipe and optionally remove it.
|
|
152
|
+
|
|
153
|
+
If `overwrite=True`, this function will remove any existing published data
|
|
154
|
+
for the provided recipe.
|
|
155
|
+
|
|
156
|
+
Returns `True` if data have already been published for the given recipe,
|
|
157
|
+
otherwise `False`.
|
|
158
|
+
"""
|
|
159
|
+
if overwrite:
|
|
160
|
+
# If `overwrite` is True, remove the existing data and return `False`.
|
|
161
|
+
remove_existing_published_data(
|
|
162
|
+
recipe_config=recipe_config,
|
|
163
|
+
)
|
|
164
|
+
return False
|
|
165
|
+
|
|
166
|
+
return check_for_existing_published_data(
|
|
167
|
+
recipe_config=recipe_config,
|
|
168
|
+
)
|
ogdc_runner/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import fsspec
|
|
4
|
+
import yaml
|
|
5
|
+
|
|
6
|
+
from ogdc_runner.constants import RECIPE_CONFIG_FILENAME
|
|
7
|
+
from ogdc_runner.exceptions import OgdcInvalidRecipeDir
|
|
8
|
+
from ogdc_runner.models.recipe_config import RecipeConfig
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_recipe_config(recipe_directory: str) -> RecipeConfig:
|
|
12
|
+
"""Extract config from a recipe configuration file (meta.yml)."""
|
|
13
|
+
recipe_path = f"{recipe_directory}/{RECIPE_CONFIG_FILENAME}"
|
|
14
|
+
try:
|
|
15
|
+
with fsspec.open(recipe_path, "rt") as config_file:
|
|
16
|
+
config_dict = yaml.safe_load(config_file)
|
|
17
|
+
except (FileNotFoundError, OSError) as err:
|
|
18
|
+
raise OgdcInvalidRecipeDir(
|
|
19
|
+
f"Recipe directory not found: {recipe_directory}"
|
|
20
|
+
) from err
|
|
21
|
+
|
|
22
|
+
config = RecipeConfig(**config_dict, recipe_directory=recipe_directory)
|
|
23
|
+
|
|
24
|
+
return config
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import fsspec
|
|
4
|
+
from hera.workflows import (
|
|
5
|
+
Artifact,
|
|
6
|
+
Container,
|
|
7
|
+
Steps,
|
|
8
|
+
Workflow,
|
|
9
|
+
)
|
|
10
|
+
from loguru import logger
|
|
11
|
+
|
|
12
|
+
from ogdc_runner.argo import (
|
|
13
|
+
ARGO_WORKFLOW_SERVICE,
|
|
14
|
+
submit_workflow,
|
|
15
|
+
)
|
|
16
|
+
from ogdc_runner.constants import SHELL_RECIPE_FILENAME
|
|
17
|
+
from ogdc_runner.inputs import make_fetch_input_template
|
|
18
|
+
from ogdc_runner.models.recipe_config import RecipeConfig
|
|
19
|
+
from ogdc_runner.publish import make_publish_template
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def make_cmd_template(
|
|
23
|
+
name: str,
|
|
24
|
+
command: str,
|
|
25
|
+
) -> Container:
|
|
26
|
+
"""Creates a command template with an optional custom image."""
|
|
27
|
+
template = Container(
|
|
28
|
+
name=name,
|
|
29
|
+
command=["sh", "-c"],
|
|
30
|
+
args=[
|
|
31
|
+
f"mkdir -p /output_dir/ && {command}",
|
|
32
|
+
],
|
|
33
|
+
inputs=[Artifact(name="input-dir", path="/input_dir/")],
|
|
34
|
+
outputs=[Artifact(name="output-dir", path="/output_dir/")],
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
return template
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def make_and_submit_shell_workflow(
|
|
41
|
+
recipe_config: RecipeConfig,
|
|
42
|
+
wait: bool,
|
|
43
|
+
) -> str:
|
|
44
|
+
"""Create and submit an argo workflow based on a shell recipe.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
recipe_config: The recipe configuration
|
|
48
|
+
wait: Whether to wait for the workflow to complete
|
|
49
|
+
|
|
50
|
+
Returns the name of the workflow as a str.
|
|
51
|
+
"""
|
|
52
|
+
# Parse commands from the shell recipe file
|
|
53
|
+
commands = parse_commands_from_recipe_file(
|
|
54
|
+
recipe_config.recipe_directory,
|
|
55
|
+
SHELL_RECIPE_FILENAME,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
with Workflow(
|
|
59
|
+
generate_name=f"{recipe_config.id}-",
|
|
60
|
+
entrypoint="steps",
|
|
61
|
+
workflows_service=ARGO_WORKFLOW_SERVICE,
|
|
62
|
+
) as w:
|
|
63
|
+
# Create command templates
|
|
64
|
+
cmd_templates = []
|
|
65
|
+
for idx, command in enumerate(commands):
|
|
66
|
+
cmd_template = make_cmd_template(
|
|
67
|
+
name=f"run-cmd-{idx}",
|
|
68
|
+
command=command,
|
|
69
|
+
)
|
|
70
|
+
cmd_templates.append(cmd_template)
|
|
71
|
+
|
|
72
|
+
# Use the multi-input fetch template
|
|
73
|
+
fetch_template = make_fetch_input_template(
|
|
74
|
+
recipe_config=recipe_config,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
# Create publication template
|
|
78
|
+
publish_template = make_publish_template(
|
|
79
|
+
recipe_id=recipe_config.id,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# Create the workflow steps
|
|
83
|
+
with Steps(name="steps"):
|
|
84
|
+
step = fetch_template()
|
|
85
|
+
for idx, cmd_template in enumerate(cmd_templates):
|
|
86
|
+
step = cmd_template(
|
|
87
|
+
name=f"step-{idx}",
|
|
88
|
+
arguments=step.get_artifact("output-dir").with_name("input-dir"), # type: ignore[union-attr]
|
|
89
|
+
)
|
|
90
|
+
# Publish final data
|
|
91
|
+
publish_template(
|
|
92
|
+
name="publish-data",
|
|
93
|
+
arguments=step.get_artifact("output-dir").with_name("input-dir"), # type: ignore[union-attr]
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# Submit the workflow
|
|
97
|
+
workflow_name = submit_workflow(w, wait=wait)
|
|
98
|
+
|
|
99
|
+
return workflow_name
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def parse_commands_from_recipe_file(recipe_dir: str, filename: str) -> list[str]:
|
|
103
|
+
"""Read commands from a recipe file.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
recipe_dir: The directory containing the recipe file
|
|
107
|
+
filename: The name of the recipe file to parse
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
A list of commands from the recipe file, with comments removed
|
|
111
|
+
"""
|
|
112
|
+
recipe_path = f"{recipe_dir}/{filename}"
|
|
113
|
+
logger.info(f"Reading recipe from {recipe_path}")
|
|
114
|
+
|
|
115
|
+
with fsspec.open(recipe_path, "rt") as f:
|
|
116
|
+
lines = f.read().split("\n")
|
|
117
|
+
commands = [line for line in lines if line and not line.startswith("#")]
|
|
118
|
+
|
|
119
|
+
return commands
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from hera.workflows import (
|
|
7
|
+
DAG,
|
|
8
|
+
Artifact,
|
|
9
|
+
Container,
|
|
10
|
+
HTTPArtifact,
|
|
11
|
+
Parameter,
|
|
12
|
+
Task,
|
|
13
|
+
Workflow,
|
|
14
|
+
script,
|
|
15
|
+
)
|
|
16
|
+
from hera.workflows.models import (
|
|
17
|
+
VolumeMount,
|
|
18
|
+
)
|
|
19
|
+
from loguru import logger
|
|
20
|
+
from pydantic import AnyUrl
|
|
21
|
+
|
|
22
|
+
from ogdc_runner.argo import (
|
|
23
|
+
ARGO_MANAGER,
|
|
24
|
+
ARGO_WORKFLOW_SERVICE,
|
|
25
|
+
OGDC_WORKFLOW_PVC,
|
|
26
|
+
submit_workflow,
|
|
27
|
+
)
|
|
28
|
+
from ogdc_runner.constants import VIZ_RECIPE_BATCH_SIZE
|
|
29
|
+
from ogdc_runner.models.recipe_config import RecipeConfig
|
|
30
|
+
from ogdc_runner.recipe import get_recipe_config
|
|
31
|
+
|
|
32
|
+
# ruff: noqa: PLC0415
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@script(
|
|
36
|
+
name="batching",
|
|
37
|
+
inputs=[
|
|
38
|
+
Parameter(name="input_url"),
|
|
39
|
+
Parameter(name="recipe_id"),
|
|
40
|
+
HTTPArtifact(
|
|
41
|
+
name="batch-input",
|
|
42
|
+
path="/mnt/workflow/{{inputs.parameters.recipe_id}}/input/input.gpkg",
|
|
43
|
+
url="{{inputs.parameters.input_url}}",
|
|
44
|
+
),
|
|
45
|
+
],
|
|
46
|
+
outputs=[
|
|
47
|
+
Artifact(
|
|
48
|
+
name="batch-output",
|
|
49
|
+
path="/mnt/workflow/{{inputs.parameters.recipe_id}}/batch",
|
|
50
|
+
),
|
|
51
|
+
],
|
|
52
|
+
image="ghcr.io/rushirajnenuji/viz-staging:latest",
|
|
53
|
+
command=["python"],
|
|
54
|
+
volume_mounts=[
|
|
55
|
+
VolumeMount(name=OGDC_WORKFLOW_PVC.name, mount_path="/mnt/workflow")
|
|
56
|
+
],
|
|
57
|
+
)
|
|
58
|
+
def batch_process(num_features) -> None: # type: ignore[no-untyped-def]
|
|
59
|
+
"""Processes data in batches."""
|
|
60
|
+
import sys
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
|
|
63
|
+
import geopandas as gpd # type: ignore[import-not-found]
|
|
64
|
+
|
|
65
|
+
# Redirect print statements to stderr instead of stdout
|
|
66
|
+
# This way they won't interfere with the JSON output
|
|
67
|
+
def print_log(message: str) -> None:
|
|
68
|
+
print(message, file=sys.stderr)
|
|
69
|
+
|
|
70
|
+
gdf = gpd.read_file("{{inputs.artifacts.batch-input.path}}")
|
|
71
|
+
results = []
|
|
72
|
+
for idx, start in enumerate(range(0, len(gdf), num_features)):
|
|
73
|
+
output_fp = Path(
|
|
74
|
+
"{{outputs.artifacts.batch-output.path}}/" + f"chunk-{idx}.gpkg"
|
|
75
|
+
)
|
|
76
|
+
print_log(f"Writing chunk {idx} to {output_fp}")
|
|
77
|
+
output_fp.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
gdf[start : start + num_features].to_file(
|
|
79
|
+
filename=output_fp,
|
|
80
|
+
driver="GPKG",
|
|
81
|
+
)
|
|
82
|
+
results.append(str(output_fp))
|
|
83
|
+
|
|
84
|
+
# Output only the JSON to stdout
|
|
85
|
+
print(json.dumps(results))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@script(
|
|
89
|
+
name="tiling",
|
|
90
|
+
inputs=[
|
|
91
|
+
Parameter(name="chunk-filepath"),
|
|
92
|
+
Parameter(name="recipe_id"),
|
|
93
|
+
],
|
|
94
|
+
image="ghcr.io/rushirajnenuji/viz-staging:latest",
|
|
95
|
+
command=["python"],
|
|
96
|
+
volume_mounts=[
|
|
97
|
+
VolumeMount(name=OGDC_WORKFLOW_PVC.name, mount_path="/mnt/workflow")
|
|
98
|
+
],
|
|
99
|
+
)
|
|
100
|
+
def tiling_process() -> None:
|
|
101
|
+
"""Creates tiles from a geospatial data chunk."""
|
|
102
|
+
import json
|
|
103
|
+
import sys
|
|
104
|
+
from pathlib import Path
|
|
105
|
+
|
|
106
|
+
from pdgstaging import ( # type: ignore[import-not-found]
|
|
107
|
+
TileStager,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Log to stderr
|
|
111
|
+
def print_log(message: str) -> None:
|
|
112
|
+
print(message, file=sys.stderr)
|
|
113
|
+
|
|
114
|
+
# Read the viz-config.json from the PVC
|
|
115
|
+
# This configuration controls how VizWorkflow processes the visualization data.
|
|
116
|
+
# For available configuration options, see:
|
|
117
|
+
# https://github.com/PermafrostDiscoveryGateway/viz-workflow/blob/feature-wf-k8s/pdgworkflow/ConfigManager.py
|
|
118
|
+
workflow_config = json.loads(
|
|
119
|
+
Path("/mnt/workflow/{{inputs.parameters.recipe_id}}/config.json").read_text()
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
tiler = TileStager(workflow_config, check_footprints=False)
|
|
123
|
+
print_log("Staging chunk file")
|
|
124
|
+
print_log("{{inputs.parameters.chunk-filepath}}")
|
|
125
|
+
tiler.stage("{{inputs.parameters.chunk-filepath}}")
|
|
126
|
+
print_log("Staging done")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def read_config_file_content(recipe_config: RecipeConfig) -> str:
|
|
130
|
+
"""Read the config.json file content from the recipe directory.
|
|
131
|
+
|
|
132
|
+
This configuration file is used by the pdgworkflow for visualization workflows.
|
|
133
|
+
When an empty config ({}) is returned, WorkflowManager will use its default behavior.
|
|
134
|
+
|
|
135
|
+
For documentation on available configuration options, see:
|
|
136
|
+
- ConfigManager documentation: https://github.com/PermafrostDiscoveryGateway/viz-workflow/blob/feature-wf-k8s/pdgworkflow/ConfigManager.py
|
|
137
|
+
- Example config: https://github.com/QGreenland-Net/ogdc-recipes/blob/main/recipes/viz-workflow/config.json
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
recipe_config: The recipe configuration containing the directory path
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
The content of the config.json file as a string, or empty JSON if file doesn't exist.
|
|
144
|
+
An empty config ({}) will cause ConfigManager to use default behavior.
|
|
145
|
+
"""
|
|
146
|
+
config_file_path = Path(recipe_config.recipe_directory) / "config.json"
|
|
147
|
+
if config_file_path.exists():
|
|
148
|
+
return config_file_path.read_text()
|
|
149
|
+
# Fallback to empty config if file doesn't exist - ConfigManager will use defaults
|
|
150
|
+
return "{}"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def make_and_submit_viz_workflow(
|
|
154
|
+
recipe_config: RecipeConfig,
|
|
155
|
+
wait: bool,
|
|
156
|
+
input_url: AnyUrl | str | None = None,
|
|
157
|
+
) -> str:
|
|
158
|
+
"""Create and submit an Argo workflow for parallel processing of geospatial data.
|
|
159
|
+
|
|
160
|
+
This workflow follows the pattern from the 'ogdc-recipe-ice-basins-pdg' Argo workflow,
|
|
161
|
+
using Hera's Python API instead of YAML.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
recipe_config: The recipe configuration
|
|
165
|
+
wait: Whether to wait for the workflow to complete
|
|
166
|
+
input_url: URL to the input data file
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
The name of the submitted workflow
|
|
170
|
+
"""
|
|
171
|
+
with Workflow(
|
|
172
|
+
generate_name=f"{recipe_config.id}-",
|
|
173
|
+
entrypoint="main",
|
|
174
|
+
namespace=ARGO_MANAGER.config.namespace,
|
|
175
|
+
service_account_name="argo-workflow",
|
|
176
|
+
workflows_service=ARGO_WORKFLOW_SERVICE,
|
|
177
|
+
volumes=[OGDC_WORKFLOW_PVC],
|
|
178
|
+
annotations={
|
|
179
|
+
"workflows.argoproj.io/description": "Visualization workflow for OGDC",
|
|
180
|
+
},
|
|
181
|
+
labels={"workflows.argoproj.io/archive-strategy": "false"},
|
|
182
|
+
) as w:
|
|
183
|
+
# Create templates outside the DAG context
|
|
184
|
+
# Read the config.json file content from the recipe directory
|
|
185
|
+
config_content = read_config_file_content(recipe_config)
|
|
186
|
+
|
|
187
|
+
stage_config_file_template = Container(
|
|
188
|
+
name="stage-viz-config",
|
|
189
|
+
image="alpine:latest",
|
|
190
|
+
command=["sh", "-c"],
|
|
191
|
+
args=[
|
|
192
|
+
f"""mkdir -p /mnt/workflow/{recipe_config.id}/input && \\
|
|
193
|
+
mkdir -p /mnt/workflow/{recipe_config.id}/batch && \\
|
|
194
|
+
mkdir -p /mnt/workflow/{recipe_config.id}/output/staged && \\
|
|
195
|
+
mkdir -p /mnt/workflow/{recipe_config.id}/output/geotiff && \\
|
|
196
|
+
mkdir -p /mnt/workflow/{recipe_config.id}/output/3dtiles && \\
|
|
197
|
+
cat > /mnt/workflow/{recipe_config.id}/config.json << 'EOF'
|
|
198
|
+
{config_content}
|
|
199
|
+
EOF"""
|
|
200
|
+
],
|
|
201
|
+
volume_mounts=[
|
|
202
|
+
VolumeMount(
|
|
203
|
+
name=OGDC_WORKFLOW_PVC.name,
|
|
204
|
+
mount_path="/mnt/workflow/",
|
|
205
|
+
)
|
|
206
|
+
],
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
# Set up the DAG
|
|
210
|
+
with DAG(name="main"):
|
|
211
|
+
# Step1: Stage the viz-config.json file
|
|
212
|
+
# This is the configuration file for the visualization workflow
|
|
213
|
+
# It also creates necessary directories required for the workflow
|
|
214
|
+
# to run successfully
|
|
215
|
+
stage_config_task = Task(
|
|
216
|
+
name="stage-viz-config",
|
|
217
|
+
template=stage_config_file_template,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
# Step 2: Batching step
|
|
221
|
+
batch_task = batch_process(
|
|
222
|
+
arguments={
|
|
223
|
+
"input_url": input_url,
|
|
224
|
+
"recipe_id": recipe_config.id,
|
|
225
|
+
"num_features": VIZ_RECIPE_BATCH_SIZE,
|
|
226
|
+
},
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# Step 3: Tiling step
|
|
230
|
+
# This step processes each chunk of vector data in parallel
|
|
231
|
+
# using the tiling_process script defined above
|
|
232
|
+
stage_task = tiling_process(
|
|
233
|
+
arguments={
|
|
234
|
+
"chunk-filepath": "{{item}}",
|
|
235
|
+
"recipe_id": recipe_config.id,
|
|
236
|
+
},
|
|
237
|
+
with_param=batch_task.get_result_as("result"),
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
# Define the workflow structure
|
|
241
|
+
# tiling depends on both download and batch tasks
|
|
242
|
+
[ # type: ignore[operator]
|
|
243
|
+
stage_config_task,
|
|
244
|
+
batch_task,
|
|
245
|
+
] >> stage_task
|
|
246
|
+
|
|
247
|
+
# Submit the workflow
|
|
248
|
+
workflow_name = submit_workflow(w, wait=wait)
|
|
249
|
+
return workflow_name
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def submit_viz_workflow_recipe(
|
|
253
|
+
*,
|
|
254
|
+
recipe_dir: str,
|
|
255
|
+
wait: bool,
|
|
256
|
+
) -> str:
|
|
257
|
+
"""Submit an OGDC recipe for parallel processing via Argo workflows.
|
|
258
|
+
|
|
259
|
+
Args:
|
|
260
|
+
recipe_dir: Path to the recipe directory
|
|
261
|
+
wait: Whether to wait for the workflow to complete
|
|
262
|
+
|
|
263
|
+
Returns:
|
|
264
|
+
The name of the submitted workflow
|
|
265
|
+
"""
|
|
266
|
+
# Get the recipe configuration
|
|
267
|
+
recipe_config = get_recipe_config(recipe_dir)
|
|
268
|
+
|
|
269
|
+
input_param = recipe_config.input.params[0]
|
|
270
|
+
if input_param.type == "url":
|
|
271
|
+
input_url = input_param.value
|
|
272
|
+
else:
|
|
273
|
+
raise NotImplementedError(
|
|
274
|
+
f"Input type '{input_param.type}' is not supported for visualization workflows. "
|
|
275
|
+
f"Only 'url' input type is currently supported."
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
# Submit the workflow
|
|
279
|
+
workflow_name = make_and_submit_viz_workflow(
|
|
280
|
+
recipe_config=recipe_config,
|
|
281
|
+
wait=wait,
|
|
282
|
+
input_url=input_url,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
logger.info(f"Completed workflow: {workflow_name}")
|
|
286
|
+
return workflow_name
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ogdc-runner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Defines OGDC recipe API(s) and submits recipes to OGDC for execution
|
|
5
|
+
Project-URL: Homepage, https://github.com/qgreenland-net/ogdc-runner
|
|
6
|
+
Project-URL: Bug Tracker, https://github.com/qgreenland-net/ogdc-runner/issues
|
|
7
|
+
Project-URL: Discussions, https://github.com/qgreenland-net/ogdc-runner/discussions
|
|
8
|
+
Project-URL: Changelog, https://github.com/qgreenland-net/ogdc-runner/releases
|
|
9
|
+
Author-email: Matt Fisher <mfisher87@gmail.com>, Rushiraj Nenuji <nenuji@nceas.ucsb.edu>, Trey Stafford <trey.stafford@colorado.edu>, Robyn Marowitz <robyn.marowitz@colorado.edu>
|
|
10
|
+
License: Copyright 2024 National Snow and Ice Data Center & Regents of the University of California
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
13
|
+
this software and associated documentation files (the "Software"), to deal in
|
|
14
|
+
the Software without restriction, including without limitation the rights to
|
|
15
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
16
|
+
of the Software, and to permit persons to whom the Software is furnished to do
|
|
17
|
+
so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Classifier: Development Status :: 1 - Planning
|
|
31
|
+
Classifier: Intended Audience :: Developers
|
|
32
|
+
Classifier: Intended Audience :: Science/Research
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Operating System :: OS Independent
|
|
35
|
+
Classifier: Programming Language :: Python
|
|
36
|
+
Classifier: Programming Language :: Python :: 3
|
|
37
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
40
|
+
Classifier: Topic :: Scientific/Engineering
|
|
41
|
+
Classifier: Typing :: Typed
|
|
42
|
+
Requires-Python: >=3.11
|
|
43
|
+
Requires-Dist: click>=8
|
|
44
|
+
Requires-Dist: fsspec
|
|
45
|
+
Requires-Dist: hera>=5.18.0
|
|
46
|
+
Requires-Dist: loguru
|
|
47
|
+
Requires-Dist: pydantic>=2.7.1
|
|
48
|
+
Requires-Dist: pyyaml
|
|
49
|
+
Provides-Extra: dev
|
|
50
|
+
Requires-Dist: mypy~=1.10.0; extra == 'dev'
|
|
51
|
+
Requires-Dist: nox; extra == 'dev'
|
|
52
|
+
Requires-Dist: pytest-cov>=3; extra == 'dev'
|
|
53
|
+
Requires-Dist: pytest>=6; extra == 'dev'
|
|
54
|
+
Requires-Dist: types-pyyaml; extra == 'dev'
|
|
55
|
+
Provides-Extra: docs
|
|
56
|
+
Requires-Dist: autodoc-pydantic; extra == 'docs'
|
|
57
|
+
Requires-Dist: furo>=2023.08.17; extra == 'docs'
|
|
58
|
+
Requires-Dist: myst-parser>=0.13; extra == 'docs'
|
|
59
|
+
Requires-Dist: sphinx-autodoc-typehints; extra == 'docs'
|
|
60
|
+
Requires-Dist: sphinx-copybutton; extra == 'docs'
|
|
61
|
+
Requires-Dist: sphinx>=7.0; extra == 'docs'
|
|
62
|
+
Requires-Dist: sphinxcontrib-mermaid; extra == 'docs'
|
|
63
|
+
Provides-Extra: test
|
|
64
|
+
Requires-Dist: mypy~=1.10.0; extra == 'test'
|
|
65
|
+
Requires-Dist: pytest-cov>=3; extra == 'test'
|
|
66
|
+
Requires-Dist: pytest-order; extra == 'test'
|
|
67
|
+
Requires-Dist: pytest>=6; extra == 'test'
|
|
68
|
+
Requires-Dist: types-pyyaml; extra == 'test'
|
|
69
|
+
Description-Content-Type: text/markdown
|
|
70
|
+
|
|
71
|
+
# ogdc-runner
|
|
72
|
+
|
|
73
|
+
<!-- SPHINX-START -->
|
|
74
|
+
|
|
75
|
+
[![Actions Status][actions-badge]][actions-link]
|
|
76
|
+
[![Documentation Status][rtd-badge]][rtd-link]
|
|
77
|
+
|
|
78
|
+
[![PyPI version][pypi-version]][pypi-link]
|
|
79
|
+
[![Conda-Forge][conda-badge]][conda-link]
|
|
80
|
+
[![PyPI platforms][pypi-platforms]][pypi-link]
|
|
81
|
+
|
|
82
|
+
[![GitHub Discussion][github-discussions-badge]][github-discussions-link]
|
|
83
|
+
|
|
84
|
+
Defines Open Geospatial Data Cloud (OGDC) recipe API(s) and submits recipes to
|
|
85
|
+
OGDC for execution.
|
|
86
|
+
|
|
87
|
+
See the [ogdc-runner ReadTheDocs](https://ogdc-runner.readthedocs.io/) for
|
|
88
|
+
detailed documentation on how to use and contribute to the `ogdc-runner`.
|
|
89
|
+
|
|
90
|
+
<!-- prettier-ignore-start -->
|
|
91
|
+
[actions-badge]: https://github.com/qgreenland-net/ogdc-runner/workflows/CI/badge.svg
|
|
92
|
+
[actions-link]: https://github.com/qgreenland-net/ogdc-runner/actions
|
|
93
|
+
[conda-badge]: https://img.shields.io/conda/vn/conda-forge/ogdc-runner
|
|
94
|
+
[conda-link]: https://github.com/conda-forge/ogdc-runner-feedstock
|
|
95
|
+
[github-discussions-badge]: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
|
|
96
|
+
[github-discussions-link]: https://github.com/qgreenland-net/ogdc-runner/discussions
|
|
97
|
+
[pypi-link]: https://pypi.org/project/ogdc-runner/
|
|
98
|
+
[pypi-platforms]: https://img.shields.io/pypi/pyversions/ogdc-runner
|
|
99
|
+
[pypi-version]: https://img.shields.io/pypi/v/ogdc-runner
|
|
100
|
+
[rtd-badge]: https://readthedocs.org/projects/ogdc-runner/badge/?version=latest
|
|
101
|
+
[rtd-link]: https://ogdc-runner.readthedocs.io/en/latest/?badge=latest
|
|
102
|
+
<!-- prettier-ignore-end -->
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
ogdc_runner/__init__.py,sha256=Ysvgdl8lVJcfsBll9SWSRz14SbpHPZiCN9mjyfsEJ7o,228
|
|
2
|
+
ogdc_runner/__main__.py,sha256=LvuUy_UAVf9KiVRwBEXhOzF6DPh04f3tkEuMH_xTw08,1903
|
|
3
|
+
ogdc_runner/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
|
|
4
|
+
ogdc_runner/_version.pyi,sha256=j5kbzfm6lOn8BzASXWjGIA1yT0OlHTWqlbyZ8Si_o0E,118
|
|
5
|
+
ogdc_runner/api.py,sha256=BbdnI-VdGckYZbm9Km7KzWrh5v9UuFLBDk7bgWwX5Qg,1624
|
|
6
|
+
ogdc_runner/argo.py,sha256=9wlVOiobqnaQjye67iYM4vek3ukZw7be7ONIZuGDNNA,6425
|
|
7
|
+
ogdc_runner/constants.py,sha256=xgg2dVCOR8_rEZnNdsN0iyblq7IIPUCJepBbcNydW_A,300
|
|
8
|
+
ogdc_runner/exceptions.py,sha256=UhQZbDRYFpxdgYTTvIy0SKiZTK5cF5xjZjxz6NZ9SJA,575
|
|
9
|
+
ogdc_runner/inputs.py,sha256=mnw95WMpRmgMBM9Bfez_jxUw5VwpKvie7dsgGc8366E,2024
|
|
10
|
+
ogdc_runner/publish.py,sha256=1KIt_A6j1h3pzzBfCfMz647--sXxmwu1Om38o7VLWvw,5431
|
|
11
|
+
ogdc_runner/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
ogdc_runner/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
ogdc_runner/models/recipe_config.py,sha256=QArFYnNEpbmVrZmnkPIGKTFfEKSLKw4Y1PeM2b6vB8M,2523
|
|
14
|
+
ogdc_runner/recipe/__init__.py,sha256=IDmYX0tmg98hpFtdylCK33m9VGE4Hod8dhKY3UbroO8,817
|
|
15
|
+
ogdc_runner/recipe/shell.py,sha256=vIYic5kigvVWSUpZwW7LhccqOfSKEoG_s6YmPDkLAaE,3498
|
|
16
|
+
ogdc_runner/recipe/viz_workflow.py,sha256=HidWnkZagxaZl3UKZFbmYCBfMb7Jyy0k6N2mc6hUQ6c,9350
|
|
17
|
+
ogdc_runner-0.1.0.dist-info/METADATA,sha256=Hh_gVCwRfKfYv0I_cm5w_oN9wnzyQnPn0px3gHT9--k,5128
|
|
18
|
+
ogdc_runner-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
19
|
+
ogdc_runner-0.1.0.dist-info/entry_points.txt,sha256=DjE7xC7HqeloRvUQ22X9VGc8vNeC1Tz0D60aGTcNA44,57
|
|
20
|
+
ogdc_runner-0.1.0.dist-info/licenses/LICENSE,sha256=70C2iMzDnhvVp90dhdtYCJdBLaOhXO-M3qBU26cy6lo,1115
|
|
21
|
+
ogdc_runner-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright 2024 National Snow and Ice Data Center & Regents of the University of California
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
4
|
+
this software and associated documentation files (the "Software"), to deal in
|
|
5
|
+
the Software without restriction, including without limitation the rights to
|
|
6
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
7
|
+
of the Software, and to permit persons to whom the Software is furnished to do
|
|
8
|
+
so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|