datatorch 0.7.7__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datatorch-0.7.7 → datatorch-0.8.0}/PKG-INFO +1 -1
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/client.py +31 -13
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/action/create.py +20 -5
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/core/env.py +2 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/directory.py +6 -0
- datatorch-0.8.0/datatorch/runner/pipelines/runner/python.py +114 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/runner.py +20 -1
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/PKG-INFO +1 -1
- {datatorch-0.7.7 → datatorch-0.8.0}/setup.py +1 -1
- datatorch-0.7.7/datatorch/runner/pipelines/runner/python.py +0 -32
- {datatorch-0.7.7 → datatorch-0.8.0}/README.md +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/agent/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/api.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/annotation.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/base.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/dataset.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/file.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/label.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/project.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/settings.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/image/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/image/bounding_box.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/image/segmentations.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/image/typings.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/sources/source.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/storage_link.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/entity/user.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/pull.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/scripts/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/scripts/import_coco.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/scripts/import_pixmask.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/scripts/utils/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/scripts/utils/simplify.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/utils.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/api/where.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/action/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/action/pull.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/action/run.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/groups.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/import_cmds/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/import_cmds/coco.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/import_cmds/pixmask.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/login.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/logout.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/pull.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/upgrade.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/main/version.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/pipeline/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/pipeline/generate.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/pipeline/run.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/pipeline/upload.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/runner/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/runner/create.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/runner/dir.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/runner/start.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/spinner.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/upload/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/upload/folder.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/cli/utils.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/core/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/core/folder.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/core/settings.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/client.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/daemon.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/log_handler.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/logging.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/monitoring.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/action/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/action/action.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/action/cache.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/action/config.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/job/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/job/job.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/pipeline.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/resolver.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/command.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/docker.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/factory.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/node.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/runner/shell.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/step/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/step/step.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/pipelines/template.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/services/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/utils/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/utils/docker.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/runner/utils/process.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/__init__.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/converters.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/files.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/hash_table.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/objects.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/package.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/string_style.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch/utils/url.py +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/SOURCES.txt +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/dependency_links.txt +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/entry_points.txt +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/not-zip-safe +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/requires.txt +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/datatorch.egg-info/top_level.txt +0 -0
- {datatorch-0.7.7 → datatorch-0.8.0}/setup.cfg +0 -0
|
@@ -9,7 +9,7 @@ from gql.transport.requests import RequestsHTTPTransport
|
|
|
9
9
|
from graphql.language.ast import DocumentNode
|
|
10
10
|
|
|
11
11
|
from datatorch.utils import normalize_api_url
|
|
12
|
-
from datatorch.core import user_settings
|
|
12
|
+
from datatorch.core import user_settings, env
|
|
13
13
|
from typing import Any, TypeVar, Type
|
|
14
14
|
|
|
15
15
|
from email.parser import HeaderParser
|
|
@@ -60,16 +60,27 @@ class Client(object):
|
|
|
60
60
|
agent: bool = False,
|
|
61
61
|
):
|
|
62
62
|
self._use_sockets = sockets
|
|
63
|
-
self._is_agent = agent
|
|
64
63
|
|
|
65
|
-
|
|
66
|
-
#
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
64
|
+
# Where to talk to: the argument, else DATATORCH_API_URL (which a
|
|
65
|
+
# runner sets for the actions it runs, so they reach the server
|
|
66
|
+
# the runner is connected to), else the stored login, else prod.
|
|
67
|
+
# Before October 2026 the stored login overrode the argument.
|
|
68
|
+
url = api_url or user_settings.api_url or "https://datatorch.io/api"
|
|
69
|
+
self._api_url = normalize_api_url(url)
|
|
70
|
+
|
|
71
|
+
# Who: an explicit key, else DATATORCH_API_KEY, else the runner's
|
|
72
|
+
# own token when running inside a runner (DATATORCH_RUNNER_TOKEN,
|
|
73
|
+
# acting as the runner), else the stored login.
|
|
74
|
+
token = api_key or os.getenv(env.API_KEY)
|
|
75
|
+
if token is None and not agent:
|
|
76
|
+
runner_token = os.getenv(env.RUNNER_TOKEN)
|
|
77
|
+
if runner_token:
|
|
78
|
+
token = runner_token
|
|
79
|
+
agent = True
|
|
80
|
+
if token is None:
|
|
81
|
+
token = user_settings.api_key
|
|
82
|
+
self._is_agent = agent
|
|
83
|
+
self._api_token = token
|
|
73
84
|
|
|
74
85
|
self._graphql_url = f"{self.api_url}/graphql"
|
|
75
86
|
|
|
@@ -79,7 +90,7 @@ class Client(object):
|
|
|
79
90
|
self.client = GqlClient(
|
|
80
91
|
transport=self.transport, fetch_schema_from_transport=True
|
|
81
92
|
)
|
|
82
|
-
self.set_api_token(
|
|
93
|
+
self.set_api_token(token)
|
|
83
94
|
|
|
84
95
|
@property
|
|
85
96
|
def token_header(self):
|
|
@@ -176,7 +187,13 @@ class Client(object):
|
|
|
176
187
|
# Never write an error body to disk as a "downloaded file"
|
|
177
188
|
result.raise_for_status()
|
|
178
189
|
|
|
179
|
-
|
|
190
|
+
# The API answers with the name and may redirect to storage, whose
|
|
191
|
+
# response can lack the header: read the first response that has it.
|
|
192
|
+
content = ""
|
|
193
|
+
for response in [result, *result.history]:
|
|
194
|
+
content = response.headers.get("content-disposition", "") or ""
|
|
195
|
+
if "filename" in content:
|
|
196
|
+
break
|
|
180
197
|
# _, value = cgi.parse_header(content)
|
|
181
198
|
parser = HeaderParser()
|
|
182
199
|
headers = parser.parsestr(f"Content-Disposition: {content}")
|
|
@@ -191,7 +208,8 @@ class Client(object):
|
|
|
191
208
|
else:
|
|
192
209
|
filename = "" # Default to an empty string if None or unexpected type
|
|
193
210
|
|
|
194
|
-
|
|
211
|
+
# Never a bare directory: with no name anywhere, the id is the name.
|
|
212
|
+
name = name or filename or id
|
|
195
213
|
name = os.path.join(directory, name)
|
|
196
214
|
name = os.path.abspath(name)
|
|
197
215
|
|
|
@@ -9,11 +9,17 @@ MODEL_SCRIPT = """import datatorch
|
|
|
9
9
|
|
|
10
10
|
image_path = datatorch.get_input("imagePath")
|
|
11
11
|
|
|
12
|
-
# Load your model and run it on image_path here.
|
|
13
|
-
#
|
|
14
|
-
|
|
12
|
+
# Load your model and run it on image_path here. Return what it found as
|
|
13
|
+
# a list of shapes in pixel coordinates; each one needs a label name and
|
|
14
|
+
# a bbox [x, y, width, height], a polygon [[x, y], ...] or points
|
|
15
|
+
# [[x, y], ...]. DataTorch puts them on the file when the run completes.
|
|
16
|
+
# Anything your model needs must be installed in the Python this runner
|
|
17
|
+
# runs with.
|
|
18
|
+
annotations = [
|
|
19
|
+
{"label": "example", "bbox": [10, 10, 100, 80], "score": 0.9},
|
|
20
|
+
]
|
|
15
21
|
|
|
16
|
-
datatorch.set_output("
|
|
22
|
+
datatorch.set_output("annotations", annotations)
|
|
17
23
|
"""
|
|
18
24
|
|
|
19
25
|
|
|
@@ -94,11 +100,20 @@ def create(name, description, json, local_name, tag):
|
|
|
94
100
|
"description": "Path of the image on disk",
|
|
95
101
|
}
|
|
96
102
|
},
|
|
97
|
-
"outputs": {
|
|
103
|
+
"outputs": {
|
|
104
|
+
"annotations": {
|
|
105
|
+
"description": "Shapes found in the image; DataTorch puts them on the file"
|
|
106
|
+
}
|
|
107
|
+
},
|
|
98
108
|
"runs": {"using": "python", "main": "run.py"},
|
|
99
109
|
}
|
|
100
110
|
with open(os.path.join(target, "run.py"), "w") as script:
|
|
101
111
|
script.write(MODEL_SCRIPT)
|
|
112
|
+
with open(os.path.join(target, "requirements.txt"), "w") as reqs:
|
|
113
|
+
reqs.write(
|
|
114
|
+
"# One package per line, as for pip. The runner installs them\n"
|
|
115
|
+
"# into this action's own environment on first run.\n"
|
|
116
|
+
)
|
|
102
117
|
else:
|
|
103
118
|
config = {
|
|
104
119
|
"$schema": "http://datatorch.io/schema/action.v1.json",
|
|
@@ -3,5 +3,7 @@ RUNNER_ID = "DATATORCH_RUNNER_ID"
|
|
|
3
3
|
# The pre-October-2026 spelling, still honored.
|
|
4
4
|
AGENT_ID = "DATATORCH_AGENT_ID"
|
|
5
5
|
API_KEY = "DATATORCH_API_KEY"
|
|
6
|
+
# A runner hands its own token to the actions it runs (October 2026).
|
|
7
|
+
RUNNER_TOKEN = "DATATORCH_RUNNER_TOKEN"
|
|
6
8
|
API_URL = "DATATORCH_API_URL"
|
|
7
9
|
CONFIG_DIR = "DATATORCH_DIR"
|
|
@@ -80,6 +80,12 @@ class RunnerDirectory(object):
|
|
|
80
80
|
"""Directory where actions are stored."""
|
|
81
81
|
return os.path.join(self.dir, "actions")
|
|
82
82
|
|
|
83
|
+
@property
|
|
84
|
+
def envs_dir(self):
|
|
85
|
+
"""Per-action Python environments (`runs.using: python` actions that
|
|
86
|
+
ship a requirements.txt): `envs/<action name>/<version>`."""
|
|
87
|
+
return os.path.join(self.dir, "envs")
|
|
88
|
+
|
|
83
89
|
def open(self, file: str, mode: str):
|
|
84
90
|
return open(os.path.join(self.dir, file), mode)
|
|
85
91
|
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import json
|
|
4
|
+
import shlex
|
|
5
|
+
import hashlib
|
|
6
|
+
|
|
7
|
+
from .runner import Runner
|
|
8
|
+
from datatorch.runner.directory import runner_directory
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PythonFailedError(Exception):
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
REQUIREMENTS_FILE = "requirements.txt"
|
|
16
|
+
STAMP_FILE = ".requirements.sha256"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PythonRunner(Runner):
|
|
20
|
+
"""Runs `main` with Python.
|
|
21
|
+
|
|
22
|
+
An action that ships a `requirements.txt` (October 2026) runs in its
|
|
23
|
+
own environment, built on first use under the runner's `envs/`
|
|
24
|
+
folder and rebuilt when the requirements change. The environment
|
|
25
|
+
sees the runner's own packages too (`--system-site-packages`), so the
|
|
26
|
+
DataTorch SDK and what the runner already has need not be listed.
|
|
27
|
+
Without a requirements file, the runner's interpreter is used as
|
|
28
|
+
before.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, *args, **kwargs):
|
|
32
|
+
super().__init__(*args, **kwargs)
|
|
33
|
+
if self.config.get("main") is None:
|
|
34
|
+
raise ValueError("A main path was not provided.")
|
|
35
|
+
|
|
36
|
+
def requirements_path(self) -> str:
|
|
37
|
+
return os.path.join(self.action.dir, REQUIREMENTS_FILE)
|
|
38
|
+
|
|
39
|
+
def env_dir(self) -> str:
|
|
40
|
+
identifier = self.action.identifier
|
|
41
|
+
parts = [p for p in str(identifier.name).lower().split("/") if p]
|
|
42
|
+
return os.path.join(runner_directory.envs_dir, *parts, str(identifier.version))
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def env_python(env_dir: str) -> str:
|
|
46
|
+
if os.name == "nt":
|
|
47
|
+
return os.path.join(env_dir, "Scripts", "python.exe")
|
|
48
|
+
return os.path.join(env_dir, "bin", "python")
|
|
49
|
+
|
|
50
|
+
async def ensure_environment(self) -> str:
|
|
51
|
+
"""The interpreter to run `main` with: the action's own when it has
|
|
52
|
+
requirements (installing them when missing or changed), else ours."""
|
|
53
|
+
requirements = self.requirements_path()
|
|
54
|
+
if not os.path.isfile(requirements):
|
|
55
|
+
return sys.executable
|
|
56
|
+
with open(requirements, "rb") as handle:
|
|
57
|
+
digest = hashlib.sha256(handle.read()).hexdigest()
|
|
58
|
+
env_dir = self.env_dir()
|
|
59
|
+
python = self.env_python(env_dir)
|
|
60
|
+
stamp = os.path.join(env_dir, STAMP_FILE)
|
|
61
|
+
current = None
|
|
62
|
+
if os.path.isfile(stamp):
|
|
63
|
+
with open(stamp, "r") as handle:
|
|
64
|
+
current = handle.read().strip()
|
|
65
|
+
if os.path.isfile(python) and current == digest:
|
|
66
|
+
return python
|
|
67
|
+
|
|
68
|
+
self.log(f"Preparing the action's Python environment in {env_dir}")
|
|
69
|
+
os.makedirs(os.path.dirname(env_dir), exist_ok=True)
|
|
70
|
+
if not os.path.isfile(python):
|
|
71
|
+
await self.monitor_cmd(
|
|
72
|
+
" ".join(
|
|
73
|
+
shlex.quote(part)
|
|
74
|
+
for part in (
|
|
75
|
+
sys.executable,
|
|
76
|
+
"-m",
|
|
77
|
+
"venv",
|
|
78
|
+
"--system-site-packages",
|
|
79
|
+
env_dir,
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
self.log(f"Installing {REQUIREMENTS_FILE} (first run, or the file changed)")
|
|
84
|
+
await self.monitor_cmd(
|
|
85
|
+
" ".join(
|
|
86
|
+
shlex.quote(part)
|
|
87
|
+
for part in (
|
|
88
|
+
python,
|
|
89
|
+
"-m",
|
|
90
|
+
"pip",
|
|
91
|
+
"install",
|
|
92
|
+
"--quiet",
|
|
93
|
+
"-r",
|
|
94
|
+
requirements,
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
with open(stamp, "w") as handle:
|
|
99
|
+
handle.write(digest)
|
|
100
|
+
return python
|
|
101
|
+
|
|
102
|
+
async def execute(self):
|
|
103
|
+
main = self.get("main").strip("/")
|
|
104
|
+
main_path = os.path.join(self.action.dir, main)
|
|
105
|
+
python = await self.ensure_environment()
|
|
106
|
+
|
|
107
|
+
# Inputs reach the action verbatim as one JSON argv argument.
|
|
108
|
+
# shlex.quote covers every shell metacharacter, apostrophes
|
|
109
|
+
# included, and the space in macOS "Application Support" paths.
|
|
110
|
+
json_input = json.dumps(self.variables.inputs)
|
|
111
|
+
|
|
112
|
+
await self.monitor_cmd(
|
|
113
|
+
" ".join(shlex.quote(part) for part in (python, main_path, json_input))
|
|
114
|
+
)
|
|
@@ -88,6 +88,25 @@ class Runner(object):
|
|
|
88
88
|
$INPUT_<NAME> via `input_env`)."""
|
|
89
89
|
return self.variables.render_command(self.config.get(key, default))
|
|
90
90
|
|
|
91
|
+
def runner_env(self) -> dict:
|
|
92
|
+
"""What an action needs to reach the server this runner is on:
|
|
93
|
+
the API url and the runner's own token (as DATATORCH_API_URL and
|
|
94
|
+
DATATORCH_RUNNER_TOKEN). The SDK's client picks them up, so an
|
|
95
|
+
action talks to the same server as the runner, as the runner,
|
|
96
|
+
instead of the machine's stored login. Anything already set in
|
|
97
|
+
the environment wins (DATATORCH_API_KEY included, since the SDK
|
|
98
|
+
prefers a key over the runner token)."""
|
|
99
|
+
from datatorch.core import env as env_names
|
|
100
|
+
from datatorch.runner.directory import runner_directory
|
|
101
|
+
|
|
102
|
+
values = {}
|
|
103
|
+
settings = runner_directory.settings
|
|
104
|
+
if settings.api_url and not os.environ.get(env_names.API_URL):
|
|
105
|
+
values[env_names.API_URL] = settings.api_url
|
|
106
|
+
if settings.runner_token and not os.environ.get(env_names.RUNNER_TOKEN):
|
|
107
|
+
values[env_names.RUNNER_TOKEN] = settings.runner_token
|
|
108
|
+
return values
|
|
109
|
+
|
|
91
110
|
def input_env(self) -> dict:
|
|
92
111
|
"""Resolved step inputs as $INPUT_<NAME> env vars for the command."""
|
|
93
112
|
return self.variables.env_inputs()
|
|
@@ -115,7 +134,7 @@ class Runner(object):
|
|
|
115
134
|
shell=True,
|
|
116
135
|
stdout=asyncio.subprocess.PIPE,
|
|
117
136
|
stderr=asyncio.subprocess.STDOUT,
|
|
118
|
-
env={**os.environ, **(env or {})},
|
|
137
|
+
env={**os.environ, **self.runner_env(), **(env or {})},
|
|
119
138
|
# Outputs are reported as single `::var::<json>` stdout lines
|
|
120
139
|
# (see check_for_output), so a line can carry an entire fetched
|
|
121
140
|
# page body — asyncio's 64KB default readline limit kills the
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import sys
|
|
3
|
-
import json
|
|
4
|
-
import shlex
|
|
5
|
-
|
|
6
|
-
from .runner import Runner
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class PythonFailedError(Exception):
|
|
10
|
-
pass
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class PythonRunner(Runner):
|
|
14
|
-
def __init__(self, *args, **kwargs):
|
|
15
|
-
super().__init__(*args, **kwargs)
|
|
16
|
-
if self.config.get("main") is None:
|
|
17
|
-
raise ValueError("A main path was not provided.")
|
|
18
|
-
|
|
19
|
-
async def execute(self):
|
|
20
|
-
main = self.get("main").strip("/")
|
|
21
|
-
main_path = os.path.join(self.action.dir, main)
|
|
22
|
-
|
|
23
|
-
# Inputs reach the action verbatim as one JSON argv argument.
|
|
24
|
-
# shlex.quote covers every shell metacharacter, apostrophes
|
|
25
|
-
# included, and the space in macOS "Application Support" paths.
|
|
26
|
-
json_input = json.dumps(self.variables.inputs)
|
|
27
|
-
|
|
28
|
-
await self.monitor_cmd(
|
|
29
|
-
" ".join(
|
|
30
|
-
shlex.quote(part) for part in (sys.executable, main_path, json_input)
|
|
31
|
-
)
|
|
32
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|