datatorch 0.7.3__tar.gz → 0.7.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datatorch-0.7.3 → datatorch-0.7.4}/PKG-INFO +1 -1
- datatorch-0.7.4/datatorch/agent/__init__.py +23 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/action/run.py +4 -4
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/groups.py +1 -1
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/pipeline/run.py +2 -2
- {datatorch-0.7.3/datatorch/cli/agent → datatorch-0.7.4/datatorch/cli/runner}/create.py +5 -5
- {datatorch-0.7.3/datatorch/cli/agent → datatorch-0.7.4/datatorch/cli/runner}/dir.py +2 -2
- {datatorch-0.7.3/datatorch/cli/agent → datatorch-0.7.4/datatorch/cli/runner}/start.py +3 -3
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/core/env.py +2 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/__init__.py +13 -10
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/client.py +3 -3
- datatorch-0.7.3/datatorch/agent/agent.py → datatorch-0.7.4/datatorch/runner/daemon.py +11 -7
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/directory.py +53 -18
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/log_handler.py +1 -1
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/__init__.py +3 -3
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/action.py +3 -3
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/cache.py +1 -1
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/config.py +2 -3
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/job/job.py +1 -1
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/pipeline.py +1 -1
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/step/step.py +2 -2
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/template.py +16 -9
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/PKG-INFO +1 -1
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/SOURCES.txt +35 -34
- {datatorch-0.7.3 → datatorch-0.7.4}/setup.py +1 -1
- {datatorch-0.7.3 → datatorch-0.7.4}/README.md +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/api.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/client.py +0 -0
- {datatorch-0.7.3/datatorch/agent/services → datatorch-0.7.4/datatorch/api/entity}/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/annotation.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/base.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/dataset.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/file.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/label.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/project.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/settings.py +0 -0
- {datatorch-0.7.3/datatorch/agent/utils → datatorch-0.7.4/datatorch/api/entity/sources}/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/sources/image/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/sources/image/bounding_box.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/sources/image/segmentations.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/sources/image/typings.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/sources/source.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/storage_link.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/entity/user.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/pull.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/scripts/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/scripts/import_coco.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/scripts/import_pixmask.py +0 -0
- {datatorch-0.7.3/datatorch/api/entity → datatorch-0.7.4/datatorch/api/scripts/utils}/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/scripts/utils/simplify.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/utils.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/api/where.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/action/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/action/create.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/action/pull.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/import_cmds/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/import_cmds/coco.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/import_cmds/pixmask.py +0 -0
- {datatorch-0.7.3/datatorch/api/entity/sources → datatorch-0.7.4/datatorch/cli/main}/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/main/login.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/main/logout.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/main/pull.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/main/upgrade.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/main/version.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/pipeline/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/pipeline/generate.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/pipeline/upload.py +0 -0
- {datatorch-0.7.3/datatorch/cli/agent → datatorch-0.7.4/datatorch/cli/runner}/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/spinner.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/upload/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/upload/folder.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/cli/utils.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/core/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/core/folder.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/core/settings.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/logging.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/monitoring.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/job/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/resolver.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/command.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/docker.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/factory.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/node.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/python.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/runner.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/shell.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/step/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/api/scripts/utils → datatorch-0.7.4/datatorch/runner/services}/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/cli/main → datatorch-0.7.4/datatorch/runner/utils}/__init__.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/utils/docker.py +0 -0
- {datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/utils/process.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/__init__.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/converters.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/files.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/hash_table.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/objects.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/package.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/string_style.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch/utils/url.py +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/dependency_links.txt +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/entry_points.txt +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/not-zip-safe +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/requires.txt +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/datatorch.egg-info/top_level.txt +0 -0
- {datatorch-0.7.3 → datatorch-0.7.4}/setup.cfg +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Deprecated: the runner package moved to `datatorch.runner` (October 2026).
|
|
2
|
+
|
|
3
|
+
`datatorch.agent` keeps exporting the package's public names for existing
|
|
4
|
+
scripts; submodule imports (`datatorch.agent.pipelines`) must switch to
|
|
5
|
+
`datatorch.runner.pipelines`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
from datatorch.runner import * # noqa: F401,F403
|
|
11
|
+
from datatorch.runner import RunnerDaemon as Agent # noqa: F401
|
|
12
|
+
from datatorch.runner import __all__ # noqa: F401
|
|
13
|
+
from datatorch.runner.directory import ( # noqa: F401
|
|
14
|
+
RunnerDirectory as AgentDirectory,
|
|
15
|
+
RunnerSettings as AgentSettings,
|
|
16
|
+
runner_directory as agent_directory,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
warnings.warn(
|
|
20
|
+
"datatorch.agent is deprecated; import datatorch.runner instead.",
|
|
21
|
+
DeprecationWarning,
|
|
22
|
+
stacklevel=2,
|
|
23
|
+
)
|
|
@@ -5,10 +5,10 @@ import os
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import logging
|
|
7
7
|
|
|
8
|
-
from datatorch.
|
|
9
|
-
from datatorch.
|
|
10
|
-
from datatorch.
|
|
11
|
-
from datatorch.
|
|
8
|
+
from datatorch.runner import logger
|
|
9
|
+
from datatorch.runner.pipelines.action import Action
|
|
10
|
+
from datatorch.runner.pipelines.template import Variables
|
|
11
|
+
from datatorch.runner.pipelines.action.config import ActionConfig
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def _prompt_for_inputs(action: Action, variables: Variables):
|
|
@@ -9,7 +9,7 @@ from .main.pull import pull
|
|
|
9
9
|
from .main.upgrade import package_upgrade
|
|
10
10
|
from .main.version import version
|
|
11
11
|
|
|
12
|
-
from .
|
|
12
|
+
from .runner import agent, runner
|
|
13
13
|
from .pipeline import pipeline
|
|
14
14
|
from .action import action
|
|
15
15
|
from .import_cmds import import_cmd
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import click
|
|
2
2
|
import asyncio
|
|
3
3
|
|
|
4
|
-
from datatorch.
|
|
5
|
-
from datatorch.
|
|
4
|
+
from datatorch.runner.pipelines import Pipeline, Job
|
|
5
|
+
from datatorch.runner import setup_logging
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@click.command(help="Runs a pipeline yaml file on local machine.")
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import click
|
|
2
2
|
import platform
|
|
3
3
|
|
|
4
|
-
from datatorch.
|
|
4
|
+
from datatorch.runner.directory import runner_directory
|
|
5
5
|
from datatorch.api import ApiClient
|
|
6
6
|
from datatorch.utils.package import get_version
|
|
7
7
|
from datatorch.core import user_settings, BASE_URL_API
|
|
@@ -32,9 +32,9 @@ def create_agent(name: str) -> dict:
|
|
|
32
32
|
@click.command()
|
|
33
33
|
@click.pass_context
|
|
34
34
|
def create(ctx):
|
|
35
|
-
agent_settings =
|
|
35
|
+
agent_settings = runner_directory.settings
|
|
36
36
|
|
|
37
|
-
if agent_settings.
|
|
37
|
+
if agent_settings.runner_id:
|
|
38
38
|
click.echo("A runner is already installed.")
|
|
39
39
|
confirmed = click.confirm(
|
|
40
40
|
"Would you like to create a new runner?", default=True
|
|
@@ -55,8 +55,8 @@ def create(ctx):
|
|
|
55
55
|
|
|
56
56
|
try:
|
|
57
57
|
agent = create_agent(name)["createRunner"]
|
|
58
|
-
agent_settings.
|
|
59
|
-
agent_settings.
|
|
58
|
+
agent_settings.runner_id = agent["runner"]["id"]
|
|
59
|
+
agent_settings.runner_token = agent["token"]
|
|
60
60
|
agent_settings.api_url = user_settings.api_url
|
|
61
61
|
except Exception as ex:
|
|
62
62
|
spinner.done(click.style("Failed to create runner.", fg="red"))
|
|
@@ -2,12 +2,12 @@ import os
|
|
|
2
2
|
import click
|
|
3
3
|
import platform
|
|
4
4
|
import subprocess
|
|
5
|
-
from datatorch.
|
|
5
|
+
from datatorch.runner import runner_directory
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@click.command(help="Opens the runner directory where settings and files are stored.")
|
|
9
9
|
def dir():
|
|
10
|
-
path =
|
|
10
|
+
path = runner_directory.dir
|
|
11
11
|
if platform.system() == "Windows":
|
|
12
12
|
os.startfile(path)
|
|
13
13
|
elif platform.system() == "Darwin":
|
|
@@ -3,11 +3,11 @@ import asyncio
|
|
|
3
3
|
import signal
|
|
4
4
|
|
|
5
5
|
from asyncio.events import AbstractEventLoop
|
|
6
|
-
from datatorch import
|
|
6
|
+
from datatorch import runner
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
def add_signal_handlers(loop: AbstractEventLoop):
|
|
10
|
-
exit_func = lambda: loop.create_task(
|
|
10
|
+
exit_func = lambda: loop.create_task(runner.stop())
|
|
11
11
|
try:
|
|
12
12
|
loop.add_signal_handler(signal.SIGINT, exit_func)
|
|
13
13
|
loop.add_signal_handler(signal.SIGTERM, exit_func)
|
|
@@ -19,6 +19,6 @@ def add_signal_handlers(loop: AbstractEventLoop):
|
|
|
19
19
|
def start():
|
|
20
20
|
loop = asyncio.new_event_loop()
|
|
21
21
|
asyncio.set_event_loop(loop)
|
|
22
|
-
loop.create_task(
|
|
22
|
+
loop.create_task(runner.start())
|
|
23
23
|
add_signal_handlers(loop)
|
|
24
24
|
loop.run_forever()
|
|
@@ -6,8 +6,8 @@ import os
|
|
|
6
6
|
from logging.handlers import RotatingFileHandler
|
|
7
7
|
from asyncio import IncompleteReadError
|
|
8
8
|
|
|
9
|
-
from .directory import
|
|
10
|
-
from .
|
|
9
|
+
from .directory import runner_directory
|
|
10
|
+
from .daemon import RunnerDaemon, tasks
|
|
11
11
|
|
|
12
12
|
from gql import Client as GqlClient
|
|
13
13
|
from gql.transport.exceptions import TransportError, TransportQueryError
|
|
@@ -17,14 +17,17 @@ from websockets.exceptions import ConnectionClosed, InvalidMessage, InvalidURI
|
|
|
17
17
|
from datatorch.api import Client as DtClient
|
|
18
18
|
from datatorch.utils.package import get_version
|
|
19
19
|
|
|
20
|
-
__all__ = ["Agent", "start", "stop"]
|
|
20
|
+
__all__ = ["RunnerDaemon", "Agent", "start", "stop"]
|
|
21
|
+
|
|
22
|
+
# Pre-rename name, kept for callers outside this package.
|
|
23
|
+
Agent = RunnerDaemon
|
|
21
24
|
|
|
22
25
|
|
|
23
26
|
logger = logging.getLogger(__name__)
|
|
24
27
|
|
|
25
28
|
|
|
26
|
-
_url =
|
|
27
|
-
_agent_token =
|
|
29
|
+
_url = runner_directory.settings.api_url
|
|
30
|
+
_agent_token = runner_directory.settings.runner_token
|
|
28
31
|
|
|
29
32
|
|
|
30
33
|
BACKOFF_INIT_WAIT = 2
|
|
@@ -33,17 +36,17 @@ BACKOFF_MAX = 900
|
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
def directories():
|
|
36
|
-
return
|
|
39
|
+
return runner_directory
|
|
37
40
|
|
|
38
41
|
|
|
39
42
|
def setup_logging() -> None:
|
|
40
|
-
logs_dir =
|
|
43
|
+
logs_dir = runner_directory.logs_dir
|
|
41
44
|
logging.basicConfig(
|
|
42
45
|
format="%(asctime)s %(name)-30s %(levelname)-8s %(message)s",
|
|
43
46
|
level=logging.WARN,
|
|
44
47
|
handlers=[
|
|
45
48
|
RotatingFileHandler(
|
|
46
|
-
os.path.join(logs_dir, "
|
|
49
|
+
os.path.join(logs_dir, "runner.log"), maxBytes=100000, backupCount=10
|
|
47
50
|
),
|
|
48
51
|
logging.StreamHandler(),
|
|
49
52
|
],
|
|
@@ -110,7 +113,7 @@ async def start() -> None:
|
|
|
110
113
|
setup_logging()
|
|
111
114
|
|
|
112
115
|
click.echo(
|
|
113
|
-
click.style(f"Starting DataTorch
|
|
116
|
+
click.style(f"Starting DataTorch Runner v{get_version()}", fg="blue", bold=True)
|
|
114
117
|
)
|
|
115
118
|
logger.debug(f"API Endpoint at {_url}")
|
|
116
119
|
backoff_wait = BACKOFF_INIT_WAIT
|
|
@@ -122,7 +125,7 @@ async def start() -> None:
|
|
|
122
125
|
try:
|
|
123
126
|
async with client as session:
|
|
124
127
|
backoff_wait = BACKOFF_INIT_WAIT
|
|
125
|
-
await
|
|
128
|
+
await RunnerDaemon.run(session)
|
|
126
129
|
|
|
127
130
|
except (asyncio.CancelledError, InvalidURI):
|
|
128
131
|
break
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import AsyncGenerator, List, Optional, cast
|
|
2
2
|
from gql import gql
|
|
3
|
-
from .directory import
|
|
3
|
+
from .directory import runner_directory
|
|
4
4
|
from gql.client import AsyncClientSession
|
|
5
5
|
|
|
6
6
|
from typing_extensions import TypedDict
|
|
@@ -195,7 +195,7 @@ class AgentApiClient(object):
|
|
|
195
195
|
}
|
|
196
196
|
"""
|
|
197
197
|
# fmt: on
|
|
198
|
-
params = {"id":
|
|
198
|
+
params = {"id": runner_directory.settings.runner_id, **metrics}
|
|
199
199
|
return self.execute(gql(mutate), params=params)
|
|
200
200
|
|
|
201
201
|
def metrics(self, metrics):
|
|
@@ -226,7 +226,7 @@ class AgentApiClient(object):
|
|
|
226
226
|
}
|
|
227
227
|
"""
|
|
228
228
|
# fmt: on
|
|
229
|
-
params = {"runnerId":
|
|
229
|
+
params = {"runnerId": runner_directory.settings.runner_id, **metrics}
|
|
230
230
|
return self.execute(mutate, params=params)
|
|
231
231
|
|
|
232
232
|
def upload_step_logs(self, step_id: str, logs: List[Log]):
|
|
@@ -7,18 +7,18 @@ from .client import AgentApiClient, AgentStepDispatch
|
|
|
7
7
|
from .pipelines.template import Variables
|
|
8
8
|
from .log_handler import AgentAPIHandler
|
|
9
9
|
from .monitoring import AgentSystemStats
|
|
10
|
-
from .directory import
|
|
10
|
+
from .directory import runner_directory
|
|
11
11
|
|
|
12
12
|
from gql.client import AsyncClientSession
|
|
13
|
-
from datatorch.
|
|
13
|
+
from datatorch.runner.pipelines.step import Step
|
|
14
14
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
tasks: List[asyncio.Task] = []
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
class
|
|
19
|
+
class RunnerDaemon(object):
|
|
20
20
|
"""
|
|
21
|
-
|
|
21
|
+
Runner protocol v2: the server sequences steps. The runner receives one
|
|
22
22
|
ready step at a time — its inputs already resolved server-side —
|
|
23
23
|
claims it, executes it, and reports the result. Job-level state and
|
|
24
24
|
cross-step dataflow live entirely on the server.
|
|
@@ -31,7 +31,7 @@ class Agent(object):
|
|
|
31
31
|
|
|
32
32
|
def __init__(self, session: AsyncClientSession):
|
|
33
33
|
self.api = AgentApiClient(session)
|
|
34
|
-
self.directory =
|
|
34
|
+
self.directory = runner_directory
|
|
35
35
|
# Running step tasks, keyed by stepId, so a server cancel/timeout
|
|
36
36
|
# signal can abort the right one. Entries are cleaned up when the
|
|
37
37
|
# step finishes.
|
|
@@ -43,10 +43,10 @@ class Agent(object):
|
|
|
43
43
|
self._init_threads()
|
|
44
44
|
|
|
45
45
|
def _init_logger(self):
|
|
46
|
-
self.logger = logging.getLogger("datatorch.
|
|
46
|
+
self.logger = logging.getLogger("datatorch.runner")
|
|
47
47
|
self.logger_api_handler = AgentAPIHandler(self.api)
|
|
48
48
|
self.logger.addHandler(self.logger_api_handler)
|
|
49
|
-
self.logger.debug("
|
|
49
|
+
self.logger.debug("Runner logger has been initialized.")
|
|
50
50
|
|
|
51
51
|
def _init_threads(self):
|
|
52
52
|
self.system_stats = AgentSystemStats(self)
|
|
@@ -145,3 +145,7 @@ class Agent(object):
|
|
|
145
145
|
logger.info(f"Step finished (name: {step_name}, id: {step_id})")
|
|
146
146
|
finally:
|
|
147
147
|
self._step_tasks.pop(step_id, None)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# Pre-rename name, kept for callers outside this package.
|
|
151
|
+
Agent = RunnerDaemon
|
|
@@ -4,15 +4,33 @@ from datatorch.utils.files import mkdir_exists
|
|
|
4
4
|
from datatorch.core import Settings, folder, env, user_settings
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class
|
|
7
|
+
class RunnerDirectory(object):
|
|
8
8
|
@staticmethod
|
|
9
9
|
def path() -> str:
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
"""The runner's working directory (actions cache, runs, logs, settings).
|
|
11
|
+
|
|
12
|
+
`DATATORCH_RUNNER_PATH` wins, then the pre-rename `DATATORCH_AGENT_PATH`,
|
|
13
|
+
then `<app dir>/runner`. An install that still has `<app dir>/agent`
|
|
14
|
+
from before the rename is moved to `runner` once, so a runner keeps
|
|
15
|
+
its identity, token and cached actions across the upgrade.
|
|
16
|
+
"""
|
|
17
|
+
explicit = os.getenv("DATATORCH_RUNNER_PATH") or os.getenv(
|
|
18
|
+
"DATATORCH_AGENT_PATH"
|
|
19
|
+
)
|
|
20
|
+
if explicit:
|
|
21
|
+
return explicit
|
|
22
|
+
app = folder.get_app_dir()
|
|
23
|
+
new = os.path.join(app, "runner")
|
|
24
|
+
old = os.path.join(app, "agent")
|
|
25
|
+
if not os.path.exists(new) and os.path.isdir(old):
|
|
26
|
+
try:
|
|
27
|
+
os.rename(old, new)
|
|
28
|
+
except OSError:
|
|
29
|
+
return old
|
|
30
|
+
return new
|
|
13
31
|
|
|
14
32
|
def __init__(self):
|
|
15
|
-
self.settings =
|
|
33
|
+
self.settings = RunnerSettings()
|
|
16
34
|
|
|
17
35
|
mkdir_exists(self.dir)
|
|
18
36
|
mkdir_exists(self.db_dir)
|
|
@@ -81,25 +99,37 @@ class AgentDirectory(object):
|
|
|
81
99
|
return os.path.join(self.dir, "temp")
|
|
82
100
|
|
|
83
101
|
|
|
84
|
-
class
|
|
102
|
+
class RunnerSettings(Settings):
|
|
103
|
+
"""The runner's own settings file. Keys are `runnerId` / `runnerToken`;
|
|
104
|
+
the pre-rename `agentId` / `agentToken` are still read, so an upgraded
|
|
105
|
+
runner keeps working until it is re-registered."""
|
|
106
|
+
|
|
85
107
|
def __init__(self):
|
|
86
|
-
super().__init__(
|
|
108
|
+
super().__init__(RunnerDirectory.path())
|
|
87
109
|
|
|
88
110
|
@property
|
|
89
|
-
def
|
|
90
|
-
return
|
|
111
|
+
def runner_id(self):
|
|
112
|
+
return (
|
|
113
|
+
os.getenv(env.RUNNER_ID)
|
|
114
|
+
or self.get("runnerId", env=env.AGENT_ID)
|
|
115
|
+
or self.get("agentId")
|
|
116
|
+
)
|
|
91
117
|
|
|
92
|
-
@
|
|
93
|
-
def
|
|
94
|
-
self.set("
|
|
118
|
+
@runner_id.setter
|
|
119
|
+
def runner_id(self, value):
|
|
120
|
+
self.set("runnerId", value)
|
|
95
121
|
|
|
96
122
|
@property
|
|
97
|
-
def
|
|
98
|
-
return self.get("agentToken")
|
|
123
|
+
def runner_token(self):
|
|
124
|
+
return self.get("runnerToken") or self.get("agentToken")
|
|
99
125
|
|
|
100
|
-
@
|
|
101
|
-
def
|
|
102
|
-
self.set("
|
|
126
|
+
@runner_token.setter
|
|
127
|
+
def runner_token(self, value):
|
|
128
|
+
self.set("runnerToken", value)
|
|
129
|
+
|
|
130
|
+
# Pre-rename names, kept for callers outside this package.
|
|
131
|
+
agent_id = runner_id
|
|
132
|
+
agent_token = runner_token
|
|
103
133
|
|
|
104
134
|
@property
|
|
105
135
|
def api_url(self):
|
|
@@ -110,4 +140,9 @@ class AgentSettings(Settings):
|
|
|
110
140
|
self.set("apiUrl", value)
|
|
111
141
|
|
|
112
142
|
|
|
113
|
-
|
|
143
|
+
runner_directory = RunnerDirectory()
|
|
144
|
+
|
|
145
|
+
# Pre-rename names, kept for callers outside this package.
|
|
146
|
+
AgentDirectory = RunnerDirectory
|
|
147
|
+
AgentSettings = RunnerSettings
|
|
148
|
+
agent_directory = runner_directory
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/__init__.py
RENAMED
|
@@ -4,7 +4,7 @@ import os
|
|
|
4
4
|
import logging
|
|
5
5
|
import typing
|
|
6
6
|
|
|
7
|
-
from datatorch.
|
|
7
|
+
from datatorch.runner.directory import runner_directory
|
|
8
8
|
from .config import ActionConfig, LATEST_VERSION
|
|
9
9
|
|
|
10
10
|
if typing.TYPE_CHECKING:
|
|
@@ -14,12 +14,12 @@ if typing.TYPE_CHECKING:
|
|
|
14
14
|
__all__ = ["Action", "get_action"]
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
logger = logging.getLogger("datatorch.
|
|
17
|
+
logger = logging.getLogger("datatorch.runner.action")
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
async def get_action(config: ActionConfig, step: "Step" = None) -> Action:
|
|
21
21
|
# Get actions directory
|
|
22
|
-
action_dir =
|
|
22
|
+
action_dir = runner_directory.action_dir(config.name, config.version)
|
|
23
23
|
folder_exists = os.path.exists(action_dir)
|
|
24
24
|
force_download = config.version == LATEST_VERSION
|
|
25
25
|
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/action.py
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
from datatorch.
|
|
2
|
-
from datatorch.
|
|
1
|
+
from datatorch.runner.pipelines.action.cache import ActionHashable, ActionHashTable
|
|
2
|
+
from datatorch.runner.pipelines.template import Variables
|
|
3
3
|
from datatorch.utils.objects import pick
|
|
4
4
|
|
|
5
5
|
from typing import Any, Dict, Union
|
|
@@ -16,7 +16,7 @@ if typing.TYPE_CHECKING:
|
|
|
16
16
|
from ..step import Step
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
logger = logging.getLogger("datatorch.
|
|
19
|
+
logger = logging.getLogger("datatorch.runner.action")
|
|
20
20
|
|
|
21
21
|
_actions_cache = ActionHashTable()
|
|
22
22
|
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/action/config.py
RENAMED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import asyncio, os, shutil
|
|
2
|
-
from datatorch import agent
|
|
3
2
|
from datatorch.utils.files import mkdir_exists
|
|
4
3
|
from logging import getLogger
|
|
5
4
|
|
|
6
5
|
from typing import Union
|
|
7
|
-
from ...directory import
|
|
6
|
+
from ...directory import runner_directory
|
|
8
7
|
|
|
9
8
|
logger = getLogger(__name__)
|
|
10
9
|
|
|
@@ -70,7 +69,7 @@ class ActionConfig(object):
|
|
|
70
69
|
self.depth: int = 1
|
|
71
70
|
|
|
72
71
|
async def download(self):
|
|
73
|
-
path =
|
|
72
|
+
path = runner_directory.action_dir(self.name, self.version)
|
|
74
73
|
if os.path.isdir(path):
|
|
75
74
|
shutil.rmtree(path)
|
|
76
75
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from datatorch.utils.objects import pick
|
|
2
|
-
from datatorch.
|
|
2
|
+
from datatorch.runner.pipelines.action.config import ActionConfig
|
|
3
3
|
import logging
|
|
4
4
|
from typing import List, Optional
|
|
5
5
|
from datetime import datetime, timezone
|
|
@@ -65,7 +65,7 @@ class Step(object):
|
|
|
65
65
|
self.run_id = run_id
|
|
66
66
|
self.api = api
|
|
67
67
|
self.rendered_inputs: Optional[dict] = None
|
|
68
|
-
self.logger = logging.getLogger(f"datatorch.
|
|
68
|
+
self.logger = logging.getLogger(f"datatorch.runner.[{self._action.full_name}]")
|
|
69
69
|
|
|
70
70
|
async def action(self) -> Action:
|
|
71
71
|
return await get_action(self._action, step=self)
|
|
@@ -4,7 +4,7 @@ import re
|
|
|
4
4
|
import typing
|
|
5
5
|
from typing import Dict
|
|
6
6
|
from jinja2 import Template
|
|
7
|
-
from ..directory import
|
|
7
|
+
from ..directory import runner_directory
|
|
8
8
|
|
|
9
9
|
if typing.TYPE_CHECKING:
|
|
10
10
|
from .step import Step
|
|
@@ -43,16 +43,23 @@ def _env_key(key: str) -> str:
|
|
|
43
43
|
return "INPUT_" + re.sub(r"[^A-Z0-9]", "_", key.upper())
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
_runner_namespace = {
|
|
47
|
+
"id": runner_directory.settings.runner_id,
|
|
48
|
+
"directory": runner_directory.dir,
|
|
49
|
+
}
|
|
50
|
+
|
|
46
51
|
global_variables = {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
52
|
+
# `${{ runner.* }}` is the spelling since October 2026; `agent.*` is
|
|
53
|
+
# the same namespace under its pre-rename name, kept for existing
|
|
54
|
+
# action yaml.
|
|
55
|
+
"runner": _runner_namespace,
|
|
56
|
+
"agent": _runner_namespace,
|
|
51
57
|
"directory": {
|
|
52
|
-
"
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
58
|
+
"runner": runner_directory.dir,
|
|
59
|
+
"agent": runner_directory.dir,
|
|
60
|
+
"actions": runner_directory.actions_dir,
|
|
61
|
+
"logs": runner_directory.logs_dir,
|
|
62
|
+
"temp": runner_directory.temp_dir,
|
|
56
63
|
},
|
|
57
64
|
"machine": {
|
|
58
65
|
"name": platform.node(),
|
|
@@ -9,36 +9,6 @@ datatorch.egg-info/not-zip-safe
|
|
|
9
9
|
datatorch.egg-info/requires.txt
|
|
10
10
|
datatorch.egg-info/top_level.txt
|
|
11
11
|
datatorch/agent/__init__.py
|
|
12
|
-
datatorch/agent/agent.py
|
|
13
|
-
datatorch/agent/client.py
|
|
14
|
-
datatorch/agent/directory.py
|
|
15
|
-
datatorch/agent/log_handler.py
|
|
16
|
-
datatorch/agent/logging.py
|
|
17
|
-
datatorch/agent/monitoring.py
|
|
18
|
-
datatorch/agent/pipelines/__init__.py
|
|
19
|
-
datatorch/agent/pipelines/pipeline.py
|
|
20
|
-
datatorch/agent/pipelines/resolver.py
|
|
21
|
-
datatorch/agent/pipelines/template.py
|
|
22
|
-
datatorch/agent/pipelines/action/__init__.py
|
|
23
|
-
datatorch/agent/pipelines/action/action.py
|
|
24
|
-
datatorch/agent/pipelines/action/cache.py
|
|
25
|
-
datatorch/agent/pipelines/action/config.py
|
|
26
|
-
datatorch/agent/pipelines/job/__init__.py
|
|
27
|
-
datatorch/agent/pipelines/job/job.py
|
|
28
|
-
datatorch/agent/pipelines/runner/__init__.py
|
|
29
|
-
datatorch/agent/pipelines/runner/command.py
|
|
30
|
-
datatorch/agent/pipelines/runner/docker.py
|
|
31
|
-
datatorch/agent/pipelines/runner/factory.py
|
|
32
|
-
datatorch/agent/pipelines/runner/node.py
|
|
33
|
-
datatorch/agent/pipelines/runner/python.py
|
|
34
|
-
datatorch/agent/pipelines/runner/runner.py
|
|
35
|
-
datatorch/agent/pipelines/runner/shell.py
|
|
36
|
-
datatorch/agent/pipelines/step/__init__.py
|
|
37
|
-
datatorch/agent/pipelines/step/step.py
|
|
38
|
-
datatorch/agent/services/__init__.py
|
|
39
|
-
datatorch/agent/utils/__init__.py
|
|
40
|
-
datatorch/agent/utils/docker.py
|
|
41
|
-
datatorch/agent/utils/process.py
|
|
42
12
|
datatorch/api/__init__.py
|
|
43
13
|
datatorch/api/api.py
|
|
44
14
|
datatorch/api/client.py
|
|
@@ -74,10 +44,6 @@ datatorch/cli/action/__init__.py
|
|
|
74
44
|
datatorch/cli/action/create.py
|
|
75
45
|
datatorch/cli/action/pull.py
|
|
76
46
|
datatorch/cli/action/run.py
|
|
77
|
-
datatorch/cli/agent/__init__.py
|
|
78
|
-
datatorch/cli/agent/create.py
|
|
79
|
-
datatorch/cli/agent/dir.py
|
|
80
|
-
datatorch/cli/agent/start.py
|
|
81
47
|
datatorch/cli/import_cmds/__init__.py
|
|
82
48
|
datatorch/cli/import_cmds/coco.py
|
|
83
49
|
datatorch/cli/import_cmds/pixmask.py
|
|
@@ -91,12 +57,47 @@ datatorch/cli/pipeline/__init__.py
|
|
|
91
57
|
datatorch/cli/pipeline/generate.py
|
|
92
58
|
datatorch/cli/pipeline/run.py
|
|
93
59
|
datatorch/cli/pipeline/upload.py
|
|
60
|
+
datatorch/cli/runner/__init__.py
|
|
61
|
+
datatorch/cli/runner/create.py
|
|
62
|
+
datatorch/cli/runner/dir.py
|
|
63
|
+
datatorch/cli/runner/start.py
|
|
94
64
|
datatorch/cli/upload/__init__.py
|
|
95
65
|
datatorch/cli/upload/folder.py
|
|
96
66
|
datatorch/core/__init__.py
|
|
97
67
|
datatorch/core/env.py
|
|
98
68
|
datatorch/core/folder.py
|
|
99
69
|
datatorch/core/settings.py
|
|
70
|
+
datatorch/runner/__init__.py
|
|
71
|
+
datatorch/runner/client.py
|
|
72
|
+
datatorch/runner/daemon.py
|
|
73
|
+
datatorch/runner/directory.py
|
|
74
|
+
datatorch/runner/log_handler.py
|
|
75
|
+
datatorch/runner/logging.py
|
|
76
|
+
datatorch/runner/monitoring.py
|
|
77
|
+
datatorch/runner/pipelines/__init__.py
|
|
78
|
+
datatorch/runner/pipelines/pipeline.py
|
|
79
|
+
datatorch/runner/pipelines/resolver.py
|
|
80
|
+
datatorch/runner/pipelines/template.py
|
|
81
|
+
datatorch/runner/pipelines/action/__init__.py
|
|
82
|
+
datatorch/runner/pipelines/action/action.py
|
|
83
|
+
datatorch/runner/pipelines/action/cache.py
|
|
84
|
+
datatorch/runner/pipelines/action/config.py
|
|
85
|
+
datatorch/runner/pipelines/job/__init__.py
|
|
86
|
+
datatorch/runner/pipelines/job/job.py
|
|
87
|
+
datatorch/runner/pipelines/runner/__init__.py
|
|
88
|
+
datatorch/runner/pipelines/runner/command.py
|
|
89
|
+
datatorch/runner/pipelines/runner/docker.py
|
|
90
|
+
datatorch/runner/pipelines/runner/factory.py
|
|
91
|
+
datatorch/runner/pipelines/runner/node.py
|
|
92
|
+
datatorch/runner/pipelines/runner/python.py
|
|
93
|
+
datatorch/runner/pipelines/runner/runner.py
|
|
94
|
+
datatorch/runner/pipelines/runner/shell.py
|
|
95
|
+
datatorch/runner/pipelines/step/__init__.py
|
|
96
|
+
datatorch/runner/pipelines/step/step.py
|
|
97
|
+
datatorch/runner/services/__init__.py
|
|
98
|
+
datatorch/runner/utils/__init__.py
|
|
99
|
+
datatorch/runner/utils/docker.py
|
|
100
|
+
datatorch/runner/utils/process.py
|
|
100
101
|
datatorch/utils/__init__.py
|
|
101
102
|
datatorch/utils/converters.py
|
|
102
103
|
datatorch/utils/files.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/agent/services → datatorch-0.7.4/datatorch/api/entity}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/agent/utils → datatorch-0.7.4/datatorch/api/entity/sources}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/api/entity → datatorch-0.7.4/datatorch/api/scripts/utils}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/api/entity/sources → datatorch-0.7.4/datatorch/cli/main}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/job/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/__init__.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/command.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/docker.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/factory.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/node.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/python.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/runner.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/runner/shell.py
RENAMED
|
File without changes
|
{datatorch-0.7.3/datatorch/agent → datatorch-0.7.4/datatorch/runner}/pipelines/step/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|