datatorch 0.7.4__tar.gz → 0.7.7__tar.gz

{datatorch-0.7.4 → datatorch-0.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatorch
-Version: 0.7.4
+Version: 0.7.7
 Summary: A CLI and library for interacting with DataTorch.
 Home-page: https://github.com/datatorch/python
 Author: DataTorch

datatorch-0.7.7/datatorch/cli/action/create.py

@@ -0,0 +1,129 @@
+import os
+import click
+import yaml
+import json as jsonlib
+
+# What a local action's run.py starts as: the image comes in, a result
+# goes out, and the model goes in the middle.
+MODEL_SCRIPT = """import datatorch
+
+image_path = datatorch.get_input("imagePath")
+
+# Load your model and run it on image_path here. Anything your model
+# needs must be installed in the Python this runner runs with.
+result = "ok"
+
+datatorch.set_output("result", result)
+"""
+
+
+@click.command(
+    help="Creates a basic hello world action in the current folder, or, with "
+    "--local, in this runner's own actions folder so a pipeline can run it as "
+    "local/<name>@<tag> without putting the code anywhere else."
+)
+@click.option(
+    "-l",
+    "--local",
+    "local_name",
+    type=str,
+    default=None,
+    metavar="NAME",
+    help="Create it under the runner's actions/local/NAME/TAG folder (lowercase, "
+    "digits, - and _). The runner reports it to DataTorch within a minute.",
+)
+@click.option(
+    "-t",
+    "--tag",
+    type=str,
+    default="v1",
+    show_default=True,
+    help="With --local: the tag folder. Pipelines pin it as local/NAME@TAG.",
+)
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    default="Hello World",
+    help="Name of action to be created.",
+)
+@click.option(
+    "-d",
+    "--description",
+    type=str,
+    default="Action created using CI template.",
+    help="Description of action to be created.",
+)
+@click.option(
+    "--json",
+    is_flag=True,
+    default=False,
+    help="Use JSON action instead of YAML. JSON format supports schemas for better typing.",
+)
+def create(name, description, json, local_name, tag):
+    target = "."
+    if local_name is not None:
+        from datatorch.runner.directory import RunnerDirectory, runner_directory
+
+        if not RunnerDirectory.LOCAL_ACTION_NAME.match(local_name):
+            raise click.BadParameter(
+                "use lowercase letters, digits, - and _", param_hint="--local"
+            )
+        if not RunnerDirectory.LOCAL_ACTION_TAG.match(tag):
+            raise click.BadParameter("not a tag folder name", param_hint="--tag")
+        target = os.path.join(runner_directory.local_actions_dir, local_name, tag)
+        os.makedirs(target, exist_ok=True)
+        if name == "Hello World":
+            name = local_name
+    if local_name is not None:
+        # A local action is almost always someone's own model: start it
+        # as one. The runner hands over the image's path; run.py returns
+        # a result. Mirrors the in-browser editor's scaffold.
+        config = {
+            "$schema": "http://datatorch.io/schema/action.v1.json",
+            "name": name,
+            "description": (
+                description
+                if description != "Action created using CI template."
+                else "Runs my model on one image."
+            ),
+            "inputs": {
+                "imagePath": {
+                    "type": "string",
+                    "required": True,
+                    "description": "Path of the image on disk",
+                }
+            },
+            "outputs": {"result": {"type": "string"}},
+            "runs": {"using": "python", "main": "run.py"},
+        }
+        with open(os.path.join(target, "run.py"), "w") as script:
+            script.write(MODEL_SCRIPT)
+    else:
+        config = {
+            "$schema": "http://datatorch.io/schema/action.v1.json",
+            "name": name,
+            "description": description,
+            "inputs": {
+                "example": {
+                    "type": "string",
+                    "description": "Input string that will get printed to console.",
+                    "default": "Default Example",
+                }
+            },
+            "outputs": {},
+            # Inputs reach shell commands as $INPUT_<NAME> environment variables;
+            # interpolating ${{ input.* }} into a command raises InputInjectionError.
+            "runs": {"using": "cmd", "command": 'echo "$INPUT_EXAMPLE"'},
+        }
+    if json:
+        with open(os.path.join(target, "action-datatorch.json"), "w") as outfile:
+            jsonlib.dump(config, outfile, indent=2, sort_keys=True)
+    else:
+        with open(os.path.join(target, "action-datatorch.yaml"), "w") as outfile:
+            yaml.dump(config, outfile, default_flow_style=False)
+    if local_name is not None:
+        click.echo(f"Created {os.path.abspath(target)}")
+        click.echo(
+            f"Reference it from a pipeline step as: action: local/{local_name}@{tag}"
+        )

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch/runner/client.py

@@ -160,6 +160,36 @@ class AgentApiClient(object):
         result = await self.execute(mutate, params={"id": step_id, "input": step_input})
         return bool(result.get("completed"))
 
+    def runner_commands(self):
+        """Subscribe to requests the server makes of this runner: read or
+        write one of its own `actions/local/<name>/<tag>` folders (October
+        2026). Answered with `complete_runner_command`."""
+        # fmt: off
+        sub = gql("""
+            subscription {
+                command: runnerCommands {
+                    id
+                    runnerId
+                    kind
+                    payload
+                }
+            }
+        """)
+        # fmt: on
+        return self.session.subscribe(sub)
+
+    def complete_runner_command(self, command_id: str, result=None, error: str = None):
+        # fmt: off
+        mutate = """
+            mutation completeRunnerCommand($id: ID!, $result: JSON, $error: String) {
+                completeRunnerCommand(id: $id, result: $result, error: $error)
+            }
+        """
+        # fmt: on
+        return self.execute(
+            gql(mutate), params={"id": command_id, "result": result, "error": error}
+        )
+
     def initial_metrics(self, metrics):
         # fmt: off
         mutate = """
@@ -198,6 +228,25 @@ class AgentApiClient(object):
         params = {"id": runner_directory.settings.runner_id, **metrics}
         return self.execute(gql(mutate), params=params)
 
+    def report_local_actions(self, actions):
+        """Tells the server which `actions/local/<name>/<tag>` folders this
+        runner holds (their action yaml text). The whole list each time: a
+        folder no longer listed is gone from the server too."""
+        # fmt: off
+        mutate = """
+            mutation reportRunnerLocalActions(
+                $id: ID!
+                $localActions: [RunnerLocalActionInput!]!
+            ) {
+                updateRunner(id: $id, input: { localActions: $localActions }) {
+                    id
+                }
+            }
+        """
+        # fmt: on
+        params = {"id": runner_directory.settings.runner_id, "localActions": actions}
+        return self.execute(gql(mutate), params=params)
+
     def metrics(self, metrics):
         # fmt: off
         mutate = """

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch/runner/daemon.py

@@ -72,11 +72,70 @@ class RunnerDaemon(object):
         logger.info("Waiting for steps.")
         cancel_task = loop.create_task(self._cancel_loop())
         tasks.append(cancel_task)
+        command_task = loop.create_task(self._command_loop())
+        tasks.append(command_task)
         try:
             async for request in self.api.agent_steps():
                 self._start_step(request.get("step"))
         finally:
             cancel_task.cancel()
+            command_task.cancel()
+
+    async def _command_loop(self):
+        """Answers the server's requests about this runner's own action
+        folders (read or write `actions/local/<name>/<tag>`). Nothing
+        outside those folders is ever read or written: the directory
+        refuses any other path. A failure is the answer's error, never a
+        crash of the loop."""
+        try:
+            async for request in self.api.runner_commands():
+                command = request.get("command") or {}
+                loop = asyncio.get_running_loop()
+                loop.create_task(self._answer_command(command))
+        except asyncio.CancelledError:
+            raise
+        except Exception as ex:
+            # A server from before runner commands has no such subscription;
+            # steps still flow, only editing from DataTorch is off.
+            logger.warning(f"Runner commands are not available: {ex}")
+
+    async def _answer_command(self, command: dict):
+        command_id = command.get("id")
+        kind = command.get("kind")
+        payload = command.get("payload") or {}
+        result, error = None, None
+        try:
+            name, tag = payload.get("name"), payload.get("tag")
+            if kind == "READ_LOCAL_ACTION":
+                result = {"files": runner_directory.local_action_files(name, tag)}
+            elif kind == "WRITE_LOCAL_ACTION":
+                files = runner_directory.write_local_action_files(
+                    name, tag, payload.get("files") or [], payload.get("delete") or []
+                )
+                result = {"files": files}
+                logger.info(f"Wrote actions/local/{name}/{tag} on request.")
+                # Report the folder's yaml now, so the picker lists it at once.
+                try:
+                    await self.api.report_local_actions(
+                        runner_directory.local_actions()
+                    )
+                except Exception as ex:
+                    logger.warning(
+                        f"Could not report local actions after a write: {ex}"
+                    )
+            else:
+                error = f"Unknown runner command '{kind}'."
+        except (ValueError, FileNotFoundError, OSError) as ex:
+            error = str(ex)
+        except Exception as ex:  # the loop must survive anything
+            logger.exception("Runner command failed.")
+            error = f"{type(ex).__name__}: {ex}"
+        try:
+            await self.api.complete_runner_command(
+                command_id, result=result, error=error
+            )
+        except Exception as ex:
+            logger.warning(f"Could not answer runner command {command_id}: {ex}")
 
     async def _cancel_loop(self):
         """Aborts a running step's task when the server signals a stop."""

datatorch-0.7.7/datatorch/runner/directory.py

@@ -0,0 +1,290 @@
+import os
+import re
+from typing import List, Union
+from datatorch.utils.files import mkdir_exists
+from datatorch.core import Settings, folder, env, user_settings
+
+# The server refuses a longer action file (MAX_LOCAL_ACTION_TEXT there).
+MAX_LOCAL_ACTION_TEXT = 64 * 1024
+
+
+class RunnerDirectory(object):
+    @staticmethod
+    def path() -> str:
+        """The runner's working directory (actions cache, runs, logs, settings).
+
+        `DATATORCH_RUNNER_PATH` wins, then the pre-rename `DATATORCH_AGENT_PATH`,
+        then `<app dir>/runner`. An install that still has `<app dir>/agent`
+        from before the rename is moved to `runner` once, so a runner keeps
+        its identity, token and cached actions across the upgrade.
+        """
+        explicit = os.getenv("DATATORCH_RUNNER_PATH") or os.getenv(
+            "DATATORCH_AGENT_PATH"
+        )
+        if explicit:
+            return explicit
+        app = folder.get_app_dir()
+        new = os.path.join(app, "runner")
+        old = os.path.join(app, "agent")
+        if not os.path.exists(new) and os.path.isdir(old):
+            try:
+                os.rename(old, new)
+            except OSError:
+                return old
+        return new
+
+    def __init__(self):
+        self.settings = RunnerSettings()
+
+        mkdir_exists(self.dir)
+        mkdir_exists(self.db_dir)
+        mkdir_exists(self.logs_dir)
+        mkdir_exists(self.temp_dir)
+        mkdir_exists(self.runs_dir)
+        mkdir_exists(self.actions_dir)
+        mkdir_exists(self.projects_dir)
+
+    @property
+    def root(self):
+        return self.path()
+
+    @property
+    def dir(self):
+        return self.path()
+
+    @property
+    def runs_dir(self):
+        return os.path.join(self.dir, "runs")
+
+    @property
+    def logs_dir(self):
+        """Directory where agent logs are stored."""
+        return os.path.join(self.dir, "logs")
+
+    @property
+    def db_dir(self):
+        """Sqlite database are stored."""
+        return os.path.join(self.dir, "db")
+
+    @property
+    def projects_dir(self):
+        """Directory where projects are stored.
+
+        Commonly used for caching project information such as annotations and
+        files.
+        """
+        return os.path.join(self.dir, "projects")
+
+    @property
+    def actions_dir(self):
+        """Directory where actions are stored."""
+        return os.path.join(self.dir, "actions")
+
+    def open(self, file: str, mode: str):
+        return open(os.path.join(self.dir, file), mode)
+
+    def action_dir(self, name: str, version: str):
+        return os.path.join(self.actions_dir, *name.lower().split("/"), version)
+
+    # Folder name and tag the server accepts for a local action
+    # (services/api/src/modules/runners/localActions.ts).
+    LOCAL_ACTION_NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
+    LOCAL_ACTION_TAG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
+    ACTION_FILES = (
+        "action-datatorch.yaml",
+        "action-datatorch.yml",
+        "action-datatorch.json",
+    )
+
+    @property
+    def local_actions_dir(self):
+        """Where a person keeps their own actions: `actions/local/<name>/<tag>`.
+
+        A pipeline step references one as `local/<name>@<tag>`; the folder
+        is run as-is and never fetched from anywhere.
+        """
+        return os.path.join(self.actions_dir, "local")
+
+    def local_actions(self) -> List[dict]:
+        """Every `actions/local/<name>/<tag>` folder holding an action file,
+        as `{name, tag, text}` with the action file's text, sorted. This is
+        what the runner reports to the server so the pipeline editor can list
+        and wire the action: the yaml only, never the code."""
+        found = []
+        root = self.local_actions_dir
+        if not os.path.isdir(root):
+            return found
+        for name in sorted(os.listdir(root)):
+            name_dir = os.path.join(root, name)
+            if not os.path.isdir(name_dir) or not self.LOCAL_ACTION_NAME.match(name):
+                continue
+            for tag in sorted(os.listdir(name_dir)):
+                tag_dir = os.path.join(name_dir, tag)
+                if not os.path.isdir(tag_dir) or not self.LOCAL_ACTION_TAG.match(tag):
+                    continue
+                for file in self.ACTION_FILES:
+                    path = os.path.join(tag_dir, file)
+                    if not os.path.isfile(path):
+                        continue
+                    try:
+                        with open(path, "r", encoding="utf-8") as handle:
+                            text = handle.read(MAX_LOCAL_ACTION_TEXT + 1)
+                    except OSError:
+                        continue
+                    # A .json file is read by the server's yaml parser as-is.
+                    found.append({"name": name, "tag": tag, "text": text})
+                    break
+        return found
+
+    # What the editor reads from and writes into a folder (the server caps
+    # the same way; this side is the one that matters).
+    MAX_LOCAL_ACTION_FILES = 40
+    MAX_LOCAL_ACTION_FILE_TEXT = 256 * 1024
+    SKIPPED_DIRS = ("__pycache__", "node_modules", ".git", "venv", ".venv")
+
+    def local_action_dir(self, name: str, tag: str) -> str:
+        """`actions/local/<name>/<tag>`, for a name and tag that are folder
+        names (anything else is refused, never joined)."""
+        if not self.LOCAL_ACTION_NAME.match(name or ""):
+            raise ValueError(f"'{name}' is not a local action folder name.")
+        if not self.LOCAL_ACTION_TAG.match(tag or ""):
+            raise ValueError(f"'{tag}' is not a tag folder name.")
+        return os.path.join(self.local_actions_dir, name, tag)
+
+    def _inside(self, folder: str, rel: str) -> str:
+        """The absolute path of `rel` inside `folder`, or a ValueError: no
+        absolute paths, no `..`, no hidden segments, no symlink escapes."""
+        rel = (rel or "").replace("\\", "/").strip()
+        if not rel or rel.startswith("/"):
+            raise ValueError(f"'{rel}': use a relative path.")
+        for part in rel.split("/"):
+            if part in ("", ".", "..") or part.startswith("."):
+                raise ValueError(f"'{rel}': not a path inside the action folder.")
+        root = os.path.realpath(folder)
+        full = os.path.realpath(os.path.join(root, *rel.split("/")))
+        if full != root and not full.startswith(root + os.sep):
+            raise ValueError(f"'{rel}': not a path inside the action folder.")
+        return full
+
+    def local_action_files(self, name: str, tag: str) -> List[dict]:
+        """The text files of one local action folder, as `{path, text, size}`,
+        sorted; binaries, hidden files, caches and big files are left out."""
+        folder = self.local_action_dir(name, tag)
+        if not os.path.isdir(folder):
+            raise FileNotFoundError(
+                f"No folder actions/local/{name}/{tag} on this runner."
+            )
+        found = []
+        for root, dirs, files in os.walk(folder):
+            dirs[:] = sorted(
+                d for d in dirs if not d.startswith(".") and d not in self.SKIPPED_DIRS
+            )
+            for file in sorted(files):
+                if file.startswith("."):
+                    continue
+                full = os.path.join(root, file)
+                if os.path.islink(full):
+                    continue
+                size = os.path.getsize(full)
+                if size > self.MAX_LOCAL_ACTION_FILE_TEXT:
+                    continue
+                try:
+                    with open(full, "r", encoding="utf-8") as handle:
+                        text = handle.read()
+                except (OSError, UnicodeDecodeError):
+                    continue
+                rel = os.path.relpath(full, folder).replace(os.sep, "/")
+                found.append({"path": rel, "text": text, "size": size})
+                if len(found) >= self.MAX_LOCAL_ACTION_FILES:
+                    return found
+        return found
+
+    def write_local_action_files(
+        self, name: str, tag: str, files: List[dict], delete: List[str] = None
+    ) -> List[dict]:
+        """Writes `files` (`{path, text}`) into, and deletes `delete` from, one
+        local action folder, creating the folder when it is new. Every path
+        is checked to be inside the folder before anything is touched."""
+        folder = self.local_action_dir(name, tag)
+        writes = [
+            (self._inside(folder, f.get("path")), f.get("text") or "")
+            for f in files or []
+        ]
+        removes = [self._inside(folder, p) for p in delete or []]
+        for _, text in writes:
+            if len(text) > self.MAX_LOCAL_ACTION_FILE_TEXT:
+                raise ValueError("A file is longer than the editor allows.")
+        mkdir_exists(folder)
+        for full, text in writes:
+            mkdir_exists(os.path.dirname(full))
+            with open(full, "w", encoding="utf-8") as handle:
+                handle.write(text)
+        for full in removes:
+            if os.path.isfile(full):
+                os.remove(full)
+        return self.local_action_files(name, tag)
+
+    def run_dir(self, task_id: str):
+        """Returns the directory for a given task"""
+        path = os.path.join(self.runs_dir, task_id)
+        mkdir_exists(path)
+        return path
+
+    def project_dir(self, project_id: str):
+        """Returns the directory for a given project"""
+        path = os.path.join(self.projects_dir, project_id)
+        mkdir_exists(path)
+        return path
+
+    @property
+    def temp_dir(self):
+        return os.path.join(self.dir, "temp")
+
+
+class RunnerSettings(Settings):
+    """The runner's own settings file. Keys are `runnerId` / `runnerToken`;
+    the pre-rename `agentId` / `agentToken` are still read, so an upgraded
+    runner keeps working until it is re-registered."""
+
+    def __init__(self):
+        super().__init__(RunnerDirectory.path())
+
+    @property
+    def runner_id(self):
+        return (
+            os.getenv(env.RUNNER_ID)
+            or self.get("runnerId", env=env.AGENT_ID)
+            or self.get("agentId")
+        )
+
+    @runner_id.setter
+    def runner_id(self, value):
+        self.set("runnerId", value)
+
+    @property
+    def runner_token(self):
+        return self.get("runnerToken") or self.get("agentToken")
+
+    @runner_token.setter
+    def runner_token(self, value):
+        self.set("runnerToken", value)
+
+    # Pre-rename names, kept for callers outside this package.
+    agent_id = runner_id
+    agent_token = runner_token
+
+    @property
+    def api_url(self):
+        return self.get("apiUrl", user_settings.api_url)
+
+    @api_url.setter
+    def api_url(self, value):
+        self.set("apiUrl", value)
+
+
+runner_directory = RunnerDirectory()
+
+# Pre-rename names, kept for callers outside this package.
+AgentDirectory = RunnerDirectory
+AgentSettings = RunnerSettings
+agent_directory = runner_directory

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch/runner/monitoring.py

@@ -2,10 +2,13 @@ import psutil
 import logging
 import platform
 import asyncio
+import hashlib
+import json
 
 from psutil._common import scpufreq
 from datetime import datetime, timezone
 from datatorch.utils.package import get_version
+from datatorch.runner.directory import runner_directory
 from collections import namedtuple
 
 logger = logging.getLogger(__name__)
@@ -78,12 +81,14 @@ class AgentSystemStats(object):
         self.sample_rate = sample_rate
         self.sample = 0
         self.report_failures = 0
+        self.local_actions_digest = None
 
     async def start(self):
         try:
             logger.info("Sending initial system metrics.")
             logger.info(self.initial_stats())
             await self.agent.api.initial_metrics(self.initial_stats())
+            await self._report_local_actions()
 
             logger.info("Starting system monitoring task.")
             await self._task_monitoring()
@@ -99,8 +104,37 @@ class AgentSystemStats(object):
             stats = self.stats()
             if self.sample != 1:
                 await self._report(stats)
+                # A folder dropped into actions/local shows up in the
+                # editor within a minute, no restart needed.
+                await self._report_local_actions()
             await asyncio.sleep(self.sample_rate)
 
+    async def _report_local_actions(self):
+        """Sends the actions/local folders when they changed since last sent.
+        A failure is logged and retried on the next sample; it never counts
+        toward the reconnect threshold (the metric report does that)."""
+        try:
+            actions = runner_directory.local_actions()
+        except Exception as ex:  # a broken folder must not stop the runner
+            logger.warning(f"Could not read local actions: {ex}")
+            return
+        digest = hashlib.sha256(
+            json.dumps(actions, sort_keys=True).encode("utf-8")
+        ).hexdigest()
+        if digest == self.local_actions_digest:
+            return
+        try:
+            await asyncio.wait_for(
+                self.agent.api.report_local_actions(actions),
+                timeout=self.REPORT_TIMEOUT,
+            )
+            self.local_actions_digest = digest
+            logger.info(
+                f"Reported {len(actions)} local action(s) from {runner_directory.local_actions_dir}."
+            )
+        except Exception as ex:
+            logger.warning(f"Could not report local actions: {ex}")
+
     async def _report(self, stats):
         """Sends one metric report; repeated failures force a reconnect."""
         try:

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch/runner/pipelines/action/__init__.py

@@ -27,6 +27,16 @@ async def get_action(config: ActionConfig, step: "Step" = None) -> Action:
         logger.debug(
             "Action found locally ({}@{}).".format(config.name, config.version)
         )
+    elif config.is_local:
+        # `local/<name>@<tag>` lives only in this runner's folder; there is
+        # nothing to download, so say where it was expected instead of
+        # failing a git clone of github.com/local/<name>.
+        raise FileNotFoundError(
+            f"Local action {config.full_name} is not on this runner: expected "
+            f"an action-datatorch.yaml in {action_dir}. Create one with "
+            f"`datatorch action create --local {config.name.split('/', 1)[1]} "
+            f"--tag {config.version}`, or pin the job to the runner that has it."
+        )
     else:
         logger.debug("Downloading action {}@{}.".format(config.name, config.version))
         await config.download()

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch/runner/pipelines/action/config.py

@@ -84,3 +84,11 @@ class ActionConfig(object):
     @property
     def full_name(self):
         return f"{self.name}@{self.version}"
+
+    # Reserved owner for a folder kept on the runner itself:
+    # `local/<name>@<tag>` → `<runner dir>/actions/local/<name>/<tag>`.
+    LOCAL_OWNER = "local"
+
+    @property
+    def is_local(self) -> bool:
+        return self.name.lower().startswith(self.LOCAL_OWNER + "/")

{datatorch-0.7.4 → datatorch-0.7.7}/datatorch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatorch
-Version: 0.7.4
+Version: 0.7.7
 Summary: A CLI and library for interacting with DataTorch.
 Home-page: https://github.com/datatorch/python
 Author: DataTorch

{datatorch-0.7.4 → datatorch-0.7.7}/setup.py

@@ -33,7 +33,7 @@ requirements = [
 
 setup(
     name="datatorch",
-    version="0.7.4",
+    version="0.7.7",
     description="A CLI and library for interacting with DataTorch.",
     author="DataTorch",
     author_email="support@datatorch.io",