datatorch 0.8.0__tar.gz → 0.8.2__tar.gz

{datatorch-0.8.0 → datatorch-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatorch
-Version: 0.8.0
+Version: 0.8.2
 Summary: A CLI and library for interacting with DataTorch.
 Home-page: https://github.com/datatorch/python
 Author: DataTorch

datatorch-0.8.2/datatorch/api/scripts/import_coco_server.py

@@ -0,0 +1,154 @@
+"""
+COCO import run by the DataTorch server: the JSON is uploaded to the
+project's default storage and the server's import job (createImport) maps
+its images to dataset files by name, creates missing labels (with their
+keypoint names and skeleton) and writes boxes, polygons and keypoints.
+"""
+
+import os
+import re
+import time
+import uuid
+import logging
+from typing import List, Optional
+
+import requests
+
+from .. import ApiClient, Dataset, Project, Where
+
+_LOGGER = logging.getLogger(__name__)
+
+_CREATE_IMPORT = """
+    mutation CreateImport($input: CreateImportInput!) {
+        createImport(input: $input) { id }
+    }
+"""
+
+_GET_IMPORT = """
+    query GetImport($id: ID!) {
+        import(id: $id) {
+            annotationsCreated
+            labelsCreated
+            filesProcessed
+            job { state logs { level message } }
+        }
+    }
+"""
+
+_DONE_STATES = ("SUCCESS", "FAILED", "CANCELED")
+
+
+class CocoImportError(Exception):
+    pass
+
+
+def _plain(message: str) -> str:
+    """Job log messages carry <b> tags for the web UI."""
+    return re.sub(r"<[^>]+>", "", message)
+
+
+def _pick_dataset(project: Project, dataset_name: Optional[str]) -> Dataset:
+    if dataset_name:
+        return project.dataset(dataset_name)
+    datasets = project.datasets()
+    if len(datasets) == 1:
+        return datasets[0]
+    names = [d.name for d in datasets]
+    raise CocoImportError(
+        f"Project has {len(datasets)} datasets, pass one with --dataset: {names}"
+    )
+
+
+def _upload(api: ApiClient, project: Project, file_path: str) -> str:
+    """
+    Uploads the COCO file as a project file and returns its id. The upload
+    route returns no id and renames on a path clash, so the file goes into a
+    fresh folder and is found again by its exact path.
+    """
+    # The server reads the file through whichever link holds it, so any link
+    # works: DataTorch Storage when the project has it, else the first one.
+    links = project.storage_links()
+    if not links:
+        raise CocoImportError("Project has no storage to upload the COCO file to")
+    link = next((l for l in links if l.name == "DataTorch Storage"), links[0])
+    folder = f"imports/{uuid.uuid4().hex}"
+    name = os.path.basename(file_path)
+    with open(file_path, "rb") as f:
+        r = requests.post(
+            f"{api.api_url}/file/v1/upload/{link.id}",
+            params={"path": folder, "import": "true"},
+            files={"file": (name, f, "application/json")},
+            headers={api.token_header: api._api_token},
+        )
+    if not r.ok:
+        raise CocoImportError(f"Upload failed ({r.status_code}): {r.text[:300]}")
+
+    files = project.files(Where(path=f"{folder}/{name}"), limit=2)
+    if len(files) != 1:
+        raise CocoImportError(f"Uploaded {name} but could not find it in the project")
+    return files[0].id
+
+
+def import_coco_server(
+    file_path: str,
+    project_string: str,
+    dataset_name: Optional[str] = None,
+    api: ApiClient = None,
+    timeout: float = 3600,
+    poll_seconds: float = 2,
+) -> dict:
+    """
+    Imports a COCO file into a dataset on the server and waits for the job.
+    Returns the import's counts and its warning messages; raises
+    CocoImportError with the job's error messages if it fails.
+    """
+    if not os.path.isfile(file_path):
+        raise ValueError(f"Provided path '{file_path}' is not a file.")
+
+    if api is None:
+        api = ApiClient()
+
+    project: Project = api.project(project_string)
+    dataset = _pick_dataset(project, dataset_name)
+
+    _LOGGER.info("Uploading %s...", os.path.basename(file_path))
+    file_id = _upload(api, project, file_path)
+
+    import_id = api.execute(
+        _CREATE_IMPORT,
+        params={
+            "input": {"datasetId": dataset.id, "fileId": file_id, "format": "COCO"}
+        },
+    )["createImport"]["id"]
+    _LOGGER.info("Import %s queued for dataset '%s'.", import_id, dataset.name)
+
+    deadline = time.time() + timeout
+    while True:
+        imp = api.execute(_GET_IMPORT, params={"id": import_id})["import"]
+        state = imp["job"]["state"]
+        if state in _DONE_STATES:
+            break
+        if time.time() > deadline:
+            raise CocoImportError(
+                f"Timed out waiting for import {import_id} (last state {state})"
+            )
+        time.sleep(poll_seconds)
+
+    logs = imp["job"]["logs"] or []
+
+    def messages(level: str) -> List[str]:
+        return [_plain(log["message"]) for log in logs if log["level"] == level]
+
+    if state != "SUCCESS":
+        # The job logs one failure at several layers; the last is the cause.
+        errors = messages("error") or [f"Import job ended {state}"]
+        raise CocoImportError(errors[-1])
+
+    return {
+        "import_id": import_id,
+        "dataset": dataset.name,
+        "files_processed": imp["filesProcessed"],
+        "labels_created": imp["labelsCreated"],
+        "annotations_created": imp["annotationsCreated"],
+        "warnings": messages("warn"),
+    }

datatorch-0.8.2/datatorch/cli/import_cmds/coco.py

@@ -0,0 +1,110 @@
+import click
+
+
+@click.command(
+    "coco",
+    help="Import a COCO formated file. By default the DataTorch server runs the "
+    "import: it matches images to dataset files by name, creates missing labels "
+    "and imports boxes, polygons and keypoints.",
+)
+@click.option(
+    "-f",
+    "--file",
+    type=click.Path(exists=True),
+    prompt=True,
+    help="Path to the json file containing coco annotations.",
+)
+@click.option(
+    "-p",
+    "--project",
+    type=str,
+    prompt=True,
+    help="Namespace identifier or UUID of project (e.g. username/project)",
+)
+@click.option(
+    "-d",
+    "--dataset",
+    type=str,
+    default=None,
+    help="Name of the dataset to import into. Defaults to the project's only dataset.",
+)
+@click.option(
+    "--client-side",
+    is_flag=True,
+    default=False,
+    help="Import from this machine with pycocotools instead of on the server. "
+    "Labels must already exist, keypoints are not imported, and the options "
+    "below apply only in this mode.",
+)
+@click.option(
+    "-m",
+    "--max-iou",
+    type=float,
+    default=0.99,
+    show_default=True,
+    help="(--client-side) If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
+)
+@click.option(
+    "-s",
+    "--simplify",
+    type=float,
+    default=1.0,
+    show_default=True,
+    help="(--client-side) Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
+)
+@click.option(
+    "--no-segmentations",
+    type=bool,
+    flag_value=False,
+    default=True,
+    help="(--client-side) Flag to disable the import of coco segmentations.",
+)
+@click.option(
+    "--import-bbox",
+    type=bool,
+    flag_value=True,
+    default=False,
+    help="(--client-side) Flag to enable to import of coco bounding boxes.",
+)
+def coco_cmd(
+    file,
+    project,
+    dataset,
+    client_side,
+    max_iou,
+    no_segmentations,
+    import_bbox,
+    simplify,
+):
+    if client_side:
+        from datatorch.api.scripts.import_coco import import_coco
+
+        import_coco(
+            file,
+            project,
+            import_bbox=import_bbox,
+            import_segmentation=no_segmentations,
+            max_iou=max_iou,
+            simplify_tolerance=simplify,
+        )
+        return
+
+    from datatorch.api.scripts.import_coco_server import (
+        CocoImportError,
+        import_coco_server,
+    )
+
+    click.echo("Uploading and importing on the server...")
+    try:
+        result = import_coco_server(file, project, dataset_name=dataset)
+    except CocoImportError as e:
+        raise click.ClickException(f"COCO import failed:\n{e}")
+
+    for warning in result["warnings"]:
+        click.secho(f"warning: {warning}", fg="yellow")
+    click.echo(
+        f"Imported into dataset '{result['dataset']}': "
+        f"{result['annotations_created']} annotations on "
+        f"{result['files_processed']} files, "
+        f"{result['labels_created']} new labels."
+    )

datatorch-0.8.2/datatorch/core/env.py

@@ -0,0 +1,18 @@
+# Edit this to set custom variables
+RUNNER_ID = "DATATORCH_RUNNER_ID"
+# The pre-October-2026 spelling, still honored.
+AGENT_ID = "DATATORCH_AGENT_ID"
+API_KEY = "DATATORCH_API_KEY"
+# A runner hands its own token to the actions it runs (October 2026).
+RUNNER_TOKEN = "DATATORCH_RUNNER_TOKEN"
+API_URL = "DATATORCH_API_URL"
+CONFIG_DIR = "DATATORCH_DIR"
+# The step's place in the record, handed to every action a runner runs
+# (0.8.1): the project the run belongs to, the run, the pipeline and the
+# step. An action that works "for this project" reads PROJECT_ID instead
+# of carrying a project input, so a pipeline forked into another project
+# works there unchanged.
+PROJECT_ID = "DATATORCH_PROJECT_ID"
+RUN_ID = "DATATORCH_RUN_ID"
+PIPELINE_ID = "DATATORCH_PIPELINE_ID"
+STEP_ID = "DATATORCH_STEP_ID"

{datatorch-0.8.0 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/runner.py

@@ -91,7 +91,9 @@ class Runner(object):
     def runner_env(self) -> dict:
         """What an action needs to reach the server this runner is on:
         the API url and the runner's own token (as DATATORCH_API_URL and
-        DATATORCH_RUNNER_TOKEN). The SDK's client picks them up, so an
+        DATATORCH_RUNNER_TOKEN), and where the step sits in the record
+        (DATATORCH_PROJECT_ID, DATATORCH_RUN_ID, DATATORCH_PIPELINE_ID,
+        DATATORCH_STEP_ID). The SDK's client picks them up, so an
         action talks to the same server as the runner, as the runner,
         instead of the machine's stored login. Anything already set in
         the environment wins (DATATORCH_API_KEY included, since the SDK
@@ -105,6 +107,21 @@ class Runner(object):
             values[env_names.API_URL] = settings.api_url
         if settings.runner_token and not os.environ.get(env_names.RUNNER_TOKEN):
             values[env_names.RUNNER_TOKEN] = settings.runner_token
+        # The step's place in the record (0.8.1), so an action can work
+        # for "this project" without a project input. Always set by the
+        # runner, never inherited: a value left in the environment by an
+        # outer process must not describe another run.
+        sections = getattr(getattr(self, "variables", None), "variables", None) or {}
+        run = sections.get("run") or {}
+        step = sections.get("step") or {}
+        for name, value in (
+            (env_names.PROJECT_ID, run.get("projectId")),
+            (env_names.RUN_ID, run.get("id")),
+            (env_names.PIPELINE_ID, run.get("pipelineId")),
+            (env_names.STEP_ID, step.get("id")),
+        ):
+            if value:
+                values[name] = str(value)
         return values
 
     def input_env(self) -> dict:

{datatorch-0.8.0 → datatorch-0.8.2}/datatorch/runner/pipelines/step/step.py

@@ -32,6 +32,10 @@ class Step(object):
             name=dispatch.get("stepName") or "",
             inputs=dispatch.get("input") or {},
             run_id=dispatch.get("runId"),
+            run_number=dispatch.get("runNumber"),
+            project_id=dispatch.get("projectId"),
+            pipeline_id=dispatch.get("pipelineId"),
+            job_id=dispatch.get("jobId"),
             api=api,
         )
 
@@ -54,6 +58,10 @@ class Step(object):
         cacheable: Union[bool, None] = None,
         inputs: dict = {},
         run_id: str = None,
+        run_number: int = None,
+        project_id: str = None,
+        pipeline_id: str = None,
+        job_id: str = None,
         api: "AgentApiClient" = None,
     ):
         self._action = ActionConfig(action)
@@ -63,6 +71,10 @@ class Step(object):
         self.inputs = inputs
         self.cacheable = cacheable
         self.run_id = run_id
+        self.run_number = run_number
+        self.project_id = project_id
+        self.pipeline_id = pipeline_id
+        self.job_id = job_id
         self.api = api
         self.rendered_inputs: Optional[dict] = None
         self.logger = logging.getLogger(f"datatorch.runner.[{self._action.full_name}]")

{datatorch-0.8.0 → datatorch-0.8.2}/datatorch/runner/pipelines/template.py

@@ -2,8 +2,10 @@ import json
 import platform
 import re
 import typing
-from typing import Dict
-from jinja2 import Template
+import uuid
+from typing import Dict, List, Tuple
+from jinja2 import Environment, Template, meta
+from jinja2.exceptions import TemplateSyntaxError
 from ..directory import runner_directory
 
 if typing.TYPE_CHECKING:
@@ -19,6 +21,17 @@ if typing.TYPE_CHECKING:
 # $INPUT_<NAME> env vars instead.
 _INPUT_REF_IN_TEMPLATE = re.compile(r"\$\{[{%][^}]*?(?<![.\w])(?:input|variable)(?!\w)")
 
+# One `${{ ... }}` expression, as written. Used to find the expressions in
+# a string before rendering so that the ones the runner does not own can
+# be kept exactly as they are (see `Variables.render`).
+_EXPRESSION = re.compile(r"\$\{\{.*?\}\}", re.DOTALL)
+
+_environment = Environment(
+    block_start_string="${%",
+    variable_start_string="${{",
+    comment_start_string="${#",
+)
+
 
 class InputInjectionError(Exception):
     """Raised when a shell command/script interpolates ${{ input.* }} /
@@ -72,6 +85,17 @@ global_variables = {
     },
 }
 
+# The namespaces the runner renders (`run` since 0.8.1: the step's place
+# in the record). `${{ ... }}` text rooted anywhere else (`steps.*`,
+# `inputs.*`, `jobs.*`, `secrets.*`, unknown names) is not a reference
+# the runner can resolve: the server resolves its own namespaces before
+# dispatch, and whatever still reads like one in a step input is DATA (a
+# log line, a fetched body, an older error message) that must reach the
+# action untouched.
+RUNNER_NAMESPACES = frozenset(
+    {*global_variables.keys(), "input", "variable", "step", "action", "run"}
+)
+
 
 class Variables(object):
     """
@@ -86,10 +110,18 @@ class Variables(object):
 
     - ``agent``, ``directory``, ``machine``, ``python`` — machine facts
     - ``step``, ``action`` — the step being executed
+    - ``run`` — the run the step belongs to (id, number, projectId,
+      pipelineId, jobId); empty values in local mode
     - ``input`` / ``variable`` — the step's own (already-resolved) inputs,
       for use inside the action's own templates (``runs.command`` etc.).
       ``variable`` is a legacy alias of ``input``; both are scoped to the
       one step. The old job-wide accumulating namespace is gone.
+
+    Everything else is inert. A ``${{ ... }}`` whose root names are not all
+    in ``RUNNER_NAMESPACES`` (or that does not parse) is left exactly as
+    written, never raising and never rewritten: the runner does not author
+    that text, so it must not interpret it. Only string inputs are rendered;
+    objects and arrays pass through as they arrive.
     """
 
     def __init__(self):
@@ -99,6 +131,19 @@ class Variables(object):
 
     def set_step(self, step: "Step"):
         self.set("step", {"id": step.id, "name": step.name})
+        # Where the step sits in the record (server mode; empty when run
+        # locally). `${{ run.projectId }}` in an action's own templates, and
+        # DATATORCH_PROJECT_ID / DATATORCH_RUN_ID in its environment.
+        self.set(
+            "run",
+            {
+                "id": getattr(step, "run_id", None),
+                "number": getattr(step, "run_number", None),
+                "projectId": getattr(step, "project_id", None),
+                "pipelineId": getattr(step, "pipeline_id", None),
+                "jobId": getattr(step, "job_id", None),
+            },
+        )
 
     def set_action(self, action: "Action"):
         self.set(
@@ -125,8 +170,54 @@ class Variables(object):
         self.variables[section] = {**self.variables[section], **variables}
 
     def render(self, string: str):
-        tp = self._template(string)
-        return tp.render({**global_variables, **self.variables})
+        """Render the runner's own namespaces in ``string``; keep every
+        other ``${{ ... }}`` exactly as written (see the class docstring).
+        """
+        if string is None or "${" not in string:
+            return string
+        protected, kept = self._protect_foreign_expressions(string)
+        try:
+            rendered = self._template(protected).render(
+                {**global_variables, **self.variables}
+            )
+        except TemplateSyntaxError:
+            # Not a template the runner can read (an unclosed `${{`, a
+            # stray `${%` in data): it is text, hand it back as it came.
+            return string
+        for token, text in kept:
+            rendered = rendered.replace(token, text)
+        return rendered
+
+    @classmethod
+    def _protect_foreign_expressions(
+        cls, string: str
+    ) -> Tuple[str, List[Tuple[str, str]]]:
+        """Swap each ``${{ ... }}`` the runner does not own for an opaque
+        token so Jinja never sees it; the caller restores them after
+        rendering. Returns the protected string and the (token, original
+        text) pairs."""
+        nonce = uuid.uuid4().hex
+        kept: List[Tuple[str, str]] = []
+
+        def swap(match: "re.Match[str]") -> str:
+            text = match.group(0)
+            if cls._owns(text):
+                return text
+            token = f"\x00{nonce}:{len(kept)}\x00"
+            kept.append((token, text))
+            return token
+
+        return _EXPRESSION.sub(swap, string), kept
+
+    @staticmethod
+    def _owns(expression: str) -> bool:
+        """True when ``expression`` (one ``${{ ... }}``) parses and every
+        name it reads is one of the runner's namespaces."""
+        try:
+            ast = _environment.parse(expression)
+        except TemplateSyntaxError:
+            return False
+        return meta.find_undeclared_variables(ast) <= RUNNER_NAMESPACES
 
     def render_command(self, string: str):
         """Render a shell command/script template.
@@ -162,12 +253,7 @@ class Variables(object):
 
     @staticmethod
     def _template(string: str) -> Template:
-        return Template(
-            string,
-            block_start_string="${%",
-            variable_start_string="${{",
-            comment_start_string="${#",
-        )
+        return _environment.from_string(string)
 
     @property
     def inputs(self) -> dict:

{datatorch-0.8.0 → datatorch-0.8.2}/datatorch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatorch
-Version: 0.8.0
+Version: 0.8.2
 Summary: A CLI and library for interacting with DataTorch.
 Home-page: https://github.com/datatorch/python
 Author: DataTorch

{datatorch-0.8.0 → datatorch-0.8.2}/datatorch.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ datatorch/api/entity/sources/image/segmentations.py
 datatorch/api/entity/sources/image/typings.py
 datatorch/api/scripts/__init__.py
 datatorch/api/scripts/import_coco.py
+datatorch/api/scripts/import_coco_server.py
 datatorch/api/scripts/import_pixmask.py
 datatorch/api/scripts/utils/__init__.py
 datatorch/api/scripts/utils/simplify.py

{datatorch-0.8.0 → datatorch-0.8.2}/setup.py

@@ -33,7 +33,7 @@ requirements = [
 
 setup(
     name="datatorch",
-    version="0.8.0",
+    version="0.8.2",
     description="A CLI and library for interacting with DataTorch.",
     author="DataTorch",
     author_email="support@datatorch.io",

datatorch-0.8.0/datatorch/cli/import_cmds/coco.py

@@ -1,59 +0,0 @@
-import click
-
-
-@click.command("coco", help="Import a COCO formated file.")
-@click.option(
-    "-f",
-    "--file",
-    type=click.Path(exists=True),
-    prompt=True,
-    help="Path to the json file containing coco annotations.",
-)
-@click.option(
-    "-p",
-    "--project",
-    type=str,
-    prompt=True,
-    help="Namespace identifier or UUID of project (e.g. username/project)",
-)
-@click.option(
-    "-m",
-    "--max-iou",
-    type=float,
-    default=0.99,
-    show_default=True,
-    help="If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
-)
-@click.option(
-    "-s",
-    "--simplify",
-    type=float,
-    default=1.0,
-    show_default=True,
-    help="Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
-)
-@click.option(
-    "--no-segmentations",
-    type=bool,
-    flag_value=False,
-    default=True,
-    help="Flag to disable the import of coco segmentations.",
-)
-@click.option(
-    "--import-bbox",
-    type=bool,
-    flag_value=True,
-    default=False,
-    help="Flag to enable to import of coco bounding boxes.",
-)
-def coco_cmd(file, project, max_iou, no_segmentations, import_bbox, simplify):
-    from datatorch.api.scripts.import_coco import import_coco
-
-    import_coco(
-        file,
-        project,
-        import_bbox=import_bbox,
-        import_segmentation=no_segmentations,
-        max_iou=max_iou,
-        simplify_tolerance=simplify,
-    )

datatorch-0.8.0/datatorch/core/env.py

@@ -1,9 +0,0 @@
-# Edit this to set custom variables
-RUNNER_ID = "DATATORCH_RUNNER_ID"
-# The pre-October-2026 spelling, still honored.
-AGENT_ID = "DATATORCH_AGENT_ID"
-API_KEY = "DATATORCH_API_KEY"
-# A runner hands its own token to the actions it runs (October 2026).
-RUNNER_TOKEN = "DATATORCH_RUNNER_TOKEN"
-API_URL = "DATATORCH_API_URL"
-CONFIG_DIR = "DATATORCH_DIR"