datatorch 0.8.1__tar.gz → 0.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. {datatorch-0.8.1 → datatorch-0.8.2}/PKG-INFO +1 -1
  2. datatorch-0.8.2/datatorch/api/scripts/import_coco_server.py +154 -0
  3. datatorch-0.8.2/datatorch/cli/import_cmds/coco.py +110 -0
  4. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/template.py +81 -10
  5. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/PKG-INFO +1 -1
  6. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/SOURCES.txt +1 -0
  7. {datatorch-0.8.1 → datatorch-0.8.2}/setup.py +1 -1
  8. datatorch-0.8.1/datatorch/cli/import_cmds/coco.py +0 -59
  9. {datatorch-0.8.1 → datatorch-0.8.2}/README.md +0 -0
  10. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/__init__.py +0 -0
  11. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/agent/__init__.py +0 -0
  12. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/__init__.py +0 -0
  13. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/api.py +0 -0
  14. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/client.py +0 -0
  15. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/__init__.py +0 -0
  16. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/annotation.py +0 -0
  17. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/base.py +0 -0
  18. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/dataset.py +0 -0
  19. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/file.py +0 -0
  20. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/label.py +0 -0
  21. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/project.py +0 -0
  22. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/settings.py +0 -0
  23. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/__init__.py +0 -0
  24. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/__init__.py +0 -0
  25. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/bounding_box.py +0 -0
  26. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/segmentations.py +0 -0
  27. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/typings.py +0 -0
  28. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/source.py +0 -0
  29. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/storage_link.py +0 -0
  30. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/user.py +0 -0
  31. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/pull.py +0 -0
  32. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/__init__.py +0 -0
  33. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/import_coco.py +0 -0
  34. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/import_pixmask.py +0 -0
  35. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/utils/__init__.py +0 -0
  36. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/utils/simplify.py +0 -0
  37. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/utils.py +0 -0
  38. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/where.py +0 -0
  39. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/__init__.py +0 -0
  40. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/__init__.py +0 -0
  41. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/create.py +0 -0
  42. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/pull.py +0 -0
  43. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/run.py +0 -0
  44. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/groups.py +0 -0
  45. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/import_cmds/__init__.py +0 -0
  46. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/import_cmds/pixmask.py +0 -0
  47. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/__init__.py +0 -0
  48. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/login.py +0 -0
  49. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/logout.py +0 -0
  50. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/pull.py +0 -0
  51. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/upgrade.py +0 -0
  52. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/version.py +0 -0
  53. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/__init__.py +0 -0
  54. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/generate.py +0 -0
  55. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/run.py +0 -0
  56. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/upload.py +0 -0
  57. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/__init__.py +0 -0
  58. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/create.py +0 -0
  59. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/dir.py +0 -0
  60. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/start.py +0 -0
  61. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/spinner.py +0 -0
  62. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/upload/__init__.py +0 -0
  63. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/upload/folder.py +0 -0
  64. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/utils.py +0 -0
  65. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/__init__.py +0 -0
  66. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/env.py +0 -0
  67. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/folder.py +0 -0
  68. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/settings.py +0 -0
  69. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/__init__.py +0 -0
  70. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/client.py +0 -0
  71. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/daemon.py +0 -0
  72. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/directory.py +0 -0
  73. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/log_handler.py +0 -0
  74. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/logging.py +0 -0
  75. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/monitoring.py +0 -0
  76. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/__init__.py +0 -0
  77. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/__init__.py +0 -0
  78. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/action.py +0 -0
  79. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/cache.py +0 -0
  80. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/config.py +0 -0
  81. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/job/__init__.py +0 -0
  82. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/job/job.py +0 -0
  83. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/pipeline.py +0 -0
  84. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/resolver.py +0 -0
  85. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/__init__.py +0 -0
  86. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/command.py +0 -0
  87. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/docker.py +0 -0
  88. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/factory.py +0 -0
  89. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/node.py +0 -0
  90. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/python.py +0 -0
  91. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/runner.py +0 -0
  92. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/shell.py +0 -0
  93. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/step/__init__.py +0 -0
  94. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/step/step.py +0 -0
  95. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/services/__init__.py +0 -0
  96. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/__init__.py +0 -0
  97. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/docker.py +0 -0
  98. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/process.py +0 -0
  99. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/__init__.py +0 -0
  100. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/converters.py +0 -0
  101. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/files.py +0 -0
  102. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/hash_table.py +0 -0
  103. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/objects.py +0 -0
  104. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/package.py +0 -0
  105. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/string_style.py +0 -0
  106. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/url.py +0 -0
  107. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/dependency_links.txt +0 -0
  108. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/entry_points.txt +0 -0
  109. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/not-zip-safe +0 -0
  110. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/requires.txt +0 -0
  111. {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/top_level.txt +0 -0
  112. {datatorch-0.8.1 → datatorch-0.8.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datatorch
3
- Version: 0.8.1
3
+ Version: 0.8.2
4
4
  Summary: A CLI and library for interacting with DataTorch.
5
5
  Home-page: https://github.com/datatorch/python
6
6
  Author: DataTorch
@@ -0,0 +1,154 @@
1
+ """
2
+ COCO import run by the DataTorch server: the JSON is uploaded to the
3
+ project's default storage and the server's import job (createImport) maps
4
+ its images to dataset files by name, creates missing labels (with their
5
+ keypoint names and skeleton) and writes boxes, polygons and keypoints.
6
+ """
7
+
8
+ import os
9
+ import re
10
+ import time
11
+ import uuid
12
+ import logging
13
+ from typing import List, Optional
14
+
15
+ import requests
16
+
17
+ from .. import ApiClient, Dataset, Project, Where
18
+
19
+ _LOGGER = logging.getLogger(__name__)
20
+
21
+ _CREATE_IMPORT = """
22
+ mutation CreateImport($input: CreateImportInput!) {
23
+ createImport(input: $input) { id }
24
+ }
25
+ """
26
+
27
+ _GET_IMPORT = """
28
+ query GetImport($id: ID!) {
29
+ import(id: $id) {
30
+ annotationsCreated
31
+ labelsCreated
32
+ filesProcessed
33
+ job { state logs { level message } }
34
+ }
35
+ }
36
+ """
37
+
38
+ _DONE_STATES = ("SUCCESS", "FAILED", "CANCELED")
39
+
40
+
41
+ class CocoImportError(Exception):
42
+ pass
43
+
44
+
45
+ def _plain(message: str) -> str:
46
+ """Job log messages carry <b> tags for the web UI."""
47
+ return re.sub(r"<[^>]+>", "", message)
48
+
49
+
50
+ def _pick_dataset(project: Project, dataset_name: Optional[str]) -> Dataset:
51
+ if dataset_name:
52
+ return project.dataset(dataset_name)
53
+ datasets = project.datasets()
54
+ if len(datasets) == 1:
55
+ return datasets[0]
56
+ names = [d.name for d in datasets]
57
+ raise CocoImportError(
58
+ f"Project has {len(datasets)} datasets, pass one with --dataset: {names}"
59
+ )
60
+
61
+
62
+ def _upload(api: ApiClient, project: Project, file_path: str) -> str:
63
+ """
64
+ Uploads the COCO file as a project file and returns its id. The upload
65
+ route returns no id and renames on a path clash, so the file goes into a
66
+ fresh folder and is found again by its exact path.
67
+ """
68
+ # The server reads the file through whichever link holds it, so any link
69
+ # works: DataTorch Storage when the project has it, else the first one.
70
+ links = project.storage_links()
71
+ if not links:
72
+ raise CocoImportError("Project has no storage to upload the COCO file to")
73
+ link = next((l for l in links if l.name == "DataTorch Storage"), links[0])
74
+ folder = f"imports/{uuid.uuid4().hex}"
75
+ name = os.path.basename(file_path)
76
+ with open(file_path, "rb") as f:
77
+ r = requests.post(
78
+ f"{api.api_url}/file/v1/upload/{link.id}",
79
+ params={"path": folder, "import": "true"},
80
+ files={"file": (name, f, "application/json")},
81
+ headers={api.token_header: api._api_token},
82
+ )
83
+ if not r.ok:
84
+ raise CocoImportError(f"Upload failed ({r.status_code}): {r.text[:300]}")
85
+
86
+ files = project.files(Where(path=f"{folder}/{name}"), limit=2)
87
+ if len(files) != 1:
88
+ raise CocoImportError(f"Uploaded {name} but could not find it in the project")
89
+ return files[0].id
90
+
91
+
92
+ def import_coco_server(
93
+ file_path: str,
94
+ project_string: str,
95
+ dataset_name: Optional[str] = None,
96
+ api: ApiClient = None,
97
+ timeout: float = 3600,
98
+ poll_seconds: float = 2,
99
+ ) -> dict:
100
+ """
101
+ Imports a COCO file into a dataset on the server and waits for the job.
102
+ Returns the import's counts and its warning messages; raises
103
+ CocoImportError with the job's error messages if it fails.
104
+ """
105
+ if not os.path.isfile(file_path):
106
+ raise ValueError(f"Provided path '{file_path}' is not a file.")
107
+
108
+ if api is None:
109
+ api = ApiClient()
110
+
111
+ project: Project = api.project(project_string)
112
+ dataset = _pick_dataset(project, dataset_name)
113
+
114
+ _LOGGER.info("Uploading %s...", os.path.basename(file_path))
115
+ file_id = _upload(api, project, file_path)
116
+
117
+ import_id = api.execute(
118
+ _CREATE_IMPORT,
119
+ params={
120
+ "input": {"datasetId": dataset.id, "fileId": file_id, "format": "COCO"}
121
+ },
122
+ )["createImport"]["id"]
123
+ _LOGGER.info("Import %s queued for dataset '%s'.", import_id, dataset.name)
124
+
125
+ deadline = time.time() + timeout
126
+ while True:
127
+ imp = api.execute(_GET_IMPORT, params={"id": import_id})["import"]
128
+ state = imp["job"]["state"]
129
+ if state in _DONE_STATES:
130
+ break
131
+ if time.time() > deadline:
132
+ raise CocoImportError(
133
+ f"Timed out waiting for import {import_id} (last state {state})"
134
+ )
135
+ time.sleep(poll_seconds)
136
+
137
+ logs = imp["job"]["logs"] or []
138
+
139
+ def messages(level: str) -> List[str]:
140
+ return [_plain(log["message"]) for log in logs if log["level"] == level]
141
+
142
+ if state != "SUCCESS":
143
+ # The job logs one failure at several layers; the last is the cause.
144
+ errors = messages("error") or [f"Import job ended {state}"]
145
+ raise CocoImportError(errors[-1])
146
+
147
+ return {
148
+ "import_id": import_id,
149
+ "dataset": dataset.name,
150
+ "files_processed": imp["filesProcessed"],
151
+ "labels_created": imp["labelsCreated"],
152
+ "annotations_created": imp["annotationsCreated"],
153
+ "warnings": messages("warn"),
154
+ }
@@ -0,0 +1,110 @@
1
+ import click
2
+
3
+
4
+ @click.command(
5
+ "coco",
6
+ help="Import a COCO formated file. By default the DataTorch server runs the "
7
+ "import: it matches images to dataset files by name, creates missing labels "
8
+ "and imports boxes, polygons and keypoints.",
9
+ )
10
+ @click.option(
11
+ "-f",
12
+ "--file",
13
+ type=click.Path(exists=True),
14
+ prompt=True,
15
+ help="Path to the json file containing coco annotations.",
16
+ )
17
+ @click.option(
18
+ "-p",
19
+ "--project",
20
+ type=str,
21
+ prompt=True,
22
+ help="Namespace identifier or UUID of project (e.g. username/project)",
23
+ )
24
+ @click.option(
25
+ "-d",
26
+ "--dataset",
27
+ type=str,
28
+ default=None,
29
+ help="Name of the dataset to import into. Defaults to the project's only dataset.",
30
+ )
31
+ @click.option(
32
+ "--client-side",
33
+ is_flag=True,
34
+ default=False,
35
+ help="Import from this machine with pycocotools instead of on the server. "
36
+ "Labels must already exist, keypoints are not imported, and the options "
37
+ "below apply only in this mode.",
38
+ )
39
+ @click.option(
40
+ "-m",
41
+ "--max-iou",
42
+ type=float,
43
+ default=0.99,
44
+ show_default=True,
45
+ help="(--client-side) If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
46
+ )
47
+ @click.option(
48
+ "-s",
49
+ "--simplify",
50
+ type=float,
51
+ default=1.0,
52
+ show_default=True,
53
+ help="(--client-side) Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
54
+ )
55
+ @click.option(
56
+ "--no-segmentations",
57
+ type=bool,
58
+ flag_value=False,
59
+ default=True,
60
+ help="(--client-side) Flag to disable the import of coco segmentations.",
61
+ )
62
+ @click.option(
63
+ "--import-bbox",
64
+ type=bool,
65
+ flag_value=True,
66
+ default=False,
67
+ help="(--client-side) Flag to enable to import of coco bounding boxes.",
68
+ )
69
+ def coco_cmd(
70
+ file,
71
+ project,
72
+ dataset,
73
+ client_side,
74
+ max_iou,
75
+ no_segmentations,
76
+ import_bbox,
77
+ simplify,
78
+ ):
79
+ if client_side:
80
+ from datatorch.api.scripts.import_coco import import_coco
81
+
82
+ import_coco(
83
+ file,
84
+ project,
85
+ import_bbox=import_bbox,
86
+ import_segmentation=no_segmentations,
87
+ max_iou=max_iou,
88
+ simplify_tolerance=simplify,
89
+ )
90
+ return
91
+
92
+ from datatorch.api.scripts.import_coco_server import (
93
+ CocoImportError,
94
+ import_coco_server,
95
+ )
96
+
97
+ click.echo("Uploading and importing on the server...")
98
+ try:
99
+ result = import_coco_server(file, project, dataset_name=dataset)
100
+ except CocoImportError as e:
101
+ raise click.ClickException(f"COCO import failed:\n{e}")
102
+
103
+ for warning in result["warnings"]:
104
+ click.secho(f"warning: {warning}", fg="yellow")
105
+ click.echo(
106
+ f"Imported into dataset '{result['dataset']}': "
107
+ f"{result['annotations_created']} annotations on "
108
+ f"{result['files_processed']} files, "
109
+ f"{result['labels_created']} new labels."
110
+ )
@@ -2,8 +2,10 @@ import json
2
2
  import platform
3
3
  import re
4
4
  import typing
5
- from typing import Dict
6
- from jinja2 import Template
5
+ import uuid
6
+ from typing import Dict, List, Tuple
7
+ from jinja2 import Environment, Template, meta
8
+ from jinja2.exceptions import TemplateSyntaxError
7
9
  from ..directory import runner_directory
8
10
 
9
11
  if typing.TYPE_CHECKING:
@@ -19,6 +21,17 @@ if typing.TYPE_CHECKING:
19
21
  # $INPUT_<NAME> env vars instead.
20
22
  _INPUT_REF_IN_TEMPLATE = re.compile(r"\$\{[{%][^}]*?(?<![.\w])(?:input|variable)(?!\w)")
21
23
 
24
+ # One `${{ ... }}` expression, as written. Used to find the expressions in
25
+ # a string before rendering so that the ones the runner does not own can
26
+ # be kept exactly as they are (see `Variables.render`).
27
+ _EXPRESSION = re.compile(r"\$\{\{.*?\}\}", re.DOTALL)
28
+
29
+ _environment = Environment(
30
+ block_start_string="${%",
31
+ variable_start_string="${{",
32
+ comment_start_string="${#",
33
+ )
34
+
22
35
 
23
36
  class InputInjectionError(Exception):
24
37
  """Raised when a shell command/script interpolates ${{ input.* }} /
@@ -72,6 +85,17 @@ global_variables = {
72
85
  },
73
86
  }
74
87
 
88
+ # The namespaces the runner renders (`run` since 0.8.1: the step's place
89
+ # in the record). `${{ ... }}` text rooted anywhere else (`steps.*`,
90
+ # `inputs.*`, `jobs.*`, `secrets.*`, unknown names) is not a reference
91
+ # the runner can resolve: the server resolves its own namespaces before
92
+ # dispatch, and whatever still reads like one in a step input is DATA (a
93
+ # log line, a fetched body, an older error message) that must reach the
94
+ # action untouched.
95
+ RUNNER_NAMESPACES = frozenset(
96
+ {*global_variables.keys(), "input", "variable", "step", "action", "run"}
97
+ )
98
+
75
99
 
76
100
  class Variables(object):
77
101
  """
@@ -92,6 +116,12 @@ class Variables(object):
92
116
  for use inside the action's own templates (``runs.command`` etc.).
93
117
  ``variable`` is a legacy alias of ``input``; both are scoped to the
94
118
  one step. The old job-wide accumulating namespace is gone.
119
+
120
+ Everything else is inert. A ``${{ ... }}`` whose root names are not all
121
+ in ``RUNNER_NAMESPACES`` (or that does not parse) is left exactly as
122
+ written, never raising and never rewritten: the runner does not author
123
+ that text, so it must not interpret it. Only string inputs are rendered;
124
+ objects and arrays pass through as they arrive.
95
125
  """
96
126
 
97
127
  def __init__(self):
@@ -140,8 +170,54 @@ class Variables(object):
140
170
  self.variables[section] = {**self.variables[section], **variables}
141
171
 
142
172
  def render(self, string: str):
143
- tp = self._template(string)
144
- return tp.render({**global_variables, **self.variables})
173
+ """Render the runner's own namespaces in ``string``; keep every
174
+ other ``${{ ... }}`` exactly as written (see the class docstring).
175
+ """
176
+ if string is None or "${" not in string:
177
+ return string
178
+ protected, kept = self._protect_foreign_expressions(string)
179
+ try:
180
+ rendered = self._template(protected).render(
181
+ {**global_variables, **self.variables}
182
+ )
183
+ except TemplateSyntaxError:
184
+ # Not a template the runner can read (an unclosed `${{`, a
185
+ # stray `${%` in data): it is text, hand it back as it came.
186
+ return string
187
+ for token, text in kept:
188
+ rendered = rendered.replace(token, text)
189
+ return rendered
190
+
191
+ @classmethod
192
+ def _protect_foreign_expressions(
193
+ cls, string: str
194
+ ) -> Tuple[str, List[Tuple[str, str]]]:
195
+ """Swap each ``${{ ... }}`` the runner does not own for an opaque
196
+ token so Jinja never sees it; the caller restores them after
197
+ rendering. Returns the protected string and the (token, original
198
+ text) pairs."""
199
+ nonce = uuid.uuid4().hex
200
+ kept: List[Tuple[str, str]] = []
201
+
202
+ def swap(match: "re.Match[str]") -> str:
203
+ text = match.group(0)
204
+ if cls._owns(text):
205
+ return text
206
+ token = f"\x00{nonce}:{len(kept)}\x00"
207
+ kept.append((token, text))
208
+ return token
209
+
210
+ return _EXPRESSION.sub(swap, string), kept
211
+
212
+ @staticmethod
213
+ def _owns(expression: str) -> bool:
214
+ """True when ``expression`` (one ``${{ ... }}``) parses and every
215
+ name it reads is one of the runner's namespaces."""
216
+ try:
217
+ ast = _environment.parse(expression)
218
+ except TemplateSyntaxError:
219
+ return False
220
+ return meta.find_undeclared_variables(ast) <= RUNNER_NAMESPACES
145
221
 
146
222
  def render_command(self, string: str):
147
223
  """Render a shell command/script template.
@@ -177,12 +253,7 @@ class Variables(object):
177
253
 
178
254
  @staticmethod
179
255
  def _template(string: str) -> Template:
180
- return Template(
181
- string,
182
- block_start_string="${%",
183
- variable_start_string="${{",
184
- comment_start_string="${#",
185
- )
256
+ return _environment.from_string(string)
186
257
 
187
258
  @property
188
259
  def inputs(self) -> dict:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datatorch
3
- Version: 0.8.1
3
+ Version: 0.8.2
4
4
  Summary: A CLI and library for interacting with DataTorch.
5
5
  Home-page: https://github.com/datatorch/python
6
6
  Author: DataTorch
@@ -33,6 +33,7 @@ datatorch/api/entity/sources/image/segmentations.py
33
33
  datatorch/api/entity/sources/image/typings.py
34
34
  datatorch/api/scripts/__init__.py
35
35
  datatorch/api/scripts/import_coco.py
36
+ datatorch/api/scripts/import_coco_server.py
36
37
  datatorch/api/scripts/import_pixmask.py
37
38
  datatorch/api/scripts/utils/__init__.py
38
39
  datatorch/api/scripts/utils/simplify.py
@@ -33,7 +33,7 @@ requirements = [
33
33
 
34
34
  setup(
35
35
  name="datatorch",
36
- version="0.8.1",
36
+ version="0.8.2",
37
37
  description="A CLI and library for interacting with DataTorch.",
38
38
  author="DataTorch",
39
39
  author_email="support@datatorch.io",
@@ -1,59 +0,0 @@
1
- import click
2
-
3
-
4
- @click.command("coco", help="Import a COCO formated file.")
5
- @click.option(
6
- "-f",
7
- "--file",
8
- type=click.Path(exists=True),
9
- prompt=True,
10
- help="Path to the json file containing coco annotations.",
11
- )
12
- @click.option(
13
- "-p",
14
- "--project",
15
- type=str,
16
- prompt=True,
17
- help="Namespace identifier or UUID of project (e.g. username/project)",
18
- )
19
- @click.option(
20
- "-m",
21
- "--max-iou",
22
- type=float,
23
- default=0.99,
24
- show_default=True,
25
- help="If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
26
- )
27
- @click.option(
28
- "-s",
29
- "--simplify",
30
- type=float,
31
- default=1.0,
32
- show_default=True,
33
- help="Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
34
- )
35
- @click.option(
36
- "--no-segmentations",
37
- type=bool,
38
- flag_value=False,
39
- default=True,
40
- help="Flag to disable the import of coco segmentations.",
41
- )
42
- @click.option(
43
- "--import-bbox",
44
- type=bool,
45
- flag_value=True,
46
- default=False,
47
- help="Flag to enable to import of coco bounding boxes.",
48
- )
49
- def coco_cmd(file, project, max_iou, no_segmentations, import_bbox, simplify):
50
- from datatorch.api.scripts.import_coco import import_coco
51
-
52
- import_coco(
53
- file,
54
- project,
55
- import_bbox=import_bbox,
56
- import_segmentation=no_segmentations,
57
- max_iou=max_iou,
58
- simplify_tolerance=simplify,
59
- )
File without changes
File without changes