datatorch 0.8.1__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datatorch-0.8.1 → datatorch-0.8.2}/PKG-INFO +1 -1
- datatorch-0.8.2/datatorch/api/scripts/import_coco_server.py +154 -0
- datatorch-0.8.2/datatorch/cli/import_cmds/coco.py +110 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/template.py +81 -10
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/PKG-INFO +1 -1
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/SOURCES.txt +1 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/setup.py +1 -1
- datatorch-0.8.1/datatorch/cli/import_cmds/coco.py +0 -59
- {datatorch-0.8.1 → datatorch-0.8.2}/README.md +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/agent/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/api.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/client.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/annotation.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/base.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/dataset.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/file.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/label.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/project.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/settings.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/bounding_box.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/segmentations.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/image/typings.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/sources/source.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/storage_link.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/entity/user.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/pull.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/import_coco.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/import_pixmask.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/utils/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/scripts/utils/simplify.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/utils.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/api/where.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/create.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/pull.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/action/run.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/groups.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/import_cmds/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/import_cmds/pixmask.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/login.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/logout.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/pull.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/upgrade.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/main/version.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/generate.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/run.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/pipeline/upload.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/create.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/dir.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/runner/start.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/spinner.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/upload/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/upload/folder.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/cli/utils.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/env.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/folder.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/core/settings.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/client.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/daemon.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/directory.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/log_handler.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/logging.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/monitoring.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/action.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/cache.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/action/config.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/job/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/job/job.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/pipeline.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/resolver.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/command.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/docker.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/factory.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/node.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/python.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/runner.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/runner/shell.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/step/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/pipelines/step/step.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/services/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/docker.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/runner/utils/process.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/__init__.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/converters.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/files.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/hash_table.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/objects.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/package.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/string_style.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch/utils/url.py +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/dependency_links.txt +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/entry_points.txt +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/not-zip-safe +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/requires.txt +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/datatorch.egg-info/top_level.txt +0 -0
- {datatorch-0.8.1 → datatorch-0.8.2}/setup.cfg +0 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
COCO import run by the DataTorch server: the JSON is uploaded to the
|
|
3
|
+
project's default storage and the server's import job (createImport) maps
|
|
4
|
+
its images to dataset files by name, creates missing labels (with their
|
|
5
|
+
keypoint names and skeleton) and writes boxes, polygons and keypoints.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
import logging
|
|
13
|
+
from typing import List, Optional
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from .. import ApiClient, Dataset, Project, Where
|
|
18
|
+
|
|
19
|
+
_LOGGER = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
_CREATE_IMPORT = """
|
|
22
|
+
mutation CreateImport($input: CreateImportInput!) {
|
|
23
|
+
createImport(input: $input) { id }
|
|
24
|
+
}
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
_GET_IMPORT = """
|
|
28
|
+
query GetImport($id: ID!) {
|
|
29
|
+
import(id: $id) {
|
|
30
|
+
annotationsCreated
|
|
31
|
+
labelsCreated
|
|
32
|
+
filesProcessed
|
|
33
|
+
job { state logs { level message } }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
_DONE_STATES = ("SUCCESS", "FAILED", "CANCELED")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CocoImportError(Exception):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _plain(message: str) -> str:
|
|
46
|
+
"""Job log messages carry <b> tags for the web UI."""
|
|
47
|
+
return re.sub(r"<[^>]+>", "", message)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _pick_dataset(project: Project, dataset_name: Optional[str]) -> Dataset:
|
|
51
|
+
if dataset_name:
|
|
52
|
+
return project.dataset(dataset_name)
|
|
53
|
+
datasets = project.datasets()
|
|
54
|
+
if len(datasets) == 1:
|
|
55
|
+
return datasets[0]
|
|
56
|
+
names = [d.name for d in datasets]
|
|
57
|
+
raise CocoImportError(
|
|
58
|
+
f"Project has {len(datasets)} datasets, pass one with --dataset: {names}"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _upload(api: ApiClient, project: Project, file_path: str) -> str:
|
|
63
|
+
"""
|
|
64
|
+
Uploads the COCO file as a project file and returns its id. The upload
|
|
65
|
+
route returns no id and renames on a path clash, so the file goes into a
|
|
66
|
+
fresh folder and is found again by its exact path.
|
|
67
|
+
"""
|
|
68
|
+
# The server reads the file through whichever link holds it, so any link
|
|
69
|
+
# works: DataTorch Storage when the project has it, else the first one.
|
|
70
|
+
links = project.storage_links()
|
|
71
|
+
if not links:
|
|
72
|
+
raise CocoImportError("Project has no storage to upload the COCO file to")
|
|
73
|
+
link = next((l for l in links if l.name == "DataTorch Storage"), links[0])
|
|
74
|
+
folder = f"imports/{uuid.uuid4().hex}"
|
|
75
|
+
name = os.path.basename(file_path)
|
|
76
|
+
with open(file_path, "rb") as f:
|
|
77
|
+
r = requests.post(
|
|
78
|
+
f"{api.api_url}/file/v1/upload/{link.id}",
|
|
79
|
+
params={"path": folder, "import": "true"},
|
|
80
|
+
files={"file": (name, f, "application/json")},
|
|
81
|
+
headers={api.token_header: api._api_token},
|
|
82
|
+
)
|
|
83
|
+
if not r.ok:
|
|
84
|
+
raise CocoImportError(f"Upload failed ({r.status_code}): {r.text[:300]}")
|
|
85
|
+
|
|
86
|
+
files = project.files(Where(path=f"{folder}/{name}"), limit=2)
|
|
87
|
+
if len(files) != 1:
|
|
88
|
+
raise CocoImportError(f"Uploaded {name} but could not find it in the project")
|
|
89
|
+
return files[0].id
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def import_coco_server(
|
|
93
|
+
file_path: str,
|
|
94
|
+
project_string: str,
|
|
95
|
+
dataset_name: Optional[str] = None,
|
|
96
|
+
api: ApiClient = None,
|
|
97
|
+
timeout: float = 3600,
|
|
98
|
+
poll_seconds: float = 2,
|
|
99
|
+
) -> dict:
|
|
100
|
+
"""
|
|
101
|
+
Imports a COCO file into a dataset on the server and waits for the job.
|
|
102
|
+
Returns the import's counts and its warning messages; raises
|
|
103
|
+
CocoImportError with the job's error messages if it fails.
|
|
104
|
+
"""
|
|
105
|
+
if not os.path.isfile(file_path):
|
|
106
|
+
raise ValueError(f"Provided path '{file_path}' is not a file.")
|
|
107
|
+
|
|
108
|
+
if api is None:
|
|
109
|
+
api = ApiClient()
|
|
110
|
+
|
|
111
|
+
project: Project = api.project(project_string)
|
|
112
|
+
dataset = _pick_dataset(project, dataset_name)
|
|
113
|
+
|
|
114
|
+
_LOGGER.info("Uploading %s...", os.path.basename(file_path))
|
|
115
|
+
file_id = _upload(api, project, file_path)
|
|
116
|
+
|
|
117
|
+
import_id = api.execute(
|
|
118
|
+
_CREATE_IMPORT,
|
|
119
|
+
params={
|
|
120
|
+
"input": {"datasetId": dataset.id, "fileId": file_id, "format": "COCO"}
|
|
121
|
+
},
|
|
122
|
+
)["createImport"]["id"]
|
|
123
|
+
_LOGGER.info("Import %s queued for dataset '%s'.", import_id, dataset.name)
|
|
124
|
+
|
|
125
|
+
deadline = time.time() + timeout
|
|
126
|
+
while True:
|
|
127
|
+
imp = api.execute(_GET_IMPORT, params={"id": import_id})["import"]
|
|
128
|
+
state = imp["job"]["state"]
|
|
129
|
+
if state in _DONE_STATES:
|
|
130
|
+
break
|
|
131
|
+
if time.time() > deadline:
|
|
132
|
+
raise CocoImportError(
|
|
133
|
+
f"Timed out waiting for import {import_id} (last state {state})"
|
|
134
|
+
)
|
|
135
|
+
time.sleep(poll_seconds)
|
|
136
|
+
|
|
137
|
+
logs = imp["job"]["logs"] or []
|
|
138
|
+
|
|
139
|
+
def messages(level: str) -> List[str]:
|
|
140
|
+
return [_plain(log["message"]) for log in logs if log["level"] == level]
|
|
141
|
+
|
|
142
|
+
if state != "SUCCESS":
|
|
143
|
+
# The job logs one failure at several layers; the last is the cause.
|
|
144
|
+
errors = messages("error") or [f"Import job ended {state}"]
|
|
145
|
+
raise CocoImportError(errors[-1])
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
"import_id": import_id,
|
|
149
|
+
"dataset": dataset.name,
|
|
150
|
+
"files_processed": imp["filesProcessed"],
|
|
151
|
+
"labels_created": imp["labelsCreated"],
|
|
152
|
+
"annotations_created": imp["annotationsCreated"],
|
|
153
|
+
"warnings": messages("warn"),
|
|
154
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@click.command(
|
|
5
|
+
"coco",
|
|
6
|
+
help="Import a COCO formated file. By default the DataTorch server runs the "
|
|
7
|
+
"import: it matches images to dataset files by name, creates missing labels "
|
|
8
|
+
"and imports boxes, polygons and keypoints.",
|
|
9
|
+
)
|
|
10
|
+
@click.option(
|
|
11
|
+
"-f",
|
|
12
|
+
"--file",
|
|
13
|
+
type=click.Path(exists=True),
|
|
14
|
+
prompt=True,
|
|
15
|
+
help="Path to the json file containing coco annotations.",
|
|
16
|
+
)
|
|
17
|
+
@click.option(
|
|
18
|
+
"-p",
|
|
19
|
+
"--project",
|
|
20
|
+
type=str,
|
|
21
|
+
prompt=True,
|
|
22
|
+
help="Namespace identifier or UUID of project (e.g. username/project)",
|
|
23
|
+
)
|
|
24
|
+
@click.option(
|
|
25
|
+
"-d",
|
|
26
|
+
"--dataset",
|
|
27
|
+
type=str,
|
|
28
|
+
default=None,
|
|
29
|
+
help="Name of the dataset to import into. Defaults to the project's only dataset.",
|
|
30
|
+
)
|
|
31
|
+
@click.option(
|
|
32
|
+
"--client-side",
|
|
33
|
+
is_flag=True,
|
|
34
|
+
default=False,
|
|
35
|
+
help="Import from this machine with pycocotools instead of on the server. "
|
|
36
|
+
"Labels must already exist, keypoints are not imported, and the options "
|
|
37
|
+
"below apply only in this mode.",
|
|
38
|
+
)
|
|
39
|
+
@click.option(
|
|
40
|
+
"-m",
|
|
41
|
+
"--max-iou",
|
|
42
|
+
type=float,
|
|
43
|
+
default=0.99,
|
|
44
|
+
show_default=True,
|
|
45
|
+
help="(--client-side) If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"-s",
|
|
49
|
+
"--simplify",
|
|
50
|
+
type=float,
|
|
51
|
+
default=1.0,
|
|
52
|
+
show_default=True,
|
|
53
|
+
help="(--client-side) Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
|
|
54
|
+
)
|
|
55
|
+
@click.option(
|
|
56
|
+
"--no-segmentations",
|
|
57
|
+
type=bool,
|
|
58
|
+
flag_value=False,
|
|
59
|
+
default=True,
|
|
60
|
+
help="(--client-side) Flag to disable the import of coco segmentations.",
|
|
61
|
+
)
|
|
62
|
+
@click.option(
|
|
63
|
+
"--import-bbox",
|
|
64
|
+
type=bool,
|
|
65
|
+
flag_value=True,
|
|
66
|
+
default=False,
|
|
67
|
+
help="(--client-side) Flag to enable to import of coco bounding boxes.",
|
|
68
|
+
)
|
|
69
|
+
def coco_cmd(
|
|
70
|
+
file,
|
|
71
|
+
project,
|
|
72
|
+
dataset,
|
|
73
|
+
client_side,
|
|
74
|
+
max_iou,
|
|
75
|
+
no_segmentations,
|
|
76
|
+
import_bbox,
|
|
77
|
+
simplify,
|
|
78
|
+
):
|
|
79
|
+
if client_side:
|
|
80
|
+
from datatorch.api.scripts.import_coco import import_coco
|
|
81
|
+
|
|
82
|
+
import_coco(
|
|
83
|
+
file,
|
|
84
|
+
project,
|
|
85
|
+
import_bbox=import_bbox,
|
|
86
|
+
import_segmentation=no_segmentations,
|
|
87
|
+
max_iou=max_iou,
|
|
88
|
+
simplify_tolerance=simplify,
|
|
89
|
+
)
|
|
90
|
+
return
|
|
91
|
+
|
|
92
|
+
from datatorch.api.scripts.import_coco_server import (
|
|
93
|
+
CocoImportError,
|
|
94
|
+
import_coco_server,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
click.echo("Uploading and importing on the server...")
|
|
98
|
+
try:
|
|
99
|
+
result = import_coco_server(file, project, dataset_name=dataset)
|
|
100
|
+
except CocoImportError as e:
|
|
101
|
+
raise click.ClickException(f"COCO import failed:\n{e}")
|
|
102
|
+
|
|
103
|
+
for warning in result["warnings"]:
|
|
104
|
+
click.secho(f"warning: {warning}", fg="yellow")
|
|
105
|
+
click.echo(
|
|
106
|
+
f"Imported into dataset '{result['dataset']}': "
|
|
107
|
+
f"{result['annotations_created']} annotations on "
|
|
108
|
+
f"{result['files_processed']} files, "
|
|
109
|
+
f"{result['labels_created']} new labels."
|
|
110
|
+
)
|
|
@@ -2,8 +2,10 @@ import json
|
|
|
2
2
|
import platform
|
|
3
3
|
import re
|
|
4
4
|
import typing
|
|
5
|
-
|
|
6
|
-
from
|
|
5
|
+
import uuid
|
|
6
|
+
from typing import Dict, List, Tuple
|
|
7
|
+
from jinja2 import Environment, Template, meta
|
|
8
|
+
from jinja2.exceptions import TemplateSyntaxError
|
|
7
9
|
from ..directory import runner_directory
|
|
8
10
|
|
|
9
11
|
if typing.TYPE_CHECKING:
|
|
@@ -19,6 +21,17 @@ if typing.TYPE_CHECKING:
|
|
|
19
21
|
# $INPUT_<NAME> env vars instead.
|
|
20
22
|
_INPUT_REF_IN_TEMPLATE = re.compile(r"\$\{[{%][^}]*?(?<![.\w])(?:input|variable)(?!\w)")
|
|
21
23
|
|
|
24
|
+
# One `${{ ... }}` expression, as written. Used to find the expressions in
|
|
25
|
+
# a string before rendering so that the ones the runner does not own can
|
|
26
|
+
# be kept exactly as they are (see `Variables.render`).
|
|
27
|
+
_EXPRESSION = re.compile(r"\$\{\{.*?\}\}", re.DOTALL)
|
|
28
|
+
|
|
29
|
+
_environment = Environment(
|
|
30
|
+
block_start_string="${%",
|
|
31
|
+
variable_start_string="${{",
|
|
32
|
+
comment_start_string="${#",
|
|
33
|
+
)
|
|
34
|
+
|
|
22
35
|
|
|
23
36
|
class InputInjectionError(Exception):
|
|
24
37
|
"""Raised when a shell command/script interpolates ${{ input.* }} /
|
|
@@ -72,6 +85,17 @@ global_variables = {
|
|
|
72
85
|
},
|
|
73
86
|
}
|
|
74
87
|
|
|
88
|
+
# The namespaces the runner renders (`run` since 0.8.1: the step's place
|
|
89
|
+
# in the record). `${{ ... }}` text rooted anywhere else (`steps.*`,
|
|
90
|
+
# `inputs.*`, `jobs.*`, `secrets.*`, unknown names) is not a reference
|
|
91
|
+
# the runner can resolve: the server resolves its own namespaces before
|
|
92
|
+
# dispatch, and whatever still reads like one in a step input is DATA (a
|
|
93
|
+
# log line, a fetched body, an older error message) that must reach the
|
|
94
|
+
# action untouched.
|
|
95
|
+
RUNNER_NAMESPACES = frozenset(
|
|
96
|
+
{*global_variables.keys(), "input", "variable", "step", "action", "run"}
|
|
97
|
+
)
|
|
98
|
+
|
|
75
99
|
|
|
76
100
|
class Variables(object):
|
|
77
101
|
"""
|
|
@@ -92,6 +116,12 @@ class Variables(object):
|
|
|
92
116
|
for use inside the action's own templates (``runs.command`` etc.).
|
|
93
117
|
``variable`` is a legacy alias of ``input``; both are scoped to the
|
|
94
118
|
one step. The old job-wide accumulating namespace is gone.
|
|
119
|
+
|
|
120
|
+
Everything else is inert. A ``${{ ... }}`` whose root names are not all
|
|
121
|
+
in ``RUNNER_NAMESPACES`` (or that does not parse) is left exactly as
|
|
122
|
+
written, never raising and never rewritten: the runner does not author
|
|
123
|
+
that text, so it must not interpret it. Only string inputs are rendered;
|
|
124
|
+
objects and arrays pass through as they arrive.
|
|
95
125
|
"""
|
|
96
126
|
|
|
97
127
|
def __init__(self):
|
|
@@ -140,8 +170,54 @@ class Variables(object):
|
|
|
140
170
|
self.variables[section] = {**self.variables[section], **variables}
|
|
141
171
|
|
|
142
172
|
def render(self, string: str):
|
|
143
|
-
|
|
144
|
-
|
|
173
|
+
"""Render the runner's own namespaces in ``string``; keep every
|
|
174
|
+
other ``${{ ... }}`` exactly as written (see the class docstring).
|
|
175
|
+
"""
|
|
176
|
+
if string is None or "${" not in string:
|
|
177
|
+
return string
|
|
178
|
+
protected, kept = self._protect_foreign_expressions(string)
|
|
179
|
+
try:
|
|
180
|
+
rendered = self._template(protected).render(
|
|
181
|
+
{**global_variables, **self.variables}
|
|
182
|
+
)
|
|
183
|
+
except TemplateSyntaxError:
|
|
184
|
+
# Not a template the runner can read (an unclosed `${{`, a
|
|
185
|
+
# stray `${%` in data): it is text, hand it back as it came.
|
|
186
|
+
return string
|
|
187
|
+
for token, text in kept:
|
|
188
|
+
rendered = rendered.replace(token, text)
|
|
189
|
+
return rendered
|
|
190
|
+
|
|
191
|
+
@classmethod
|
|
192
|
+
def _protect_foreign_expressions(
|
|
193
|
+
cls, string: str
|
|
194
|
+
) -> Tuple[str, List[Tuple[str, str]]]:
|
|
195
|
+
"""Swap each ``${{ ... }}`` the runner does not own for an opaque
|
|
196
|
+
token so Jinja never sees it; the caller restores them after
|
|
197
|
+
rendering. Returns the protected string and the (token, original
|
|
198
|
+
text) pairs."""
|
|
199
|
+
nonce = uuid.uuid4().hex
|
|
200
|
+
kept: List[Tuple[str, str]] = []
|
|
201
|
+
|
|
202
|
+
def swap(match: "re.Match[str]") -> str:
|
|
203
|
+
text = match.group(0)
|
|
204
|
+
if cls._owns(text):
|
|
205
|
+
return text
|
|
206
|
+
token = f"\x00{nonce}:{len(kept)}\x00"
|
|
207
|
+
kept.append((token, text))
|
|
208
|
+
return token
|
|
209
|
+
|
|
210
|
+
return _EXPRESSION.sub(swap, string), kept
|
|
211
|
+
|
|
212
|
+
@staticmethod
|
|
213
|
+
def _owns(expression: str) -> bool:
|
|
214
|
+
"""True when ``expression`` (one ``${{ ... }}``) parses and every
|
|
215
|
+
name it reads is one of the runner's namespaces."""
|
|
216
|
+
try:
|
|
217
|
+
ast = _environment.parse(expression)
|
|
218
|
+
except TemplateSyntaxError:
|
|
219
|
+
return False
|
|
220
|
+
return meta.find_undeclared_variables(ast) <= RUNNER_NAMESPACES
|
|
145
221
|
|
|
146
222
|
def render_command(self, string: str):
|
|
147
223
|
"""Render a shell command/script template.
|
|
@@ -177,12 +253,7 @@ class Variables(object):
|
|
|
177
253
|
|
|
178
254
|
@staticmethod
|
|
179
255
|
def _template(string: str) -> Template:
|
|
180
|
-
return
|
|
181
|
-
string,
|
|
182
|
-
block_start_string="${%",
|
|
183
|
-
variable_start_string="${{",
|
|
184
|
-
comment_start_string="${#",
|
|
185
|
-
)
|
|
256
|
+
return _environment.from_string(string)
|
|
186
257
|
|
|
187
258
|
@property
|
|
188
259
|
def inputs(self) -> dict:
|
|
@@ -33,6 +33,7 @@ datatorch/api/entity/sources/image/segmentations.py
|
|
|
33
33
|
datatorch/api/entity/sources/image/typings.py
|
|
34
34
|
datatorch/api/scripts/__init__.py
|
|
35
35
|
datatorch/api/scripts/import_coco.py
|
|
36
|
+
datatorch/api/scripts/import_coco_server.py
|
|
36
37
|
datatorch/api/scripts/import_pixmask.py
|
|
37
38
|
datatorch/api/scripts/utils/__init__.py
|
|
38
39
|
datatorch/api/scripts/utils/simplify.py
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
import click
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
@click.command("coco", help="Import a COCO formated file.")
|
|
5
|
-
@click.option(
|
|
6
|
-
"-f",
|
|
7
|
-
"--file",
|
|
8
|
-
type=click.Path(exists=True),
|
|
9
|
-
prompt=True,
|
|
10
|
-
help="Path to the json file containing coco annotations.",
|
|
11
|
-
)
|
|
12
|
-
@click.option(
|
|
13
|
-
"-p",
|
|
14
|
-
"--project",
|
|
15
|
-
type=str,
|
|
16
|
-
prompt=True,
|
|
17
|
-
help="Namespace identifier or UUID of project (e.g. username/project)",
|
|
18
|
-
)
|
|
19
|
-
@click.option(
|
|
20
|
-
"-m",
|
|
21
|
-
"--max-iou",
|
|
22
|
-
type=float,
|
|
23
|
-
default=0.99,
|
|
24
|
-
show_default=True,
|
|
25
|
-
help="If an annotations matches this iou then it will not be added. Set to zero to disable. Disabling will significantly increase performance.",
|
|
26
|
-
)
|
|
27
|
-
@click.option(
|
|
28
|
-
"-s",
|
|
29
|
-
"--simplify",
|
|
30
|
-
type=float,
|
|
31
|
-
default=1.0,
|
|
32
|
-
show_default=True,
|
|
33
|
-
help="Simplification tolerance for new segmentation annotations. Set to zero to disable. Disabling will significantly increase performance.",
|
|
34
|
-
)
|
|
35
|
-
@click.option(
|
|
36
|
-
"--no-segmentations",
|
|
37
|
-
type=bool,
|
|
38
|
-
flag_value=False,
|
|
39
|
-
default=True,
|
|
40
|
-
help="Flag to disable the import of coco segmentations.",
|
|
41
|
-
)
|
|
42
|
-
@click.option(
|
|
43
|
-
"--import-bbox",
|
|
44
|
-
type=bool,
|
|
45
|
-
flag_value=True,
|
|
46
|
-
default=False,
|
|
47
|
-
help="Flag to enable to import of coco bounding boxes.",
|
|
48
|
-
)
|
|
49
|
-
def coco_cmd(file, project, max_iou, no_segmentations, import_bbox, simplify):
|
|
50
|
-
from datatorch.api.scripts.import_coco import import_coco
|
|
51
|
-
|
|
52
|
-
import_coco(
|
|
53
|
-
file,
|
|
54
|
-
project,
|
|
55
|
-
import_bbox=import_bbox,
|
|
56
|
-
import_segmentation=no_segmentations,
|
|
57
|
-
max_iou=max_iou,
|
|
58
|
-
simplify_tolerance=simplify,
|
|
59
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|