datashare-python 0.8.19__py3-none-any.whl → 0.8.23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datashare_python/conftest.py +9 -5
- datashare_python/objects.py +207 -105
- datashare_python/template.py +24 -7
- datashare_python/utils.py +52 -8
- datashare_python/worker-template.tar.gz +0 -0
- {datashare_python-0.8.19.dist-info → datashare_python-0.8.23.dist-info}/METADATA +3 -1
- {datashare_python-0.8.19.dist-info → datashare_python-0.8.23.dist-info}/RECORD +9 -9
- {datashare_python-0.8.19.dist-info → datashare_python-0.8.23.dist-info}/WHEEL +1 -1
- {datashare_python-0.8.19.dist-info → datashare_python-0.8.23.dist-info}/entry_points.txt +0 -0
datashare_python/conftest.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import shutil
|
|
2
3
|
from asyncio import AbstractEventLoop
|
|
3
4
|
from collections.abc import AsyncGenerator, Generator, Iterator, Sequence
|
|
4
5
|
from pathlib import Path
|
|
@@ -53,7 +54,6 @@ _INDEX_BODY = {
|
|
|
53
54
|
"join": {"type": "join", "relations": {"Document": "NamedEntity"}},
|
|
54
55
|
"contentType": {"type": "keyword"},
|
|
55
56
|
"content": {"type": "text"},
|
|
56
|
-
"contentTranslated": {"type": "text"},
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
}
|
|
@@ -197,10 +197,7 @@ def index_docs_ops(
|
|
|
197
197
|
docs: list[Document], index_name: str
|
|
198
198
|
) -> Generator[dict, None, None]:
|
|
199
199
|
for doc in docs:
|
|
200
|
-
op = {
|
|
201
|
-
"_op_type": "index",
|
|
202
|
-
"_index": index_name,
|
|
203
|
-
}
|
|
200
|
+
op = {"_op_type": "index", "_index": index_name}
|
|
204
201
|
doc = doc.model_dump(by_alias=True) # noqa: PLW2901
|
|
205
202
|
op.update(doc)
|
|
206
203
|
if "path" in op:
|
|
@@ -305,3 +302,10 @@ async def all_done(task_client: DatashareTaskClient, not_done: list[str]) -> boo
|
|
|
305
302
|
@pytest.fixture # noqa: F405
|
|
306
303
|
def typer_asyncio_patch() -> None:
|
|
307
304
|
nest_asyncio.apply()
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def clear_dirs(config: WorkerConfig) -> None:
|
|
308
|
+
shutil.rmtree(str(config.artifacts_root))
|
|
309
|
+
config.artifacts_root.mkdir(parents=True, exist_ok=True)
|
|
310
|
+
shutil.rmtree(str(config.workdir))
|
|
311
|
+
config.workdir.mkdir(parents=True, exist_ok=True)
|
datashare_python/objects.py
CHANGED
|
@@ -6,8 +6,12 @@ from datetime import UTC, datetime
|
|
|
6
6
|
from enum import StrEnum, unique
|
|
7
7
|
from io import BytesIO
|
|
8
8
|
from pathlib import Path
|
|
9
|
-
from typing import Annotated, Any, Literal, Self, TypeVar, cast
|
|
9
|
+
from typing import Annotated, Any, ClassVar, Literal, Self, TypeVar, cast
|
|
10
10
|
|
|
11
|
+
import langcodes
|
|
12
|
+
from pydantic_core import PydanticCustomError, ValidationError, core_schema
|
|
13
|
+
from pydantic_core.core_schema import PlainValidatorFunctionSchema
|
|
14
|
+
from pydantic_extra_types.language_code import LanguageName
|
|
11
15
|
from temporalio import workflow
|
|
12
16
|
|
|
13
17
|
from .constants import TIKA_METADATA_RESOURCENAME
|
|
@@ -31,7 +35,14 @@ from icij_common.pydantic_utils import (
|
|
|
31
35
|
merge_configs,
|
|
32
36
|
no_enum_values_config,
|
|
33
37
|
)
|
|
34
|
-
from pydantic import
|
|
38
|
+
from pydantic import (
|
|
39
|
+
AfterValidator,
|
|
40
|
+
BeforeValidator,
|
|
41
|
+
Field,
|
|
42
|
+
GetCoreSchemaHandler,
|
|
43
|
+
TypeAdapter,
|
|
44
|
+
model_validator,
|
|
45
|
+
)
|
|
35
46
|
from pydantic import BaseModel as _BaseModel
|
|
36
47
|
from pydantic.main import IncEx
|
|
37
48
|
|
|
@@ -50,110 +61,45 @@ class DatashareModel(BaseModel):
|
|
|
50
61
|
model_config = merge_configs(BaseModel.model_config, lowercamel_case_config())
|
|
51
62
|
|
|
52
63
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
CREATED = "CREATED"
|
|
56
|
-
QUEUED = "QUEUED"
|
|
57
|
-
RUNNING = "RUNNING"
|
|
58
|
-
ERROR = "ERROR"
|
|
59
|
-
DONE = "DONE"
|
|
60
|
-
CANCELLED = "CANCELLED"
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
READY_STATES = frozenset({TaskState.DONE, TaskState.ERROR, TaskState.CANCELLED})
|
|
64
|
+
class DatashareLanguage(str):
|
|
65
|
+
_language_type_adapter: ClassVar[TypeAdapter] = TypeAdapter(LanguageName)
|
|
64
66
|
|
|
67
|
+
@classmethod
|
|
68
|
+
def _validate(cls, __input_value: str, _: core_schema.ValidationInfo) -> Self:
|
|
69
|
+
if __input_value != __input_value.upper():
|
|
70
|
+
raise PydanticCustomError(
|
|
71
|
+
"datashare_language", "Invalid Datashare language, expected uppercase"
|
|
72
|
+
)
|
|
73
|
+
try:
|
|
74
|
+
# Use pydantic provided validation
|
|
75
|
+
cls._language_type_adapter.validate_python(__input_value.title())
|
|
76
|
+
except ValidationError as e:
|
|
77
|
+
raise PydanticCustomError(
|
|
78
|
+
"datashare_language", "Unknown Datashare language"
|
|
79
|
+
) from e
|
|
80
|
+
return cls(__input_value)
|
|
65
81
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def model_dump(
|
|
76
|
-
self,
|
|
77
|
-
*,
|
|
78
|
-
mode: Literal["json", "python"] | str = "python",
|
|
79
|
-
include: IncEx | None = None,
|
|
80
|
-
exclude: IncEx | None = None,
|
|
81
|
-
context: Any | None = None,
|
|
82
|
-
exclude_unset: bool = False,
|
|
83
|
-
exclude_defaults: bool = False,
|
|
84
|
-
exclude_none: bool = False,
|
|
85
|
-
round_trip: bool = False,
|
|
86
|
-
warnings: bool | Literal["none", "warn", "error"] = True,
|
|
87
|
-
fallback: Callable[[Any], Any] | None = None,
|
|
88
|
-
serialize_as_any: bool = False,
|
|
89
|
-
) -> dict[str, Any]:
|
|
90
|
-
return super().model_dump(
|
|
91
|
-
by_alias=True,
|
|
92
|
-
mode=mode,
|
|
93
|
-
include=include,
|
|
94
|
-
exclude=exclude,
|
|
95
|
-
context=context,
|
|
96
|
-
exclude_unset=exclude_unset,
|
|
97
|
-
exclude_defaults=exclude_defaults,
|
|
98
|
-
exclude_none=exclude_none,
|
|
99
|
-
round_trip=round_trip,
|
|
100
|
-
warnings=warnings,
|
|
101
|
-
fallback=fallback,
|
|
102
|
-
serialize_as_any=serialize_as_any,
|
|
82
|
+
@classmethod
|
|
83
|
+
def __get_pydantic_core_schema__(
|
|
84
|
+
cls, source: type[Any], handler: GetCoreSchemaHandler
|
|
85
|
+
) -> core_schema.AfterValidatorFunctionSchema:
|
|
86
|
+
return core_schema.with_info_after_validator_function(
|
|
87
|
+
cls._validate,
|
|
88
|
+
core_schema.str_schema(),
|
|
89
|
+
serialization=core_schema.to_string_ser_schema(),
|
|
103
90
|
)
|
|
104
91
|
|
|
92
|
+
@property
|
|
93
|
+
def as_language_name(self) -> LanguageName:
|
|
94
|
+
return LanguageName(self.title())
|
|
105
95
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
class TaskError(Message):
|
|
112
|
-
type: str = Field(frozen=True, alias="@type", default="TaskError")
|
|
113
|
-
name: str
|
|
114
|
-
message: str
|
|
115
|
-
cause: str | None = None
|
|
116
|
-
stacktrace: list[StacktraceItem] = Field(default_factory=list)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def _datetime_now() -> datetime:
|
|
120
|
-
return datetime.now(UTC)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
class User(Message):
|
|
124
|
-
type: str = Field(
|
|
125
|
-
frozen=True, alias="@type", default="org.icij.datashare.user.User"
|
|
126
|
-
)
|
|
127
|
-
id: str
|
|
128
|
-
name: str | None = None
|
|
129
|
-
email: str | None = None
|
|
130
|
-
provider: str | None = None
|
|
131
|
-
details: dict = dict()
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
class Task(Message):
|
|
135
|
-
type: str = Field(frozen=True, alias="@type", default="Task")
|
|
136
|
-
id: str
|
|
137
|
-
name: str
|
|
138
|
-
args: dict[str, object] | None = None
|
|
139
|
-
state: TaskState = TaskState.CREATED
|
|
140
|
-
result: TaskResult | None = None
|
|
141
|
-
error: TaskError | None = None
|
|
142
|
-
progress: float | None = None
|
|
143
|
-
created_at: datetime = Field(default_factory=_datetime_now)
|
|
144
|
-
completed_at: datetime | None = None
|
|
145
|
-
retries_left: int | None = None
|
|
146
|
-
max_retries: int | None = None
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
@dataclass(frozen=True)
|
|
150
|
-
class TaskGroup:
|
|
151
|
-
name: str
|
|
96
|
+
@property
|
|
97
|
+
def alpha2(self) -> str | None:
|
|
98
|
+
return self.as_language_name.alpha2
|
|
152
99
|
|
|
153
100
|
@property
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
return cls(name="PYTHON")
|
|
101
|
+
def alpha3(self) -> str:
|
|
102
|
+
return self.as_language_name.alpha3
|
|
157
103
|
|
|
158
104
|
|
|
159
105
|
@unique
|
|
@@ -195,21 +141,64 @@ class FilesystemDocument(DatashareModel):
|
|
|
195
141
|
raise ValueError(f"invalid location: {self.path}")
|
|
196
142
|
|
|
197
143
|
|
|
144
|
+
class IETFLanguage(str):
|
|
145
|
+
@classmethod
|
|
146
|
+
def __get_pydantic_core_schema__(
|
|
147
|
+
cls, source: Any, handler: GetCoreSchemaHandler
|
|
148
|
+
) -> PlainValidatorFunctionSchema:
|
|
149
|
+
return core_schema.no_info_plain_validator_function(cls.validate)
|
|
150
|
+
|
|
151
|
+
@classmethod
|
|
152
|
+
def validate(cls, v: Any) -> Self:
|
|
153
|
+
tag = langcodes.get(str(v))
|
|
154
|
+
if not tag.is_valid():
|
|
155
|
+
raise ValueError(f"Invalid IETF language: {v}")
|
|
156
|
+
return cls(v)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
Language = DatashareLanguage | IETFLanguage
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _from_sentences(value: Any) -> Any:
|
|
163
|
+
if isinstance(value, list):
|
|
164
|
+
return " ".join(value)
|
|
165
|
+
return value
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class Translation(BaseModel): # No camelcase here we don't know why
|
|
169
|
+
source_language: DatashareLanguage
|
|
170
|
+
target_language: Language
|
|
171
|
+
translator: str
|
|
172
|
+
content: Annotated[str, BeforeValidator(_from_sentences)]
|
|
173
|
+
|
|
174
|
+
|
|
198
175
|
class Document(DatashareModel):
|
|
199
176
|
id: str
|
|
200
|
-
language:
|
|
177
|
+
language: DatashareLanguage
|
|
201
178
|
index: str | None = None
|
|
202
179
|
root_document: str | None = None
|
|
203
180
|
content: str | None = None
|
|
181
|
+
content_text_length: int | None = None
|
|
204
182
|
content_type: str | None = None
|
|
205
183
|
path: Path | None = None
|
|
206
184
|
tags: list[str] = Field(default_factory=list)
|
|
207
|
-
content_translated:
|
|
208
|
-
|
|
185
|
+
content_translated: list[Translation] | None = Field(
|
|
186
|
+
default=None,
|
|
187
|
+
# es translator is using snake_case, we must do the same
|
|
188
|
+
alias="content_translated",
|
|
209
189
|
)
|
|
210
190
|
metadata: dict[str, Any] | None = None
|
|
211
191
|
type: str = Field(default="Document", frozen=True)
|
|
212
192
|
|
|
193
|
+
@model_validator(mode="before")
|
|
194
|
+
@classmethod
|
|
195
|
+
def _initialize_content_length_from_content(cls, data: Any) -> Any:
|
|
196
|
+
if isinstance(data, dict):
|
|
197
|
+
content_length = data.get("content_text_length")
|
|
198
|
+
if content_length is None and (content := data.get(DOC_CONTENT)):
|
|
199
|
+
data["content_text_length"] = len(content)
|
|
200
|
+
return data
|
|
201
|
+
|
|
213
202
|
@classmethod
|
|
214
203
|
def from_es(cls, es_doc: dict) -> Self:
|
|
215
204
|
sources = es_doc[SOURCE]
|
|
@@ -217,8 +206,9 @@ class Document(DatashareModel):
|
|
|
217
206
|
id=es_doc[ID_],
|
|
218
207
|
index=es_doc.get(INDEX_),
|
|
219
208
|
content=sources.get(DOC_CONTENT),
|
|
220
|
-
content_translated=sources.get(DOC_CONTENT_TRANSLATED,
|
|
221
|
-
|
|
209
|
+
content_translated=sources.get(DOC_CONTENT_TRANSLATED, []),
|
|
210
|
+
content_text_length=sources.get("content_text_length"),
|
|
211
|
+
language=DatashareLanguage(sources[DOC_LANGUAGE]),
|
|
222
212
|
root_document=sources.get(DOC_ROOT_ID),
|
|
223
213
|
tags=sources.get("tags", []),
|
|
224
214
|
path=sources.get(DOC_PATH),
|
|
@@ -258,10 +248,122 @@ class Document(DatashareModel):
|
|
|
258
248
|
)
|
|
259
249
|
|
|
260
250
|
|
|
251
|
+
def _is_absolute_path(v: bytes | BytesIO | Path) -> Any:
|
|
252
|
+
if isinstance(v, Path) and not v.is_absolute():
|
|
253
|
+
raise ValueError("artifact path must be absolute")
|
|
254
|
+
return v
|
|
255
|
+
|
|
256
|
+
|
|
261
257
|
@dataclass(frozen=True)
|
|
262
258
|
class DocArtifact:
|
|
263
259
|
project: str
|
|
264
260
|
doc_id: str
|
|
265
|
-
artifact: bytes | BytesIO
|
|
261
|
+
artifact: Annotated[bytes | BytesIO | Path, AfterValidator(_is_absolute_path)]
|
|
266
262
|
filename: str
|
|
267
263
|
metadata_key: str
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@unique
|
|
267
|
+
class TaskState(StrEnum):
|
|
268
|
+
CREATED = "CREATED"
|
|
269
|
+
QUEUED = "QUEUED"
|
|
270
|
+
RUNNING = "RUNNING"
|
|
271
|
+
ERROR = "ERROR"
|
|
272
|
+
DONE = "DONE"
|
|
273
|
+
CANCELLED = "CANCELLED"
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
READY_STATES = frozenset({TaskState.DONE, TaskState.ERROR, TaskState.CANCELLED})
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class StacktraceItem(DatashareModel):
|
|
280
|
+
name: str
|
|
281
|
+
file: str
|
|
282
|
+
lineno: int
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
class Message(DatashareModel):
|
|
286
|
+
type: str = Field(frozen=True, alias="@type")
|
|
287
|
+
|
|
288
|
+
def model_dump(
|
|
289
|
+
self,
|
|
290
|
+
*,
|
|
291
|
+
mode: Literal["json", "python"] | str = "python",
|
|
292
|
+
include: IncEx | None = None,
|
|
293
|
+
exclude: IncEx | None = None,
|
|
294
|
+
context: Any | None = None,
|
|
295
|
+
exclude_unset: bool = False,
|
|
296
|
+
exclude_defaults: bool = False,
|
|
297
|
+
exclude_none: bool = False,
|
|
298
|
+
round_trip: bool = False,
|
|
299
|
+
warnings: bool | Literal["none", "warn", "error"] = True,
|
|
300
|
+
fallback: Callable[[Any], Any] | None = None,
|
|
301
|
+
serialize_as_any: bool = False,
|
|
302
|
+
) -> dict[str, Any]:
|
|
303
|
+
return super().model_dump(
|
|
304
|
+
by_alias=True,
|
|
305
|
+
mode=mode,
|
|
306
|
+
include=include,
|
|
307
|
+
exclude=exclude,
|
|
308
|
+
context=context,
|
|
309
|
+
exclude_unset=exclude_unset,
|
|
310
|
+
exclude_defaults=exclude_defaults,
|
|
311
|
+
exclude_none=exclude_none,
|
|
312
|
+
round_trip=round_trip,
|
|
313
|
+
warnings=warnings,
|
|
314
|
+
fallback=fallback,
|
|
315
|
+
serialize_as_any=serialize_as_any,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class TaskResult(Message):
|
|
320
|
+
type: str = Field(frozen=True, alias="@type", default="TaskResult")
|
|
321
|
+
value: object
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class TaskError(Message):
|
|
325
|
+
type: str = Field(frozen=True, alias="@type", default="TaskError")
|
|
326
|
+
name: str
|
|
327
|
+
message: str
|
|
328
|
+
cause: str | None = None
|
|
329
|
+
stacktrace: list[StacktraceItem] = Field(default_factory=list)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _datetime_now() -> datetime:
|
|
333
|
+
return datetime.now(UTC)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
class User(Message):
|
|
337
|
+
type: str = Field(
|
|
338
|
+
frozen=True, alias="@type", default="org.icij.datashare.user.User"
|
|
339
|
+
)
|
|
340
|
+
id: str
|
|
341
|
+
name: str | None = None
|
|
342
|
+
email: str | None = None
|
|
343
|
+
provider: str | None = None
|
|
344
|
+
details: dict = dict()
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class Task(Message):
|
|
348
|
+
type: str = Field(frozen=True, alias="@type", default="Task")
|
|
349
|
+
id: str
|
|
350
|
+
name: str
|
|
351
|
+
args: dict[str, object] | None = None
|
|
352
|
+
state: TaskState = TaskState.CREATED
|
|
353
|
+
result: TaskResult | None = None
|
|
354
|
+
error: TaskError | None = None
|
|
355
|
+
progress: float | None = None
|
|
356
|
+
created_at: datetime = Field(default_factory=_datetime_now)
|
|
357
|
+
completed_at: datetime | None = None
|
|
358
|
+
retries_left: int | None = None
|
|
359
|
+
max_retries: int | None = None
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
@dataclass(frozen=True)
|
|
363
|
+
class TaskGroup:
|
|
364
|
+
name: str
|
|
365
|
+
|
|
366
|
+
@property
|
|
367
|
+
@classmethod
|
|
368
|
+
def python(cls) -> Self:
|
|
369
|
+
return cls(name="PYTHON")
|
datashare_python/template.py
CHANGED
|
@@ -41,7 +41,8 @@ def build_template_tarball() -> None:
|
|
|
41
41
|
is_hidden = path.name.startswith(".") or any(
|
|
42
42
|
"." in p for p in path.parts[:-1]
|
|
43
43
|
)
|
|
44
|
-
|
|
44
|
+
skip = is_hidden or not path.is_file() or path.suffix not in ALLOWED_EXTS
|
|
45
|
+
if path.name != "Dockerfile" and skip:
|
|
45
46
|
continue
|
|
46
47
|
tar.add(path, arcname=path.relative_to(template_dir))
|
|
47
48
|
|
|
@@ -75,8 +76,11 @@ def _update_pyproject_toml(
|
|
|
75
76
|
pyproject_toml["tool"]["uv"].pop("index", None)
|
|
76
77
|
|
|
77
78
|
project = pyproject_toml["project"]
|
|
79
|
+
project["name"] = package_name.replace("_", "-").lower()
|
|
80
|
+
project["version"] = "0.1.0"
|
|
78
81
|
project["authors"] = []
|
|
79
82
|
project.pop("urls", None)
|
|
83
|
+
project.pop("description", None)
|
|
80
84
|
project["dependencies"] = sorted(
|
|
81
85
|
d
|
|
82
86
|
for d in project["dependencies"]
|
|
@@ -100,16 +104,29 @@ def _update_pyproject_toml(
|
|
|
100
104
|
"worker_template", package_name
|
|
101
105
|
)
|
|
102
106
|
entry_points["datashare.activities"]["activities"] = activities_entry_point
|
|
107
|
+
|
|
108
|
+
deps = entry_points["datashare.dependencies"]["dependencies"]
|
|
109
|
+
deps = deps.replace("worker_template", package_name)
|
|
110
|
+
entry_points["datashare.dependencies"]["dependencies"] = deps
|
|
111
|
+
|
|
112
|
+
cfg_cls_entry_point = entry_points["datashare.worker_config_cls"][
|
|
113
|
+
"worker_config_cls"
|
|
114
|
+
]
|
|
115
|
+
cfg_cls_entry_point = cfg_cls_entry_point.replace("worker_template", package_name)
|
|
116
|
+
entry_points["datashare.worker_config_cls"]["worker_config_cls"] = (
|
|
117
|
+
cfg_cls_entry_point
|
|
118
|
+
)
|
|
119
|
+
|
|
103
120
|
hatch_sdist = pyproject_toml["tool"]["hatch"]["build"]["targets"]["wheel"]
|
|
104
121
|
hatch_sdist["packages"] = [
|
|
105
122
|
i if i != "worker_template" else package_name for i in hatch_sdist["packages"]
|
|
106
123
|
]
|
|
107
124
|
|
|
108
|
-
hatch_sdist = pyproject_toml["tool"]["hatch"]["build"]["targets"]["
|
|
109
|
-
if "
|
|
110
|
-
hatch_sdist["
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
125
|
+
hatch_sdist = pyproject_toml["tool"]["hatch"]["build"]["targets"]["wheel"]
|
|
126
|
+
if "packages" in hatch_sdist:
|
|
127
|
+
hatch_sdist["packages"] = [package_name]
|
|
128
|
+
|
|
129
|
+
build_system = pyproject_toml["build-system"]
|
|
130
|
+
build_system["package"] = package_name
|
|
114
131
|
|
|
115
132
|
return pyproject_toml
|
datashare_python/utils.py
CHANGED
|
@@ -3,8 +3,10 @@ import contextlib
|
|
|
3
3
|
import contextvars
|
|
4
4
|
import inspect
|
|
5
5
|
import json
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
6
8
|
import threading
|
|
7
|
-
from collections.abc import Awaitable, Callable, Coroutine
|
|
9
|
+
from collections.abc import Awaitable, Callable, Coroutine, Iterable
|
|
8
10
|
from copy import deepcopy
|
|
9
11
|
from dataclasses import dataclass
|
|
10
12
|
from datetime import timedelta
|
|
@@ -24,7 +26,7 @@ from temporalio.common import RetryPolicy, SearchAttributeKey
|
|
|
24
26
|
from temporalio.exceptions import ApplicationError
|
|
25
27
|
|
|
26
28
|
from .constants import METADATA_JSON
|
|
27
|
-
from .objects import DocArtifact
|
|
29
|
+
from .objects import DocArtifact, DocumentLocation, FilesystemDocument
|
|
28
30
|
from .types_ import ProgressRateHandler, RawProgressHandler
|
|
29
31
|
|
|
30
32
|
DependencyLabel = str | None
|
|
@@ -83,7 +85,6 @@ class WorkflowWithProgress:
|
|
|
83
85
|
async def update_progress(self, signal: ProgressSignal) -> None:
|
|
84
86
|
async with self._update_lock:
|
|
85
87
|
# TODO: remove this log
|
|
86
|
-
workflow.logger.debug("recording progress signal %s", signal)
|
|
87
88
|
key = (signal.run_id, signal.activity_id)
|
|
88
89
|
self._progress[key] = signal.to_progress()
|
|
89
90
|
progress = sum(p.current for p in self._progress.values())
|
|
@@ -480,11 +481,17 @@ def write_artifact(root: Path, artifact: DocArtifact) -> Path:
|
|
|
480
481
|
# TODO: if transcriptions are too large we could also serialize them
|
|
481
482
|
# as jsonl
|
|
482
483
|
artifact_path: Path = artif_dir / artifact.filename
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
484
|
+
match artifact.artifact:
|
|
485
|
+
case bytes():
|
|
486
|
+
artifact_path.write_bytes(artifact.artifact)
|
|
487
|
+
case BytesIO():
|
|
488
|
+
with artifact_path.open("wb") as f:
|
|
489
|
+
f.write(artifact.artifact.read())
|
|
490
|
+
case Path():
|
|
491
|
+
shutil.move(artifact.artifact, artifact_path)
|
|
492
|
+
case _:
|
|
493
|
+
msg = f"unsupported artifact type: {artifact.artifact.__class__.__name__}"
|
|
494
|
+
raise ValueError(msg)
|
|
488
495
|
meta_path = root / _metadata_path(artifact.doc_id, project=artifact.project)
|
|
489
496
|
meta = _read_artifact_metadata(root, artifact) if meta_path.exists() else dict()
|
|
490
497
|
meta[artifact.metadata_key] = artifact.filename
|
|
@@ -557,3 +564,40 @@ def activity_workdir(
|
|
|
557
564
|
wf_context=wf_context, act_context=act_context, run_context=run_context
|
|
558
565
|
)
|
|
559
566
|
return workdir.joinpath(project, ctx_path)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def symlink_embedded_document_to_workdir(
|
|
570
|
+
doc: FilesystemDocument, artifacts_root: Path, *, workdir: Path
|
|
571
|
+
) -> FilesystemDocument:
|
|
572
|
+
match doc.location:
|
|
573
|
+
case DocumentLocation.ARTIFACTS:
|
|
574
|
+
symlinks_dir = workdir / doc.index / "symlinks"
|
|
575
|
+
symlinks_dir.mkdir(parents=True, exist_ok=True)
|
|
576
|
+
symlink_path = Path(*doc.path.parts[:-1], doc.id)
|
|
577
|
+
# Replace the "raw" with the doc id
|
|
578
|
+
doc_ext = Path(doc.resource_name).suffix
|
|
579
|
+
symlink_path = symlink_path.relative_to(Path(doc.index))
|
|
580
|
+
symlink_path = symlinks_dir / f"{symlink_path}{doc_ext}"
|
|
581
|
+
symlink_path.parent.mkdir(parents=True, exist_ok=True)
|
|
582
|
+
artifact_path = artifacts_root / doc.path
|
|
583
|
+
with contextlib.suppress(FileExistsError):
|
|
584
|
+
os.symlink(artifact_path, symlink_path)
|
|
585
|
+
return FilesystemDocument(
|
|
586
|
+
path=symlink_path.relative_to(workdir),
|
|
587
|
+
id=doc.id,
|
|
588
|
+
location=DocumentLocation.WORKDIR,
|
|
589
|
+
index=doc.index,
|
|
590
|
+
resource_name=doc.resource_name,
|
|
591
|
+
)
|
|
592
|
+
case DocumentLocation.ORIGINAL:
|
|
593
|
+
return doc
|
|
594
|
+
case _:
|
|
595
|
+
raise ValueError(f"unsupported location {doc.location}")
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def read_jsonl(path: Path) -> Iterable[dict]:
|
|
599
|
+
with path.open() as f:
|
|
600
|
+
for line in f:
|
|
601
|
+
line = line.strip() # noqa: PLW2901
|
|
602
|
+
if line:
|
|
603
|
+
yield json.loads(line)
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datashare-python
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.23
|
|
4
4
|
Summary: Manage Python tasks and local resources in Datashare
|
|
5
5
|
Project-URL: Homepage, https://icij.github.io/datashare-python/
|
|
6
6
|
Project-URL: Documentation, https://icij.github.io/datashare-python/
|
|
@@ -12,8 +12,10 @@ Requires-Dist: aiohttp~=3.11
|
|
|
12
12
|
Requires-Dist: alive-progress~=3.2
|
|
13
13
|
Requires-Dist: hatchling~=1.27
|
|
14
14
|
Requires-Dist: icij-common[elasticsearch]~=0.8.2
|
|
15
|
+
Requires-Dist: langcodes~=3.5
|
|
15
16
|
Requires-Dist: nest-asyncio~=1.6
|
|
16
17
|
Requires-Dist: orjson~=3.11
|
|
18
|
+
Requires-Dist: pydantic-extra-types[pycountry]>=2.11.1
|
|
17
19
|
Requires-Dist: python-json-logger~=4.0
|
|
18
20
|
Requires-Dist: pyyaml~=6.0
|
|
19
21
|
Requires-Dist: temporalio~=1.23
|
|
@@ -2,26 +2,26 @@ datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,2
|
|
|
2
2
|
datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
|
|
4
4
|
datashare_python/config.py,sha256=bQ9N7nxFnFCQ4RefDHjS06H-JlG3tWPVbnlVqjm2zGw,4566
|
|
5
|
-
datashare_python/conftest.py,sha256=
|
|
5
|
+
datashare_python/conftest.py,sha256=kqpvHJg9_-xYKvxFO8NIVWhr-VS5jj98GUx-SnxJksY,8849
|
|
6
6
|
datashare_python/constants.py,sha256=a8-ceZKBVMXydcoNQ35fSjFjxeJ7dt-N6eAvqtPpf9g,320
|
|
7
7
|
datashare_python/dependencies.py,sha256=5aM6v_WUfIn6SmYMEECh8os1aZOJjuOOCIuk-Y7a6Ak,4139
|
|
8
8
|
datashare_python/discovery.py,sha256=dKCZFTa3R-Weg4LL39wL85vstsol6tTh3g0jbpeOh4E,7122
|
|
9
9
|
datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
|
|
10
10
|
datashare_python/interceptors.py,sha256=Pl7GodPO4KbfflmacpW-vOUgLazjlXSlDNENbpOUt1c,6725
|
|
11
11
|
datashare_python/logging_.py,sha256=XUnZ0LB6pfuijvmAlWboDuAf4Itf-qluT7W8k4tadkM,4617
|
|
12
|
-
datashare_python/objects.py,sha256=
|
|
12
|
+
datashare_python/objects.py,sha256=567AgpYo1nCZ37vuCFIDyk4xneCyHA5AEbr4vtX2cr4,10972
|
|
13
13
|
datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
|
|
14
|
-
datashare_python/template.py,sha256=
|
|
14
|
+
datashare_python/template.py,sha256=N1uM8TRXh5tQPV7BC1oJj-Pdm2Gs_SkjaXv50RnNYso,4682
|
|
15
15
|
datashare_python/types_.py,sha256=9Hk1XqpdXbM1TnEzwvJ5G9ABbaCZW9KgBTtiPBVn_7k,649
|
|
16
|
-
datashare_python/utils.py,sha256=
|
|
17
|
-
datashare_python/worker-template.tar.gz,sha256=
|
|
16
|
+
datashare_python/utils.py,sha256=9xc9oOeE6qeBFC0ExvtR2jFSGpKe2qy9BbhHxb6EGvU,20757
|
|
17
|
+
datashare_python/worker-template.tar.gz,sha256=_IgunEnO5ix-7-1bZh8AYWuT086et4nOHSaH0s2owvY,276391
|
|
18
18
|
datashare_python/worker.py,sha256=czrN9Z0fPFX-6KHinX8Orx4vb9tpta2e7Qs6H0NiYyE,7534
|
|
19
19
|
datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
|
|
20
20
|
datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
|
|
21
21
|
datashare_python/cli/task.py,sha256=8mvKGS21bZ14BgZ0Uo-dfameljkaI2ZBha80ywCy-E8,5822
|
|
22
22
|
datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
|
|
23
23
|
datashare_python/cli/worker.py,sha256=eIMgUatSmpQpeu0MSMMKBrmx3GWm_eQ-aLfVrgl513o,5773
|
|
24
|
-
datashare_python-0.8.
|
|
25
|
-
datashare_python-0.8.
|
|
26
|
-
datashare_python-0.8.
|
|
27
|
-
datashare_python-0.8.
|
|
24
|
+
datashare_python-0.8.23.dist-info/METADATA,sha256=uWHZ4hXJvDCgHBy6eHr25moaGh1ASq2yGQFTA_CqRrA,1015
|
|
25
|
+
datashare_python-0.8.23.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
26
|
+
datashare_python-0.8.23.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
|
|
27
|
+
datashare_python-0.8.23.dist-info/RECORD,,
|
|
File without changes
|