datashare-python 0.9.9__py3-none-any.whl → 0.10.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datashare_python/cli/worker.py +1 -1
- datashare_python/config.py +17 -7
- datashare_python/conftest.py +176 -44
- datashare_python/constants.py +0 -2
- datashare_python/dependencies.py +0 -16
- datashare_python/mimetypes_.py +1105 -0
- datashare_python/objects.py +95 -85
- datashare_python/utils.py +255 -40
- datashare_python/worker-template.tar.gz +0 -0
- datashare_python/worker.py +14 -3
- {datashare_python-0.9.9.dist-info → datashare_python-0.10.0.dev0.dist-info}/METADATA +3 -2
- datashare_python-0.10.0.dev0.dist-info/RECORD +28 -0
- datashare_python-0.9.9.dist-info/RECORD +0 -27
- {datashare_python-0.9.9.dist-info → datashare_python-0.10.0.dev0.dist-info}/WHEEL +0 -0
- {datashare_python-0.9.9.dist-info → datashare_python-0.10.0.dev0.dist-info}/entry_points.txt +0 -0
datashare_python/utils.py
CHANGED
|
@@ -3,22 +3,34 @@ import contextlib
|
|
|
3
3
|
import fcntl
|
|
4
4
|
import inspect
|
|
5
5
|
import json
|
|
6
|
+
import logging
|
|
6
7
|
import os
|
|
7
8
|
import shutil
|
|
9
|
+
import sys
|
|
8
10
|
import threading
|
|
9
11
|
import time
|
|
10
|
-
from collections.abc import
|
|
12
|
+
from collections.abc import (
|
|
13
|
+
AsyncIterable,
|
|
14
|
+
Awaitable,
|
|
15
|
+
Callable,
|
|
16
|
+
Coroutine,
|
|
17
|
+
Generator,
|
|
18
|
+
Iterable,
|
|
19
|
+
Sequence,
|
|
20
|
+
)
|
|
11
21
|
from copy import deepcopy
|
|
12
22
|
from dataclasses import dataclass
|
|
13
23
|
from datetime import timedelta
|
|
14
|
-
from functools import wraps
|
|
24
|
+
from functools import cache, wraps
|
|
15
25
|
from hashlib import sha256
|
|
16
26
|
from io import BytesIO
|
|
17
27
|
from pathlib import Path
|
|
18
|
-
from typing import Any, ParamSpec, TypeVar
|
|
28
|
+
from typing import Any, ParamSpec, Self, TypeVar
|
|
19
29
|
from uuid import uuid4
|
|
20
30
|
|
|
21
31
|
import temporalio
|
|
32
|
+
from aiofile import async_open
|
|
33
|
+
from lru import LRU
|
|
22
34
|
from pydantic import ValidationError
|
|
23
35
|
from temporalio import activity, workflow
|
|
24
36
|
from temporalio.api.common.v1 import Payload
|
|
@@ -40,9 +52,18 @@ from datashare_python.types_ import (
|
|
|
40
52
|
)
|
|
41
53
|
|
|
42
54
|
from .constants import MANIFEST_JSON, METADATA_JSON
|
|
43
|
-
from .
|
|
55
|
+
from .mimetypes_ import types_map
|
|
56
|
+
from .objects import (
|
|
57
|
+
BaseModel,
|
|
58
|
+
DocArtifact,
|
|
59
|
+
DocumentLocation,
|
|
60
|
+
ProcessedFile,
|
|
61
|
+
WorkerPaths,
|
|
62
|
+
)
|
|
44
63
|
from .types_ import RawAsyncProgressHandler
|
|
45
64
|
|
|
65
|
+
logger = logging.getLogger(__name__)
|
|
66
|
+
|
|
46
67
|
DependencyLabel = str | None
|
|
47
68
|
DependencySetup = Callable[..., None]
|
|
48
69
|
DependencyAsyncSetup = Callable[..., Coroutine[None, None, None]]
|
|
@@ -134,7 +155,7 @@ def _retry_policy_with_default(retry_policy: RetryPolicy | None) -> RetryPolicy:
|
|
|
134
155
|
|
|
135
156
|
|
|
136
157
|
async def execute_activity(
|
|
137
|
-
activity: Callable,
|
|
158
|
+
activity: Callable | str,
|
|
138
159
|
task_queue: str,
|
|
139
160
|
arg: Any = temporalio.common._arg_unset,
|
|
140
161
|
*,
|
|
@@ -164,7 +185,7 @@ def positional_args_only(activity_fn: Callable[P, T]) -> Callable[P, T]:
|
|
|
164
185
|
params = list(sig.parameters.values())
|
|
165
186
|
keyword_only = {p.name for p in params if p.kind == inspect.Parameter.KEYWORD_ONLY}
|
|
166
187
|
|
|
167
|
-
if
|
|
188
|
+
if inspect.iscoroutinefunction(activity_fn):
|
|
168
189
|
|
|
169
190
|
@wraps(activity_fn)
|
|
170
191
|
async def wrapper(*args, **kwargs) -> T:
|
|
@@ -211,7 +232,7 @@ def with_retriables(
|
|
|
211
232
|
if retriables is None:
|
|
212
233
|
|
|
213
234
|
def decorator(activity_fn: Callable[P, T]) -> Callable[P, T]:
|
|
214
|
-
if
|
|
235
|
+
if inspect.iscoroutinefunction(activity_fn):
|
|
215
236
|
|
|
216
237
|
@wraps(activity_fn)
|
|
217
238
|
async def wrapper(*args, **kwargs) -> T:
|
|
@@ -233,7 +254,7 @@ def with_retriables(
|
|
|
233
254
|
return decorator
|
|
234
255
|
|
|
235
256
|
def decorator(activity_fn: Callable[P, T]) -> Callable[P, T]:
|
|
236
|
-
if
|
|
257
|
+
if inspect.iscoroutinefunction(activity_fn):
|
|
237
258
|
|
|
238
259
|
@wraps(activity_fn)
|
|
239
260
|
async def wrapper(*args, **kwargs) -> T:
|
|
@@ -269,7 +290,7 @@ def activity_defn(
|
|
|
269
290
|
activity_fn = with_retriables(retriables)(activity_fn)
|
|
270
291
|
activity_fn = activity.defn(activity_fn, name=name)
|
|
271
292
|
|
|
272
|
-
is_async =
|
|
293
|
+
is_async = inspect.iscoroutinefunction(activity_fn)
|
|
273
294
|
if is_async:
|
|
274
295
|
|
|
275
296
|
@wraps(activity_fn)
|
|
@@ -304,6 +325,20 @@ def to_raw_async_progress(
|
|
|
304
325
|
return raw
|
|
305
326
|
|
|
306
327
|
|
|
328
|
+
def to_incremental_async_progress(
|
|
329
|
+
progress: RawAsyncProgressHandler,
|
|
330
|
+
) -> RawAsyncProgressHandler:
|
|
331
|
+
|
|
332
|
+
offset = 0
|
|
333
|
+
|
|
334
|
+
async def incremental(p: int) -> None:
|
|
335
|
+
nonlocal offset
|
|
336
|
+
offset += p
|
|
337
|
+
await progress(offset)
|
|
338
|
+
|
|
339
|
+
return incremental
|
|
340
|
+
|
|
341
|
+
|
|
307
342
|
def to_raw_sync_progress(
|
|
308
343
|
progress: SyncProgressRateHandler, max_progress: int
|
|
309
344
|
) -> RawSyncProgressHandler:
|
|
@@ -395,7 +430,7 @@ def write_artifact(
|
|
|
395
430
|
)
|
|
396
431
|
manifest[artifact.type] = manifest_entry
|
|
397
432
|
manifest_path.write_text(json.dumps(manifest))
|
|
398
|
-
return artifact_path.relative_to(
|
|
433
|
+
return artifact_path.relative_to(root)
|
|
399
434
|
|
|
400
435
|
|
|
401
436
|
@contextlib.contextmanager
|
|
@@ -412,23 +447,29 @@ def artifact_lock(
|
|
|
412
447
|
artifacts_dir: Path, lock_timeout_ms: int
|
|
413
448
|
) -> Generator[None, None, None]:
|
|
414
449
|
lock_path = artifacts_dir / f"{MANIFEST_JSON}.lock"
|
|
450
|
+
acquired = False
|
|
415
451
|
fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY)
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
452
|
+
try:
|
|
453
|
+
with _ARTIFACT_LOCK, _set_event():
|
|
454
|
+
start = time.time()
|
|
455
|
+
timeout_s = lock_timeout_ms / 1000
|
|
456
|
+
while time.time() - start < timeout_s:
|
|
457
|
+
try:
|
|
458
|
+
_, acquired = fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB), True
|
|
459
|
+
except OSError:
|
|
460
|
+
# TODO: we shouldn't block here, use asyncio instead
|
|
461
|
+
time.sleep(0.02)
|
|
462
|
+
continue
|
|
463
|
+
yield
|
|
464
|
+
return
|
|
465
|
+
msg = (
|
|
466
|
+
f"failed to acquire lock on {lock_path} in less than "
|
|
467
|
+
f"{lock_timeout_ms} ms."
|
|
468
|
+
)
|
|
469
|
+
raise TimeoutError(msg)
|
|
470
|
+
finally:
|
|
471
|
+
if acquired:
|
|
472
|
+
lock_path.unlink(missing_ok=True)
|
|
432
473
|
|
|
433
474
|
|
|
434
475
|
def _read_manifest_backward_compatible(
|
|
@@ -526,35 +567,44 @@ def activity_workdir(
|
|
|
526
567
|
|
|
527
568
|
|
|
528
569
|
def symlink_embedded_document_to_workdir(
|
|
529
|
-
doc:
|
|
530
|
-
) ->
|
|
570
|
+
doc: ProcessedFile, paths: WorkerPaths
|
|
571
|
+
) -> ProcessedFile:
|
|
531
572
|
match doc.location:
|
|
532
573
|
case DocumentLocation.ARTIFACTS:
|
|
533
|
-
symlinks_dir = workdir / doc.
|
|
574
|
+
symlinks_dir = paths.workdir / doc.project / "symlinks"
|
|
534
575
|
symlinks_dir.mkdir(parents=True, exist_ok=True)
|
|
535
|
-
symlink_path = Path(*doc.path.parts[:-1], doc.
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
symlink_path = symlink_path.relative_to(Path(doc.index))
|
|
539
|
-
symlink_path = symlinks_dir / f"{symlink_path}{doc_ext}"
|
|
576
|
+
symlink_path = Path(*doc.path.parts[1:-1], doc.resource_name)
|
|
577
|
+
symlink_path = symlinks_dir / symlink_path
|
|
578
|
+
# Replace the "raw" with the resource name
|
|
540
579
|
symlink_path.parent.mkdir(parents=True, exist_ok=True)
|
|
541
|
-
artifact_path =
|
|
580
|
+
artifact_path = paths.artifacts / doc.path
|
|
542
581
|
with contextlib.suppress(FileExistsError):
|
|
543
582
|
os.symlink(artifact_path, symlink_path)
|
|
544
|
-
return
|
|
545
|
-
path=symlink_path.relative_to(workdir),
|
|
583
|
+
return ProcessedFile(
|
|
584
|
+
path=symlink_path.relative_to(paths.workdir),
|
|
546
585
|
id=doc.id,
|
|
547
586
|
location=DocumentLocation.WORKDIR,
|
|
548
|
-
|
|
587
|
+
project=doc.project,
|
|
549
588
|
resource_name=doc.resource_name,
|
|
589
|
+
n_pages=doc.n_pages,
|
|
550
590
|
)
|
|
551
|
-
case DocumentLocation.
|
|
591
|
+
case DocumentLocation.FILESYSTEM:
|
|
552
592
|
return doc
|
|
553
593
|
case _:
|
|
554
594
|
raise ValueError(f"unsupported location {doc.location}")
|
|
555
595
|
|
|
556
596
|
|
|
557
|
-
def
|
|
597
|
+
def artifact_path(
|
|
598
|
+
doc_id: str,
|
|
599
|
+
artifact_type: type[DocArtifact],
|
|
600
|
+
*,
|
|
601
|
+
project: str,
|
|
602
|
+
root: Path,
|
|
603
|
+
) -> Path:
|
|
604
|
+
return root / artifacts_dir(doc_id, project=project) / artifact_type.filename
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _read_jsonl(path: Path) -> Iterable[dict]:
|
|
558
608
|
with path.open() as f:
|
|
559
609
|
for line in f:
|
|
560
610
|
line = line.strip() # noqa: PLW2901
|
|
@@ -562,6 +612,77 @@ def read_jsonl(path: Path) -> Iterable[dict]:
|
|
|
562
612
|
yield json.loads(line)
|
|
563
613
|
|
|
564
614
|
|
|
615
|
+
M = TypeVar("M", bound=BaseModel)
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def read_jsonl_as(path: Path, cls: type[M]) -> Iterable[M]:
|
|
619
|
+
return (cls.model_validate(d) for d in _read_jsonl(path))
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
async def async_read_jsonl_as(
|
|
623
|
+
path: Path, processed_file_cls: type[M]
|
|
624
|
+
) -> AsyncIterable[M]:
|
|
625
|
+
async with async_open(path, "r") as f:
|
|
626
|
+
async for line in f:
|
|
627
|
+
line = line.strip() # noqa: PLW2901
|
|
628
|
+
if line:
|
|
629
|
+
yield processed_file_cls.model_validate_json(line)
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
@cache
|
|
633
|
+
def ext_to_mime_types(ext: str) -> set[str]:
|
|
634
|
+
# All particular cases
|
|
635
|
+
match ext:
|
|
636
|
+
case ".nxml" | ".dclg" | ".dclg_xml":
|
|
637
|
+
return ext_to_mime_types(".xml")
|
|
638
|
+
case ".qmd" | ".rmd":
|
|
639
|
+
return ext_to_mime_types(".md")
|
|
640
|
+
case ".xbrl":
|
|
641
|
+
return ext_to_mime_types(".html")
|
|
642
|
+
case ".j2c" | ".j2k" | ".jpc" | ".jpf" | ".jpx":
|
|
643
|
+
return ext_to_mime_types(".jp2")
|
|
644
|
+
case ".rgba":
|
|
645
|
+
return ext_to_mime_types(".png")
|
|
646
|
+
case ".svm":
|
|
647
|
+
return ext_to_mime_types(".wmf")
|
|
648
|
+
case ".uof" | ".uop" | ".uos" | ".uot":
|
|
649
|
+
return ext_to_mime_types(".xml")
|
|
650
|
+
case ".dclg.xml":
|
|
651
|
+
return ext_to_mime_types(".xml")
|
|
652
|
+
try:
|
|
653
|
+
return types_map()[ext]
|
|
654
|
+
except KeyError as e:
|
|
655
|
+
raise ValueError(f"unsupported mimetype {ext}") from e
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
async def publish_and_consume(
|
|
659
|
+
publisher: asyncio.Task,
|
|
660
|
+
publisher_completion_callback: Callable[[], None],
|
|
661
|
+
*,
|
|
662
|
+
consumer: asyncio.Task,
|
|
663
|
+
) -> tuple[Any, Any]:
|
|
664
|
+
# Publish and consume concurrently
|
|
665
|
+
logger.debug("starting publish and subscribe")
|
|
666
|
+
done, pending = await asyncio.wait(
|
|
667
|
+
[publisher, consumer], return_when=asyncio.FIRST_COMPLETED
|
|
668
|
+
)
|
|
669
|
+
for d in done:
|
|
670
|
+
# Stop everything case of exception
|
|
671
|
+
exc = d.exception()
|
|
672
|
+
if exc:
|
|
673
|
+
for p in pending:
|
|
674
|
+
p.cancel()
|
|
675
|
+
raise exc
|
|
676
|
+
# Wait for publish to be done and push the poison pill to stop consuming
|
|
677
|
+
p_res = await publisher
|
|
678
|
+
publisher_completion_callback()
|
|
679
|
+
logger.debug("done publishing, waiting for consumer to complete...")
|
|
680
|
+
# Wait for consumption to be done
|
|
681
|
+
c_res = await consumer
|
|
682
|
+
logger.debug("done consuming !")
|
|
683
|
+
return p_res, c_res
|
|
684
|
+
|
|
685
|
+
|
|
565
686
|
class _PydanticPayloadConverter(CompositePayloadConverter):
|
|
566
687
|
def __init__(self) -> None:
|
|
567
688
|
json_payload_converter = PydanticJSONPlainPayloadConverter(
|
|
@@ -594,3 +715,97 @@ class _PydanticPayloadConverter(CompositePayloadConverter):
|
|
|
594
715
|
PYDANTIC_DATA_CONVERTER = DataConverter(
|
|
595
716
|
payload_converter_class=_PydanticPayloadConverter
|
|
596
717
|
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
class SharedResources:
|
|
721
|
+
def __init__(
|
|
722
|
+
self,
|
|
723
|
+
cache_size: int = 1,
|
|
724
|
+
eviction_callback: Callable[[Any, Any], None] | None = None,
|
|
725
|
+
) -> None:
|
|
726
|
+
self._eviction_callback = eviction_callback
|
|
727
|
+
self._cache = LRU(cache_size, self._eviction_callback)
|
|
728
|
+
self._sentinel = object()
|
|
729
|
+
|
|
730
|
+
def __enter__(self) -> Self:
|
|
731
|
+
return self
|
|
732
|
+
|
|
733
|
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None: # noqa: ANN001
|
|
734
|
+
if self._eviction_callback is not None:
|
|
735
|
+
for k, v in self._cache.items():
|
|
736
|
+
self._eviction_callback(k, v)
|
|
737
|
+
|
|
738
|
+
def get_or_cache_resource(
|
|
739
|
+
self, key: str, default_factory: Callable[[], Any]
|
|
740
|
+
) -> Any:
|
|
741
|
+
value = self._cache.get(key, self._sentinel)
|
|
742
|
+
if value is not self._sentinel:
|
|
743
|
+
return value
|
|
744
|
+
# Get the value first to be sure to return the right one in case of concurrent
|
|
745
|
+
# access.
|
|
746
|
+
#
|
|
747
|
+
# Additionally, the factory can be long to complete (that's one of the reason
|
|
748
|
+
# we want to cache its results). Because we don't lock the factory call,
|
|
749
|
+
# together with the cache update, in case of concurrent access,
|
|
750
|
+
# quicker factories will complete first and will stay longer in the cache.
|
|
751
|
+
# This is fine, the alternative is to lock factory + cache update to avoid
|
|
752
|
+
# concurrent access but this will mean waiting for the first factory call to
|
|
753
|
+
# complete, this can potentially imply longer waits than no getting the value
|
|
754
|
+
# from the cache
|
|
755
|
+
value = default_factory()
|
|
756
|
+
self._cache[key] = value
|
|
757
|
+
return value
|
|
758
|
+
|
|
759
|
+
async def async_get_or_cache_resource(
|
|
760
|
+
self, key: str, default_factory: Callable[[], Awaitable[Any]]
|
|
761
|
+
) -> Any:
|
|
762
|
+
value = self._cache.get(key, self._sentinel)
|
|
763
|
+
if value is not self._sentinel:
|
|
764
|
+
return value
|
|
765
|
+
# Get the value first to be sure to return the right one in case of concurrent
|
|
766
|
+
# access.
|
|
767
|
+
#
|
|
768
|
+
# Additionally, the factory can be long to complete (that's one of the reason
|
|
769
|
+
# we want to cache its results). Because we don't lock the factory call,
|
|
770
|
+
# together with the cache update, in case of concurrent access,
|
|
771
|
+
# quicker factories will complete first and will stay longer in the cache.
|
|
772
|
+
# This is fine, the alternative is to lock factory + cache update to avoid
|
|
773
|
+
# concurrent access but this will mean waiting for the first factory call to
|
|
774
|
+
# complete, this can potentially imply longer waits than no getting the value
|
|
775
|
+
# from the cache
|
|
776
|
+
value = await default_factory()
|
|
777
|
+
self._cache[key] = value
|
|
778
|
+
return value
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def close_cm_callback(key: str, value: Any) -> None: # noqa: ARG001
|
|
782
|
+
if hasattr(value, "__exit__"):
|
|
783
|
+
value.__exit__(*sys.exc_info())
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def enter_cm(
|
|
787
|
+
cm_factory: Callable[[], contextlib.AbstractContextManager],
|
|
788
|
+
) -> Callable[[], contextlib.AbstractContextManager]: # noqa: ARG001
|
|
789
|
+
@wraps(cm_factory)
|
|
790
|
+
def factory() -> contextlib.AbstractContextManager:
|
|
791
|
+
cm = cm_factory()
|
|
792
|
+
cm.__enter__()
|
|
793
|
+
return cm
|
|
794
|
+
|
|
795
|
+
return factory
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def async_enter_cm(
|
|
799
|
+
cm_factory: Callable[[], contextlib.AbstractAsyncContextManager],
|
|
800
|
+
) -> Callable[[], Awaitable[contextlib.AbstractAsyncContextManager]]: # noqa: ARG001
|
|
801
|
+
@wraps(cm_factory)
|
|
802
|
+
async def wrapper() -> contextlib.AbstractAsyncContextManager:
|
|
803
|
+
cm = cm_factory()
|
|
804
|
+
await cm.__aenter__()
|
|
805
|
+
return cm
|
|
806
|
+
|
|
807
|
+
return wrapper
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def config_cache_key(config: BaseModel) -> str:
|
|
811
|
+
return str(hash(config))
|
|
Binary file
|
datashare_python/worker.py
CHANGED
|
@@ -61,6 +61,16 @@ class DatashareWorker(Worker):
|
|
|
61
61
|
raise ValueError("worker is not running")
|
|
62
62
|
await self._async_context_run_task
|
|
63
63
|
|
|
64
|
+
async def __aexit__(self, exc_type: type[BaseException] | None, *args: Any) -> None:
|
|
65
|
+
await super().__aexit__(exc_type, *args)
|
|
66
|
+
logger.info("worker %s exited", self._config["identity"])
|
|
67
|
+
|
|
68
|
+
async def __aenter__(self) -> "DatashareWorker":
|
|
69
|
+
await super().__aenter__()
|
|
70
|
+
# TODO: remove me
|
|
71
|
+
logger.info("worker %s started working", self._config["identity"])
|
|
72
|
+
return self
|
|
73
|
+
|
|
64
74
|
|
|
65
75
|
def datashare_worker(
|
|
66
76
|
client: TemporalClient,
|
|
@@ -71,7 +81,8 @@ def datashare_worker(
|
|
|
71
81
|
task_queue: str,
|
|
72
82
|
# Scale horizontally be default for activities, each worker processes one activity
|
|
73
83
|
# at a time
|
|
74
|
-
|
|
84
|
+
max_concurrent_activities: int = 1,
|
|
85
|
+
max_activities_per_second: float = 20.0,
|
|
75
86
|
sandboxed: bool = True,
|
|
76
87
|
) -> DatashareWorker:
|
|
77
88
|
if workflows is None:
|
|
@@ -91,7 +102,6 @@ def datashare_worker(
|
|
|
91
102
|
)
|
|
92
103
|
logger.warning(_SEPARATE_IO_AND_CPU_ACTIVITIES)
|
|
93
104
|
|
|
94
|
-
max_concurrent_activities = max_concurrent_io_activities
|
|
95
105
|
if isinstance(activity_executor, ThreadPoolExecutor):
|
|
96
106
|
max_concurrent_activities = 1
|
|
97
107
|
if workflows:
|
|
@@ -118,6 +128,7 @@ def datashare_worker(
|
|
|
118
128
|
# several of them
|
|
119
129
|
workflow_task_poller_behavior=PollerBehaviorSimpleMaximum(5),
|
|
120
130
|
workflow_runner=wf_runner,
|
|
131
|
+
max_activities_per_second=max_activities_per_second,
|
|
121
132
|
)
|
|
122
133
|
|
|
123
134
|
|
|
@@ -204,7 +215,7 @@ async def worker_context(
|
|
|
204
215
|
workflows=workflows,
|
|
205
216
|
activities=acts,
|
|
206
217
|
task_queue=task_queue,
|
|
207
|
-
|
|
218
|
+
max_concurrent_activities=worker_config.max_concurrent_activities,
|
|
208
219
|
sandboxed=sandboxed,
|
|
209
220
|
)
|
|
210
221
|
async with worker:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datashare-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0.dev0
|
|
4
4
|
Summary: Manage Python tasks and local resources in Datashare
|
|
5
5
|
Project-URL: Homepage, https://icij.github.io/datashare-python/
|
|
6
6
|
Project-URL: Documentation, https://icij.github.io/datashare-python/
|
|
@@ -8,6 +8,7 @@ Project-URL: Repository, https://github.com/ICIJ/datashare-python
|
|
|
8
8
|
Project-URL: Issues, https://github.com/ICIJ/datashare-python/issues
|
|
9
9
|
Author-email: Clément Doumouro <cdoumouro@icij.org>, Clément Doumouro <clement.doumouro@gmail.com>, Lion Summerbell <lsummerbell@icij.org>
|
|
10
10
|
Requires-Python: <4,>=3.11
|
|
11
|
+
Requires-Dist: aiofile~=3.11
|
|
11
12
|
Requires-Dist: aiohttp~=3.11
|
|
12
13
|
Requires-Dist: alive-progress~=3.2
|
|
13
14
|
Requires-Dist: hatchling~=1.27
|
|
@@ -17,6 +18,6 @@ Requires-Dist: lru-dict~=1.4
|
|
|
17
18
|
Requires-Dist: pydantic-extra-types[pycountry]>=2.11.1
|
|
18
19
|
Requires-Dist: python-json-logger~=4.0
|
|
19
20
|
Requires-Dist: pyyaml~=6.0
|
|
20
|
-
Requires-Dist: temporalio~=1.
|
|
21
|
+
Requires-Dist: temporalio~=1.31
|
|
21
22
|
Requires-Dist: tomlkit~=0.14.0
|
|
22
23
|
Requires-Dist: typer<0.25.1,>=0.15.4
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,22
|
|
2
|
+
datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
|
|
4
|
+
datashare_python/config.py,sha256=V3LBXdxzAt8_UcHiPw6I4HN7RTo-Fns6xd50eZ8nHtQ,4116
|
|
5
|
+
datashare_python/conftest.py,sha256=-cEtlPEl6EagrxB5GTvjD92QQats3ItyFuIKkOKTf1s,13623
|
|
6
|
+
datashare_python/constants.py,sha256=-Irwd2bbnFGIkLVJi_SiQ7PM2eNW3Ia_co4SE6z4j-A,352
|
|
7
|
+
datashare_python/dependencies.py,sha256=Q-BIhCwWu-GGp9omYNhENZ88cF0QJ8spfaMZX40mmNU,4386
|
|
8
|
+
datashare_python/discovery.py,sha256=3mUck2GUSKu2DSP9-MM1OdVM4FqFKqfBeSQDQZxgh2g,7177
|
|
9
|
+
datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
|
|
10
|
+
datashare_python/interceptors.py,sha256=6ZJG7VJN5EHZkO6XD3nJyRCzCDb1Nh3x24VL1ip2Iyw,13314
|
|
11
|
+
datashare_python/logging_.py,sha256=96KNXhP-JR5ybu9dld4VLtF7v7PfiphOCR1Nio-mmUs,4483
|
|
12
|
+
datashare_python/mimetypes_.py,sha256=mjQgaqnSFZEcl3LoLDGBaBBjTTzwYfSNZND9Fmkk2zs,49682
|
|
13
|
+
datashare_python/objects.py,sha256=sSmXlsnj3PH_LZJwliUZcyXlVHbRHM7q19k_bLscAgs,16085
|
|
14
|
+
datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
|
|
15
|
+
datashare_python/template.py,sha256=N1uM8TRXh5tQPV7BC1oJj-Pdm2Gs_SkjaXv50RnNYso,4682
|
|
16
|
+
datashare_python/types_.py,sha256=80XTD481HxjbkS330pqHHmlsM-cxUyGGgcJnG8_vYR8,1134
|
|
17
|
+
datashare_python/utils.py,sha256=9RhgubRztlCdDPaAgmnbnMzwFkiCL_azt74yKFUhqjc,26627
|
|
18
|
+
datashare_python/worker-template.tar.gz,sha256=EdE9rOAFuy01ghQAvLf8M0pvxh0nCuIs40Z-3rnr0QQ,278668
|
|
19
|
+
datashare_python/worker.py,sha256=9BqtVTell9lhyHwdvpuYS_SNvxHGkS75HGhu1E_OhMM,8224
|
|
20
|
+
datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
|
|
21
|
+
datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
|
|
22
|
+
datashare_python/cli/task.py,sha256=GsyR5o0fN4Pd8tJKAUgivAvCBKT3L4Ay5j_BCW1XBHk,5828
|
|
23
|
+
datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
|
|
24
|
+
datashare_python/cli/worker.py,sha256=x0p3a1_NsZPbmV7-WSVMRyW3_iadYcgyZKbDvWgymQc,5790
|
|
25
|
+
datashare_python-0.10.0.dev0.dist-info/METADATA,sha256=3g8Tbq3QGjfHXMEp0YUop3xOCDSv7ptglIDrOmBHXW8,1017
|
|
26
|
+
datashare_python-0.10.0.dev0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
27
|
+
datashare_python-0.10.0.dev0.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
|
|
28
|
+
datashare_python-0.10.0.dev0.dist-info/RECORD,,
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,22
|
|
2
|
-
datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
|
|
4
|
-
datashare_python/config.py,sha256=1-uksky9GwmPh85E3JYt6X4hgjP68gUmZjZM7qf3VUI,3779
|
|
5
|
-
datashare_python/conftest.py,sha256=28mDi35cpGlH3gOofNWPzjPRp5FvZawE2JZTDVFnY6w,9184
|
|
6
|
-
datashare_python/constants.py,sha256=Fq8aX80SIznES2tC_Sj9I85ppq0p1wpfVmLXJAUd7EA,387
|
|
7
|
-
datashare_python/dependencies.py,sha256=JdKZX3TlbYQk1YLvscGEdXQNJ9TFjtbxmYwlxNyqGOg,4793
|
|
8
|
-
datashare_python/discovery.py,sha256=3mUck2GUSKu2DSP9-MM1OdVM4FqFKqfBeSQDQZxgh2g,7177
|
|
9
|
-
datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
|
|
10
|
-
datashare_python/interceptors.py,sha256=6ZJG7VJN5EHZkO6XD3nJyRCzCDb1Nh3x24VL1ip2Iyw,13314
|
|
11
|
-
datashare_python/logging_.py,sha256=96KNXhP-JR5ybu9dld4VLtF7v7PfiphOCR1Nio-mmUs,4483
|
|
12
|
-
datashare_python/objects.py,sha256=r-2qGpUR1u6X4484x8988Xbn8L8yjAg2kvhQgE68V1c,16193
|
|
13
|
-
datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
|
|
14
|
-
datashare_python/template.py,sha256=N1uM8TRXh5tQPV7BC1oJj-Pdm2Gs_SkjaXv50RnNYso,4682
|
|
15
|
-
datashare_python/types_.py,sha256=80XTD481HxjbkS330pqHHmlsM-cxUyGGgcJnG8_vYR8,1134
|
|
16
|
-
datashare_python/utils.py,sha256=5vheHeDvMKa6KESf8zYbIT4w69CLRF_ahD-CHq66R1w,19960
|
|
17
|
-
datashare_python/worker-template.tar.gz,sha256=nW0JEZJNXWFlPcMfnNT4Yw31P9wtTv2Nuv9kCb80g5s,282769
|
|
18
|
-
datashare_python/worker.py,sha256=cmqLRJ0_b1xm0qjjAZrRMOVXfhdrErimA_QY0xMtxRo,7774
|
|
19
|
-
datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
|
|
20
|
-
datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
|
|
21
|
-
datashare_python/cli/task.py,sha256=GsyR5o0fN4Pd8tJKAUgivAvCBKT3L4Ay5j_BCW1XBHk,5828
|
|
22
|
-
datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
|
|
23
|
-
datashare_python/cli/worker.py,sha256=eIMgUatSmpQpeu0MSMMKBrmx3GWm_eQ-aLfVrgl513o,5773
|
|
24
|
-
datashare_python-0.9.9.dist-info/METADATA,sha256=uEbimB84kKuvwFlje3KjnVARqGhFlZZziRrBCdV3_g8,982
|
|
25
|
-
datashare_python-0.9.9.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
26
|
-
datashare_python-0.9.9.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
|
|
27
|
-
datashare_python-0.9.9.dist-info/RECORD,,
|
|
File without changes
|
{datashare_python-0.9.9.dist-info → datashare_python-0.10.0.dev0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|