datashare-python 0.4.0__tar.gz → 0.6.0__tar.gz

{datashare_python-0.4.0 → datashare_python-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datashare-python
-Version: 0.4.0
+Version: 0.6.0
 Summary: Manage Pythoœn tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/cli/__init__.py

@@ -6,7 +6,6 @@ import typer
 from icij_common.logging_utils import setup_loggers
 
 import datashare_python
-from datashare_python.cli.local import local_app
 from datashare_python.cli.project import project_app
 from datashare_python.cli.task import task_app
 from datashare_python.cli.utils import AsyncTyper
@@ -16,7 +15,6 @@ cli_app = AsyncTyper(
     context_settings={"help_option_names": ["-h", "--help"]},
     pretty_exceptions_enable=False,
 )
-cli_app.add_typer(local_app)
 cli_app.add_typer(project_app)
 cli_app.add_typer(task_app)
 cli_app.add_typer(worker_app)

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/cli/worker.py

@@ -24,7 +24,6 @@ _LIST_ACTIVITY_NAMES_HELP = "activity names filters (supports regexes)"
 _START_WORKER_WORKFLOWS_HELP = "workflow names run by the worker (supports regexes)"
 _START_WORKER_ACTIVITIES_HELP = "activity names run by the worker (supports regexes)"
 _START_WORKER_DEPS_HELP = "worker lifetime dependencies name in the registry"
-_START_WORKER_CONFIG_HELP = "worker config class key the worker configs registry"
 _START_WORKER_WORKER_ID_PREFIX_HELP = "worker ID prefix"
 _START_WORKER_CONFIG_PATH_HELP = (
     "path to a worker config YAML file,"
@@ -86,9 +85,6 @@ async def start(
     dependencies: Annotated[
         str | None, typer.Option(help=_START_WORKER_DEPS_HELP)
     ] = None,
-    worker_config_name: Annotated[
-        str | None, typer.Option(help=_START_WORKER_CONFIG_HELP)
-    ] = None,
     config_path: Annotated[
         Path | None,
         typer.Option(
@@ -107,10 +103,7 @@ async def start(
     ] = None,
 ) -> None:
     registered_wfs, registered_acts, registered_deps, worker_config_cls = discover(
-        workflows,
-        act_names=activities,
-        deps_name=dependencies,
-        worker_config_name=worker_config_name,
+        workflows, act_names=activities, deps_name=dependencies
     )
     if config_path is not None:
         with config_path.open() as f:

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/conftest.py

@@ -4,6 +4,7 @@ from collections.abc import AsyncGenerator, Generator, Iterator, Sequence
 from pathlib import Path
 
 import aiohttp
+import nest_asyncio
 import pytest
 from elasticsearch._async.helpers import async_streaming_bulk
 from icij_common.es import DOC_ROOT_ID, ES_DOCUMENT_TYPE, ID, ESClient
@@ -200,6 +201,8 @@ def index_docs_ops(
         }
         doc = doc.model_dump(by_alias=True)  # noqa: PLW2901
         op.update(doc)
+        if "path" in op:
+            op["path"] = str(op["path"])
         op["_id"] = doc[ID]
         op["routing"] = doc[DOC_ROOT_ID]
         op["type"] = ES_DOCUMENT_TYPE
@@ -294,3 +297,8 @@ async def all_done(task_client: DatashareTaskClient, not_done: list[str]) -> boo
                 return False
             not_done.remove(t_id)
     return True
+
+
+@pytest.fixture  # noqa: F405
+def typer_asyncio_patch() -> None:
+    nest_asyncio.apply()

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/discovery.py

@@ -13,11 +13,11 @@ logger = logging.getLogger(__name__)
 Activity = ActivityWithProgress | Callable | type
 
 _DEPENDENCIES = "dependencies"
-_WORKER_CONFIGS = "worker_configs"
+_WORKER_CONFIG_CLS = "worker_config_cls"
 _WORKFLOW_GROUP = "datashare.workflows"
 _ACTIVITIES_GROUP = "datashare.activities"
 _DEPENDENCIES_GROUP = "datashare.dependencies"
-_WORKER_CONFIGS_GROUP = "datashare.worker_configs"
+_WORKER_CONFIG_CLS_GROUP = "datashare.worker_config_cls"
 
 _RegisteredWorkflow = tuple[str, type]
 _RegisteredActivity = tuple[str, Activity]
@@ -31,11 +31,7 @@ _Discovery = tuple[
 
 
 def discover(
-    wf_names: list[str] | None,
-    *,
-    act_names: list[str] | None,
-    deps_name: str | None,
-    worker_config_name: str | None,
+    wf_names: list[str] | None, *, act_names: list[str] | None, deps_name: str | None
 ) -> _Discovery:
     discovered = ""
     wfs = None
@@ -82,11 +78,8 @@ def discover(
             f"- {n_deps} dependenc{'ies' if n_deps > 1 else 'y'}:"
             f" {', '.join(deps_names)}"
         )
-    if worker_config_name is not None:
-        worker_config_cls = discover_worker_configs(worker_config_name)
-        discovered += f"- worker config class: {worker_config_cls}"
-    else:
-        worker_config_cls = WorkerConfig
+    worker_config_cls = discover_worker_config_cls()
+    discovered += f"- worker config class: {worker_config_cls}"
     logger.info("discovered:\n%s", discovered)
     return wfs, acts, deps, worker_config_cls
 
@@ -147,28 +140,15 @@ def discover_dependencies(name: str) -> _Dependencies:
         raise LookupError(msg) from e
 
 
-def discover_worker_configs(name: str) -> type[WorkerConfig]:
-    impls = entry_points(name=_WORKER_CONFIGS, group=_WORKER_CONFIGS_GROUP)
+def discover_worker_config_cls() -> type[WorkerConfig]:
+    impls = entry_points(name=_WORKER_CONFIG_CLS, group=_WORKER_CONFIG_CLS_GROUP)
     if not impls:
-        available_impls = entry_points(group=_WORKER_CONFIGS_GROUP)
-        msg = (
-            f'failed to find worker config: "{name}", '
-            f"available dependencies: {available_impls}"
-        )
-        raise LookupError(msg)
+        return WorkerConfig
     if len(impls) > 1:
-        msg = f'found multiple worker configs for name "{name}": {impls}'
+        msg = f'found multiple registered worker configs classes": {impls}'
         raise ValueError(msg)
-    deps_registry = impls[_WORKER_CONFIGS].load()
-    try:
-        return deps_registry[name]
-    except KeyError as e:
-        available = list(deps_registry)
-        msg = (
-            f'failed to find worker config for name "{name}", available worker '
-            f"configs: {available}"
-        )
-        raise LookupError(msg) from e
+    deps_registry = impls[_WORKER_CONFIG_CLS].load()
+    return deps_registry
 
 
 def _parse_wf_name(wf_type: type) -> str:

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/objects.py

@@ -3,6 +3,8 @@ from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from enum import StrEnum, unique
+from io import BytesIO
+from pathlib import Path
 from typing import Any, Literal, Self, TypeVar
 
 from temporalio import workflow
@@ -142,6 +144,7 @@ class Document(DatashareModel):
     language: str
     content: str | None = None
     content_type: str | None = None
+    path: Path | None = None
     tags: list[str] = Field(default_factory=list)
     content_translated: dict[str, str] = Field(
         default_factory=dict, alias="content_translated"
@@ -159,3 +162,12 @@ class Document(DatashareModel):
             root_document=sources[DOC_ROOT_ID],
             tags=sources.get("tags", []),
         )
+
+
+@dataclass(frozen=True)
+class DocArtifact:
+    project: str
+    doc_id: str
+    artifact: bytes | BytesIO
+    filename: str
+    metadata_key: str

{datashare_python-0.4.0 → datashare_python-0.6.0}/datashare_python/utils.py

@@ -13,6 +13,7 @@ from datetime import timedelta
 from functools import partial, wraps
 from hashlib import sha256
 from inspect import signature
+from io import BytesIO
 from pathlib import Path
 from typing import Any, ParamSpec, TypeVar
 from uuid import uuid4
@@ -34,6 +35,7 @@ from temporalio.common import RetryPolicy, SearchAttributeKey
 from temporalio.exceptions import ApplicationError
 
 from .constants import METADATA_JSON
+from .objects import DocArtifact
 from .types_ import ProgressRateHandler, RawProgressHandler
 
 DependencyLabel = str | None
@@ -431,49 +433,43 @@ class LogWithWorkerIDMixin:
         return handlers
 
 
-def safe_dir(filename: str) -> Path:
-    filename = filename.split(".", maxsplit=1)[0]
-    parts = (p for p in (filename[:2], filename[2:4]) if p)
+def safe_dir(doc_id: str) -> Path:
+    if len(doc_id) < 4:
+        raise ValueError(f"expected doc_id to be at least 4, found {doc_id}")
+    parts = (p for p in (doc_id[:2], doc_id[2:4]) if p)
     return Path(*parts)
 
 
-def artifacts_dir(project: str, *, filename: str) -> Path:
-    return Path(project, safe_dir(filename), filename)
+def _artifacts_dir(doc_id: str, *, project: str) -> Path:
+    return Path(project, safe_dir(doc_id), doc_id)
 
 
-def metadata_path(filename: str, *, project: str) -> Path:
-    metadata_path = artifacts_dir(project, filename=filename) / METADATA_JSON
+def _metadata_path(doc_id: str, *, project: str) -> Path:
+    metadata_path = _artifacts_dir(doc_id, project=project) / METADATA_JSON
     return metadata_path
 
 
-def _read_artifact_metadata(root: Path, project: str, *, filename: str) -> dict:
-    m_path = root / metadata_path(filename, project=project)
+def _read_artifact_metadata(root: Path, artifact: DocArtifact) -> dict:
+    m_path = root / _metadata_path(artifact.filename, project=artifact.project)
     return json.loads(m_path.read_text())
 
 
-def write_artifact(
-    artifact: bytes,
-    root: Path,
-    *,
-    project: str,
-    filename: str,
-    metadata_key: str,
-    metadata_value: str,
-) -> Path:
-    artif_dir = root / artifacts_dir(project, filename=filename)
+def write_artifact(root: Path, artifact: DocArtifact) -> Path:
+    artif_dir = root / _artifacts_dir(artifact.doc_id, project=artifact.project)
     artif_dir.mkdir(exist_ok=True, parents=True)
     # TODO: if transcriptions are too large we could also serialize them
     #  as jsonl
-    transcription_path = artif_dir / metadata_value
-    transcription_path.write_bytes(artifact)
-    try:
-        meta = _read_artifact_metadata(root, project, filename=filename)
-    except FileNotFoundError:
-        meta = dict()
-    meta[metadata_key] = metadata_value
-    meta_path = root / artifacts_dir(project, filename=filename) / METADATA_JSON
+    artifact_path: Path = artif_dir / artifact.filename
+    if isinstance(artifact.artifact, bytes):
+        artifact_path.write_bytes(artifact.artifact)
+    elif isinstance(artifact_path, BytesIO):
+        with artifact_path.open("wb") as f:
+            f.write(artifact.artifact.read())
+    meta_path = root / _metadata_path(artifact.doc_id, project=artifact.project)
+    meta = _read_artifact_metadata(root, artifact) if meta_path.exists() else dict()
+    meta[artifact.metadata_key] = artifact.filename
     meta_path.write_text(json.dumps(meta))
-    return transcription_path.relative_to(artif_dir)
+    return artifact_path.relative_to(artif_dir)
 
 
 def debuggable_name(

{datashare_python-0.4.0 → datashare_python-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.4.0"
+version = "0.6.0"
 description = "Manage Pythoœn tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },

datashare_python-0.4.0/datashare_python/cli/local.py

@@ -1,33 +0,0 @@
-from typing import Annotated
-
-import typer
-
-from datashare_python.cli.utils import AsyncTyper
-from datashare_python.constants import DEFAULT_NAMESPACE, DEFAULT_TEMPORAL_ADDRESS
-from datashare_python.local_client import LocalClient
-
-_REGISTER_NAMESPACE_HELP = "register namespace"
-_TEMPORAL_URL_HELP = "address for temporal server"
-_NAMESPACE_HELP = "namespace name"
-_LOCAL = "local"
-
-local_app = AsyncTyper(name=_LOCAL)
-
-
-@local_app.async_command(help=_REGISTER_NAMESPACE_HELP)
-async def register_namespace(
-    namespace: Annotated[
-        str, typer.Option("--namespace", "-n", help=_NAMESPACE_HELP)
-    ] = DEFAULT_NAMESPACE,
-    temporal_address: Annotated[
-        str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
-    ] = DEFAULT_TEMPORAL_ADDRESS,
-) -> None:
-    """Create namespace
-
-    :param namespace: namespace
-    :param temporal_address: target host
-    """
-    client = LocalClient()
-
-    await client.register_namespace(temporal_address, namespace)

datashare_python-0.4.0/datashare_python/local_client.py

@@ -1,69 +0,0 @@
-import asyncio
-import importlib
-import logging
-from functools import partial
-
-from google.protobuf.duration_pb2 import Duration
-from temporalio.api.workflowservice.v1 import (
-    ListNamespacesRequest,
-    RegisterNamespaceRequest,
-)
-from temporalio.service import ConnectConfig, ServiceClient
-
-LOGGER = logging.getLogger(__name__)
-
-
-class LocalClient:
-    @staticmethod
-    async def start_workers(
-        temporal_address: str, worker_paths: list[str], *args, **kwargs
-    ) -> None:
-        """Start worker modules, defined as a list of worker paths
-
-        :param temporal_address: temporal address string (host:port)
-        :param worker_paths: list of worker modules
-        """
-        workers = []
-
-        for worker_path in worker_paths:
-            try:
-                module_parts = worker_path.split(".")
-                module_path = ".".join(module_parts[:-1])
-                worker_method = module_parts[-1]
-                module = importlib.import_module(module_path)
-                worker = getattr(module, worker_method)
-                worker_partial = partial(worker, *args, **kwargs)
-
-                workers.append(worker_partial(target_host=temporal_address))
-
-                LOGGER.info("'%s' imported successfully.", worker_path)
-            except (ModuleNotFoundError, AttributeError):
-                LOGGER.error("'%s' not found in path. Skipping.", worker_path)
-                continue
-
-        await asyncio.gather(*workers, return_exceptions=True)
-
-    @staticmethod
-    async def register_namespace(temporal_address: str, namespace: str) -> None:
-        """Register a temporal namespace
-
-        :param temporal_address: temporal address string
-        :param namespace: namespace string
-        """
-        client = await ServiceClient.connect(
-            ConnectConfig(target_host=temporal_address)
-        )
-        list_resp = await client.workflow_service.list_namespaces(
-            ListNamespacesRequest()
-        )
-
-        if namespace in [ns.namespace_info.name for ns in list_resp.namespaces]:
-            return
-
-        await client.workflow_service.register_namespace(
-            RegisterNamespaceRequest(
-                namespace=namespace,
-                # retain for thirty days
-                workflow_execution_retention_period=Duration(seconds=30 * 24 * 60 * 60),
-            )
-        )