datashare-python 0.5.0__tar.gz → 0.6.1__tar.gz

{datashare_python-0.5.0 → datashare_python-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datashare-python
-Version: 0.5.0
+Version: 0.6.1
 Summary: Manage Pythoœn tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/cli/__init__.py

@@ -6,7 +6,6 @@ import typer
 from icij_common.logging_utils import setup_loggers
 
 import datashare_python
-from datashare_python.cli.local import local_app
 from datashare_python.cli.project import project_app
 from datashare_python.cli.task import task_app
 from datashare_python.cli.utils import AsyncTyper
@@ -16,7 +15,6 @@ cli_app = AsyncTyper(
     context_settings={"help_option_names": ["-h", "--help"]},
     pretty_exceptions_enable=False,
 )
-cli_app.add_typer(local_app)
 cli_app.add_typer(project_app)
 cli_app.add_typer(task_app)
 cli_app.add_typer(worker_app)

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/cli/task.py

@@ -10,8 +10,7 @@ import typer
 from alive_progress import alive_bar
 
 from datashare_python.cli.utils import AsyncTyper, eprint
-from datashare_python.constants import PYTHON_TASK_GROUP
-from datashare_python.objects import READY_STATES, Task, TaskError, TaskState
+from datashare_python.objects import READY_STATES, Task, TaskError, TaskGroup, TaskState
 from datashare_python.task_client import DatashareTaskClient
 
 logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ async def start(
     group: Annotated[
         str | None,
         typer.Option("--group", "-g", help=_GROUP_HELP),
-    ] = PYTHON_TASK_GROUP.name,
+    ] = TaskGroup.python,  # noqa: F821
     ds_address: Annotated[
         str, typer.Option("--ds-address", "-a", help=_DS_URL_HELP)
     ] = DEFAULT_DS_ADDRESS,

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/config.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import ClassVar
 
 from icij_common.es import ESClient
@@ -87,6 +88,10 @@ class WorkerConfig(ICIJSettings, LogWithWorkerIDMixin, BaseModel):
 
     max_concurrent_io_activities: int = 5
 
+    docs_root: Path | None = None
+    artifacts_root: Path | None = None
+    workdir: Path | None = None
+
     def to_es_client(self) -> ESClient:
         return self.elasticsearch.to_es_client(self.datashare.api_key)
 

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/conftest.py

@@ -4,6 +4,7 @@ from collections.abc import AsyncGenerator, Generator, Iterator, Sequence
 from pathlib import Path
 
 import aiohttp
+import nest_asyncio
 import pytest
 from elasticsearch._async.helpers import async_streaming_bulk
 from icij_common.es import DOC_ROOT_ID, ES_DOCUMENT_TYPE, ID, ESClient
@@ -200,6 +201,8 @@ def index_docs_ops(
         }
         doc = doc.model_dump(by_alias=True)  # noqa: PLW2901
         op.update(doc)
+        if "path" in op:
+            op["path"] = str(op["path"])
         op["_id"] = doc[ID]
         op["routing"] = doc[DOC_ROOT_ID]
         op["type"] = ES_DOCUMENT_TYPE
@@ -259,6 +262,7 @@ def doc_2() -> Document:
 def doc_3() -> Document:
     return Document(
         id="doc-3",
+        index=TEST_PROJECT,
         root_document="root-3",
         language="SPANISH",
         content="traduce este texto al inglés",
@@ -294,3 +298,8 @@ async def all_done(task_client: DatashareTaskClient, not_done: list[str]) -> boo
                 return False
             not_done.remove(t_id)
     return True
+
+
+@pytest.fixture  # noqa: F405
+def typer_asyncio_patch() -> None:
+    nest_asyncio.apply()

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/constants.py

@@ -1,11 +1,8 @@
 from pathlib import Path
 
-from .objects import TaskGroup
-
 PACKAGE_DIR = Path(__file__).parent
 PACKAGE_ROOT = PACKAGE_DIR.parent
 
-PYTHON_TASK_GROUP = TaskGroup(name="PYTHON")
 
 DEFAULT_TEMPORAL_ADDRESS = "temporal:7233"
 
@@ -14,3 +11,5 @@ DEFAULT_DS_ADDRESS = "http://localhost:8080"
 DEFAULT_NAMESPACE = "datashare-default"
 
 METADATA_JSON = "metadata.json"
+
+TIKA_METADATA_RESOURCENAME = "tika_metadata_resourcename"

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/objects.py

@@ -3,12 +3,26 @@ from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from enum import StrEnum, unique
-from typing import Any, Literal, Self, TypeVar
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Literal, Self, TypeVar, cast
 
 from temporalio import workflow
 
+from .constants import TIKA_METADATA_RESOURCENAME
+
 with workflow.unsafe.imports_passed_through():
-    from icij_common.es import DOC_CONTENT, DOC_LANGUAGE, DOC_ROOT_ID, ID_, SOURCE
+    from icij_common.es import (
+        DOC_CONTENT,
+        DOC_CONTENT_TRANSLATED,
+        DOC_LANGUAGE,
+        DOC_METADATA,
+        DOC_PATH,
+        DOC_ROOT_ID,
+        ID_,
+        INDEX_,
+        SOURCE,
+    )
 
 from icij_common.pydantic_utils import (
     icij_config,
@@ -135,17 +149,56 @@ class Task(Message):
 class TaskGroup:
     name: str
 
+    @property
+    @classmethod
+    def python(cls) -> Self:
+        return cls(name="PYTHON")
+
+
+@unique
+class DocumentLocation(StrEnum):
+    ORIGINAL = "original"
+    ARTIFACTS = "artifacts"
+    WORKDIR = "workdir"
+
+
+class FilesystemDocument(DatashareModel):
+    id: str
+    path: Path
+    index: str
+    location: DocumentLocation
+    resource_name: str
+
+    def locate(
+        self, original_root: Path, *, artifacts_root: Path, workdir: Path
+    ) -> Path:
+        from datashare_python.utils import artifacts_dir  # noqa: PLC0415
+
+        project = self.index
+        match self.location:
+            case DocumentLocation.ORIGINAL:
+                return original_root / self.path
+            case DocumentLocation.ARTIFACTS:
+                return artifacts_root / artifacts_dir(self.id, project=project) / "raw"
+            case DocumentLocation.WORKDIR:
+                return workdir / self.path
+            case _:
+                raise ValueError(f"invalid location: {self.path}")
+
 
 class Document(DatashareModel):
     id: str
-    root_document: str
     language: str
+    index: str | None = None
+    root_document: str | None = None
     content: str | None = None
     content_type: str | None = None
+    path: Path | None = None
     tags: list[str] = Field(default_factory=list)
     content_translated: dict[str, str] = Field(
         default_factory=dict, alias="content_translated"
     )
+    metadata: dict[str, Any] | None = None
     type: str = Field(default="Document", frozen=True)
 
     @classmethod
@@ -153,9 +206,49 @@ class Document(DatashareModel):
         sources = es_doc[SOURCE]
         return cls(
             id=es_doc[ID_],
-            content=sources[DOC_CONTENT],
-            content_translated=sources.get("content_translated", dict()),
+            index=es_doc.get(INDEX_),
+            content=sources.get(DOC_CONTENT),
+            content_translated=sources.get(DOC_CONTENT_TRANSLATED, dict()),
             language=sources[DOC_LANGUAGE],
             root_document=sources[DOC_ROOT_ID],
             tags=sources.get("tags", []),
+            path=sources.get(DOC_PATH),
+            metadata=sources.get(DOC_METADATA),
+        )
+
+    def to_filesystem(self) -> FilesystemDocument:
+        from .utils import artifacts_dir  # noqa: PLC0415
+
+        if self.metadata is None:
+            raise ValueError(
+                "can't compute filesyste path for document withtout metadata"
+            )
+        resource_name = cast(str, self.metadata[TIKA_METADATA_RESOURCENAME])
+        if self.root_document is None:
+            path = self.path
+            location = DocumentLocation.ORIGINAL
+        else:
+            if self.index is None:
+                msg = (
+                    f"can't compute filesystem path for embedded doc {self.id} without"
+                    f" index"
+                )
+                raise ValueError(msg)
+            path = artifacts_dir(doc_id=self.id, project=self.index) / "raw"
+            location = DocumentLocation.ARTIFACTS
+        return FilesystemDocument(
+            id=self.id,
+            path=path,
+            index=self.index,
+            location=location,
+            resource_name=resource_name,
         )
+
+
+@dataclass(frozen=True)
+class DocArtifact:
+    project: str
+    doc_id: str
+    artifact: bytes | BytesIO
+    filename: str
+    metadata_key: str

{datashare_python-0.5.0 → datashare_python-0.6.1}/datashare_python/utils.py

@@ -13,6 +13,7 @@ from datetime import timedelta
 from functools import partial, wraps
 from hashlib import sha256
 from inspect import signature
+from io import BytesIO
 from pathlib import Path
 from typing import Any, ParamSpec, TypeVar
 from uuid import uuid4
@@ -34,6 +35,7 @@ from temporalio.common import RetryPolicy, SearchAttributeKey
 from temporalio.exceptions import ApplicationError
 
 from .constants import METADATA_JSON
+from .objects import DocArtifact
 from .types_ import ProgressRateHandler, RawProgressHandler
 
 DependencyLabel = str | None
@@ -431,49 +433,43 @@ class LogWithWorkerIDMixin:
         return handlers
 
 
-def safe_dir(filename: str) -> Path:
-    filename = filename.split(".", maxsplit=1)[0]
-    parts = (p for p in (filename[:2], filename[2:4]) if p)
+def safe_dir(doc_id: str) -> Path:
+    if len(doc_id) < 4:
+        raise ValueError(f"expected doc_id to be at least 4, found {doc_id}")
+    parts = (p for p in (doc_id[:2], doc_id[2:4]) if p)
     return Path(*parts)
 
 
-def artifacts_dir(project: str, *, filename: str) -> Path:
-    return Path(project, safe_dir(filename), filename)
+def artifacts_dir(doc_id: str, *, project: str) -> Path:
+    return Path(project, safe_dir(doc_id), doc_id)
 
 
-def metadata_path(filename: str, *, project: str) -> Path:
-    metadata_path = artifacts_dir(project, filename=filename) / METADATA_JSON
+def _metadata_path(doc_id: str, *, project: str) -> Path:
+    metadata_path = artifacts_dir(doc_id, project=project) / METADATA_JSON
     return metadata_path
 
 
-def _read_artifact_metadata(root: Path, project: str, *, filename: str) -> dict:
-    m_path = root / metadata_path(filename, project=project)
+def _read_artifact_metadata(root: Path, artifact: DocArtifact) -> dict:
+    m_path = root / _metadata_path(artifact.filename, project=artifact.project)
     return json.loads(m_path.read_text())
 
 
-def write_artifact(
-    artifact: bytes,
-    root: Path,
-    *,
-    project: str,
-    filename: str,
-    metadata_key: str,
-    metadata_value: str,
-) -> Path:
-    artif_dir = root / artifacts_dir(project, filename=filename)
+def write_artifact(root: Path, artifact: DocArtifact) -> Path:
+    artif_dir = root / artifacts_dir(artifact.doc_id, project=artifact.project)
     artif_dir.mkdir(exist_ok=True, parents=True)
     # TODO: if transcriptions are too large we could also serialize them
     #  as jsonl
-    transcription_path = artif_dir / metadata_value
-    transcription_path.write_bytes(artifact)
-    try:
-        meta = _read_artifact_metadata(root, project, filename=filename)
-    except FileNotFoundError:
-        meta = dict()
-    meta[metadata_key] = metadata_value
-    meta_path = root / artifacts_dir(project, filename=filename) / METADATA_JSON
+    artifact_path: Path = artif_dir / artifact.filename
+    if isinstance(artifact.artifact, bytes):
+        artifact_path.write_bytes(artifact.artifact)
+    elif isinstance(artifact_path, BytesIO):
+        with artifact_path.open("wb") as f:
+            f.write(artifact.artifact.read())
+    meta_path = root / _metadata_path(artifact.doc_id, project=artifact.project)
+    meta = _read_artifact_metadata(root, artifact) if meta_path.exists() else dict()
+    meta[artifact.metadata_key] = artifact.filename
     meta_path.write_text(json.dumps(meta))
-    return transcription_path.relative_to(artif_dir)
+    return artifact_path.relative_to(artif_dir)
 
 
 def debuggable_name(
@@ -483,17 +479,23 @@ def debuggable_name(
     displayable_file_name = [c[:component_size_limit] for c in path.parts]
     uuid = sha256(str(path).encode()).hexdigest() if deterministic else uuid4().hex
     uuid = uuid[:20]
-    return f"{uuid}-{'__'.join(displayable_file_name)}"
+    return f"{uuid}-{'--'.join(displayable_file_name)}"
 
 
-def activity_contextual_id(*, wf_context: bool = False) -> str:
+def activity_contextual_id(
+    *, wf_context: bool = True, act_context: bool = False, run_context: bool = False
+) -> str:
+    contextual_id = []
     act_info = activity.info()
-    act_id = act_info.activity_id
-    act_run_id = act_info.activity_id
-    act_type = act_info.activity_type
-    contextual_id = f"{act_type}-{act_id}-{act_run_id}"
+    if not wf_context and not act_context:
+        raise ValueError("at least one of wf_context and act_context must be True")
     if wf_context:
-        wf_id = act_info.workflow_id
-        wf_run_id = act_info.workflow_run_id
-        contextual_id += f"-{wf_id}-{wf_run_id}"
-    return contextual_id
+        contextual_id.append(act_info.workflow_id)
+        if run_context:
+            contextual_id.append(act_info.workflow_run_id)
+    if act_context:
+        contextual_id.append(act_info.activity_type)
+        contextual_id.append(act_info.activity_id)
+        if run_context:
+            contextual_id.append(act_info.activity_run_id)
+    return "-".join(contextual_id)

{datashare_python-0.5.0 → datashare_python-0.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.5.0"
+version = "0.6.1"
 description = "Manage Pythoœn tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },

datashare_python-0.5.0/datashare_python/cli/local.py

@@ -1,33 +0,0 @@
-from typing import Annotated
-
-import typer
-
-from datashare_python.cli.utils import AsyncTyper
-from datashare_python.constants import DEFAULT_NAMESPACE, DEFAULT_TEMPORAL_ADDRESS
-from datashare_python.local_client import LocalClient
-
-_REGISTER_NAMESPACE_HELP = "register namespace"
-_TEMPORAL_URL_HELP = "address for temporal server"
-_NAMESPACE_HELP = "namespace name"
-_LOCAL = "local"
-
-local_app = AsyncTyper(name=_LOCAL)
-
-
-@local_app.async_command(help=_REGISTER_NAMESPACE_HELP)
-async def register_namespace(
-    namespace: Annotated[
-        str, typer.Option("--namespace", "-n", help=_NAMESPACE_HELP)
-    ] = DEFAULT_NAMESPACE,
-    temporal_address: Annotated[
-        str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
-    ] = DEFAULT_TEMPORAL_ADDRESS,
-) -> None:
-    """Create namespace
-
-    :param namespace: namespace
-    :param temporal_address: target host
-    """
-    client = LocalClient()
-
-    await client.register_namespace(temporal_address, namespace)

datashare_python-0.5.0/datashare_python/local_client.py

@@ -1,69 +0,0 @@
-import asyncio
-import importlib
-import logging
-from functools import partial
-
-from google.protobuf.duration_pb2 import Duration
-from temporalio.api.workflowservice.v1 import (
-    ListNamespacesRequest,
-    RegisterNamespaceRequest,
-)
-from temporalio.service import ConnectConfig, ServiceClient
-
-LOGGER = logging.getLogger(__name__)
-
-
-class LocalClient:
-    @staticmethod
-    async def start_workers(
-        temporal_address: str, worker_paths: list[str], *args, **kwargs
-    ) -> None:
-        """Start worker modules, defined as a list of worker paths
-
-        :param temporal_address: temporal address string (host:port)
-        :param worker_paths: list of worker modules
-        """
-        workers = []
-
-        for worker_path in worker_paths:
-            try:
-                module_parts = worker_path.split(".")
-                module_path = ".".join(module_parts[:-1])
-                worker_method = module_parts[-1]
-                module = importlib.import_module(module_path)
-                worker = getattr(module, worker_method)
-                worker_partial = partial(worker, *args, **kwargs)
-
-                workers.append(worker_partial(target_host=temporal_address))
-
-                LOGGER.info("'%s' imported successfully.", worker_path)
-            except (ModuleNotFoundError, AttributeError):
-                LOGGER.error("'%s' not found in path. Skipping.", worker_path)
-                continue
-
-        await asyncio.gather(*workers, return_exceptions=True)
-
-    @staticmethod
-    async def register_namespace(temporal_address: str, namespace: str) -> None:
-        """Register a temporal namespace
-
-        :param temporal_address: temporal address string
-        :param namespace: namespace string
-        """
-        client = await ServiceClient.connect(
-            ConnectConfig(target_host=temporal_address)
-        )
-        list_resp = await client.workflow_service.list_namespaces(
-            ListNamespacesRequest()
-        )
-
-        if namespace in [ns.namespace_info.name for ns in list_resp.namespaces]:
-            return
-
-        await client.workflow_service.register_namespace(
-            RegisterNamespaceRequest(
-                namespace=namespace,
-                # retain for thirty days
-                workflow_execution_retention_period=Duration(seconds=30 * 24 * 60 * 60),
-            )
-        )