datashare-python 0.11.4__py3-none-any.whl → 0.12.1__py3-none-any.whl

datashare_python/config.py

@@ -1,5 +1,6 @@
+from datetime import timedelta
 from enum import StrEnum
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from icij_common.es import ESClient
 from icij_common.pydantic_utils import ICIJSettings
@@ -12,7 +13,9 @@ import datashare_python
 from .objects import BaseModel, WorkerPaths
 from .task_client import DatashareTaskClient
 from .types_ import TemporalClient
-from .utils import PYDANTIC_DATA_CONVERTER, SharedResources, close_cm_callback
+
+if TYPE_CHECKING:
+    from .utils import SharedResources
 
 _ALL_LOGGERS = [datashare_python.__name__]
 
@@ -63,6 +66,8 @@ class TemporalClientConfig(BaseModel):
     _client: TemporalClient | None = PrivateAttr(default=None)
 
     async def to_client(self) -> TemporalClient:
+        from .utils import PYDANTIC_DATA_CONVERTER  # noqa: PLC0415
+
         if self._client is None:
             runtime = None
             if self.prometheus_host is not None:
@@ -70,6 +75,7 @@ class TemporalClientConfig(BaseModel):
                     metrics=PrometheusConfig(bind_address="0.0.0.0:9000")
                 )
                 runtime = Runtime(telemetry=telemetry_config)
+
             self._client = await TemporalClient.connect(
                 target_host=self.host,
                 namespace=self.namespace,
@@ -97,7 +103,9 @@ class ResourceCacheConfig(BaseModel):
     size: int = 1
     exit_context_managers: bool = True
 
-    def to_resource_cache(self) -> SharedResources:
+    def to_resource_cache(self) -> "SharedResources":
+        from .utils import SharedResources, close_cm_callback  # noqa: PLC0415
+
         eviction_callback = None
         if self.exit_context_managers:
             eviction_callback = close_cm_callback
@@ -134,3 +142,8 @@ class WorkerConfig(ICIJSettings, BaseModel):
 
     async def to_temporal_client(self) -> TemporalClient:
         return await self.temporal.to_client()
+
+
+class ActivityTimeouts(BaseModel):
+    start_to_close: timedelta | None = None
+    heartbeat: timedelta = timedelta(minutes=1)

datashare_python/mimetypes_.py

@@ -75,6 +75,7 @@ def types_map() -> dict[str, set[str]]:
         ".bmp": {"image/bmp"},
         ".book": {"application/vnd.framemaker"},
         ".box": {"application/vnd.previewsystems.box"},
+        ".boxnote": {"application/octet-stream"},
         ".boz": {"application/x-bzip2"},
         ".bpk": {"application/octet-stream"},
         ".bufr": {"application/x-bufr"},
@@ -172,6 +173,7 @@ def types_map() -> dict[str, set[str]]:
         ".davmount": {"application/davmount+xml"},
         ".dbf": {"application/dbf", "application/dbase"},
         ".dbk": {"application/docbook+xml"},
+        ".dclx": {"application/octet-stream"},
         ".dcr": {"application/x-director"},
         ".dcurl": {"text/vnd.curl.dcurl"},
         ".dcx": {"image/vnd.zbrush.dcx", "image/x-dcx"},
@@ -226,6 +228,8 @@ def types_map() -> dict[str, set[str]]:
         ".dxf": {"image/vnd.dxf"},
         ".dxp": {"application/vnd.spotfire.dxp"},
         ".dxr": {"application/x-director"},
+        ".ebc": {"application/octet-stream"},
+        ".ebcdic": {"application/octet-stream"},
         ".ecelp4800": {"audio/vnd.nuera.ecelp4800"},
         ".ecelp7470": {"audio/vnd.nuera.ecelp7470"},
         ".ecelp9600": {"audio/vnd.nuera.ecelp9600"},

datashare_python/objects.py

@@ -2,14 +2,15 @@ import hashlib
 import json
 import logging
 import os
-from abc import ABC
-from collections.abc import Awaitable, Callable
+import traceback
+from abc import ABC, abstractmethod
+from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from enum import StrEnum, unique
 from io import BytesIO
 from pathlib import Path
-from typing import Annotated, Any, ClassVar, Literal, Self, TypeVar, cast
+from typing import Annotated, Any, ClassVar, Literal, Self, TypeVar, cast, final
 
 import langcodes
 from icij_common.registrable import Registrable
@@ -50,6 +51,7 @@ from pydantic import (
     Discriminator,
     Field,
     GetCoreSchemaHandler,
+    Tag,
     TypeAdapter,
     model_validator,
 )
@@ -59,8 +61,9 @@ from pydantic.main import IncEx
 logger = logging.getLogger(__name__)
 
 
-T = TypeVar("T")
-Predicate = Callable[[T], bool] | Callable[[T], Awaitable[bool]]
+Routing = str
+DocID = str
+DocRoute = tuple[DocID, Routing]
 
 
 class BaseModel(_BaseModel):
@@ -118,68 +121,34 @@ class DatashareLanguage(str):
         return self.as_language_name.alpha3
 
 
-class WorkerPaths(BaseModel):
-    filesystem: Path
-    artifacts: Path
-    workdir: Path
-
-
 @unique
-class DocumentLocation(StrEnum):
+class FileLocation(StrEnum):
     FILESYSTEM = "filesystem"
     ARTIFACTS = "artifacts"
     WORKDIR = "workdir"
 
 
-def _is_relative(value: Path) -> Path:
-    if value.is_absolute():
-        raise ValueError(
-            f"FilesystemDocument path should always be relative, found {value}"
-        )
-    return value
-
-
-class ProcessedFile(BaseModel):
-    id: str
-    path: Annotated[Path, AfterValidator(_is_relative)]
-    project: str
-    location: DocumentLocation
-    resource_name: str
-    n_pages: int
-    parent: "ProcessedFile | None" = None
-
-    @classmethod
-    def from_doc(cls, doc: "Document") -> Self:
-        return doc.to_processed_file()
-
-    def child(self, path: Path, paths: WorkerPaths) -> Self:
-        return ProcessedFile(
-            id=self.id,
-            path=path.relative_to(paths.workdir),
-            project=self.project,
-            location=DocumentLocation.WORKDIR,
-            resource_name=path.name,
-            n_pages=self.n_pages,
-            parent=self,
-        )
+class WorkerPaths(BaseModel):
+    filesystem: Path
+    artifacts: Path
+    workdir: Path
 
-    def locate(self, paths: WorkerPaths) -> Path:
-        from datashare_python.utils import artifacts_dir  # noqa: PLC0415
-
-        match self.location:
-            case DocumentLocation.FILESYSTEM:
-                return paths.filesystem / self.path
-            case DocumentLocation.ARTIFACTS:
-                project = self.project
-                return paths.artifacts / artifacts_dir(self.id, project=project) / "raw"
-            case DocumentLocation.WORKDIR:
-                return paths.workdir / self.path
+    def locate(self, path: Path, location: FileLocation) -> Path:
+        match location:
+            case FileLocation.FILESYSTEM:
+                return self.filesystem / path
+            case FileLocation.ARTIFACTS:
+                return self.artifacts / path
+            case FileLocation.WORKDIR:
+                return self.workdir / path
             case _:
-                raise ValueError(f"invalid location: {self.path}")
+                raise ValueError(f"invalid location: {path}")
 
 
-class ProcessedPage(ProcessedFile):
-    page_number: int
+class FromParent[P](ABC):
+    @classmethod
+    @abstractmethod
+    def from_parent(cls, parent: P, *args, **kwargs) -> Self: ...
 
 
 class IETFLanguage(str):
@@ -213,12 +182,9 @@ class Translation(BaseModel):  # No camelcase here we don't know why
     content: Annotated[str, BeforeValidator(_from_sentences)]
 
 
-Routing = str
-DocID = str
-DocRoute = tuple[DocID, Routing]
-
-
 class Document(DatashareModel):
+    """ES datashare document"""
+
     id: str
     language: DatashareLanguage
     index: str | None = None
@@ -285,7 +251,7 @@ class Document(DatashareModel):
             id=es_doc[ID_],
             index=es_doc.get(INDEX_),
             content=sources.get(DOC_CONTENT),
-            content_translated=sources.get(DOC_CONTENT_TRANSLATED, []),
+            content_translated=sources.get(DOC_CONTENT_TRANSLATED),
             content_text_length=sources.get("content_text_length"),
             language=DatashareLanguage(sources[DOC_LANGUAGE]),
             root_document=sources.get(DOC_ROOT_ID),
@@ -295,58 +261,57 @@ class Document(DatashareModel):
             metadata=sources.get(DOC_METADATA),
         )
 
-    def to_processed_file(self) -> ProcessedFile:
-        from .utils import artifacts_dir  # noqa: PLC0415
+    @property
+    def project(self) -> str:
+        if self.index is None:
+            raise ValueError("missing index")
+        return self.index
 
+    @property
+    def doc_id(self) -> str:
+        return self.id
+
+    @property
+    def route(self) -> DocRoute:
+        if self.root_document:
+            return self.id, self.id
+        return self.id, self.root_document
+
+    @property
+    def resource_name(self) -> str:
         if self.metadata is None:
-            raise ValueError(
-                "can't compute filesystem path for document withtout metadata"
-            )
+            raise ValueError("missing metadata")
         resource_name = self.metadata.get(TIKA_METADATA_RESOURCENAME)
         if resource_name is None:
-            # Backward compat for project before resourcename was mandatory
-            if self.is_root_document:
-                resource_name = self.path.name
-            else:
-                msg = (
-                    f"doc {self} can't turn ES document into a"
-                    f" {ProcessedFile.__name__} without tika metadata resources name"
-                    f" in metadata: {self.metadata}"
-                )
-                raise ValueError(msg)
-        resource_name = cast(str, resource_name)
-        if self.is_root_document:
-            path = self.path
-            location = DocumentLocation.FILESYSTEM
-        else:
-            if self.index is None:
-                msg = (
-                    f"can't compute filesystem path for embedded doc {self.id} without"
-                    f" index"
-                )
-                raise ValueError(msg)
-            path = artifacts_dir(doc_id=self.id, project=self.index) / "raw"
-            location = DocumentLocation.ARTIFACTS
-        # The filesystem dod is alway relative to the base location, let's make sure
-        # we store a relative path otherwise joining with the location will fail
-        if path.parts and path.parts[0] == os.path.sep:
-            path = Path(*path.parts[1:])
-        n_pages = 1
-        if self.metadata:
-            n_pages = self.metadata.get("tika_metadata_xmptpg_npages", n_pages)
-        return ProcessedFile(
-            id=self.id,
-            path=path,
-            project=self.index,
-            location=location,
-            resource_name=resource_name,
-            n_pages=n_pages,
+            msg = f"missing {TIKA_METADATA_RESOURCENAME} in metadata"
+            raise KeyError(msg)
+        cast(str, resource_name)
+        return resource_name
+
+
+def _is_relative(value: Path) -> Path:
+    if value.is_absolute():
+        raise ValueError(
+            f"WorkerFilePath path should always be relative, found {value}"
         )
+    return value
 
-    def to_route(self) -> DocRoute:
-        if self.root_document is not None:
-            return self.root_document, self.id
-        return self.id, self.id
+
+class WorkerFilePath(BaseModel):
+    """Path of a file produced or processed by a worker, located somewhere on the DS
+    filesystem, artifactsdir or workdir
+    """
+
+    path: Annotated[Path, AfterValidator(_is_relative)]
+    location: FileLocation
+
+    def locate(self, paths: WorkerPaths) -> Path:
+        return paths.locate(self.path, self.location)
+
+    @classmethod
+    def relative_to_workdir(cls, path: Path, paths: WorkerPaths) -> Self:
+        path = path.relative_to(paths.workdir)
+        return cls(path=path, location=FileLocation.WORKDIR)
 
 
 def _is_absolute_path(v: bytes | BytesIO | Path) -> Any:
@@ -617,3 +582,297 @@ class TaskGroup:
     @classmethod
     def python(cls) -> Self:
         return cls(name="PYTHON")
+
+
+class Error(BaseModel):
+    title: str
+    detail: str | None
+
+    @classmethod
+    def from_exception(cls, exception: BaseException) -> "Error":
+        title = exception.__class__.__name__
+        trace_lines = traceback.format_exception(
+            None, value=exception, tb=exception.__traceback__
+        )
+        detail = f"{exception}\n{''.join(trace_lines)}"
+        error = Error(title=title, detail=detail)
+        return error
+
+    def without_detail(self) -> "Error":
+        return Error(title=self.title, detail=None)
+
+
+class ErrorSource(BaseModel, ABC):
+    """Source of the processing error."""
+
+    doc_id: str
+    project: str
+
+
+class ErrorSourceWithPages(ErrorSource, ABC):
+    """Source of the processing error and number of pages processed at that time"""
+
+    n_pages: int
+
+
+class ProcessingError[S: ErrorSource](BaseModel):
+    """Error occurring while processing a document or intermediate result"""
+
+    source: S
+    error: Error
+
+    @final
+    @classmethod
+    def from_exception(cls, source: S, exception: BaseException) -> Self:
+        return cls(source=source, error=Error.from_exception(exception))
+
+
+class ProcessingReport(DatashareModel):
+    """Basic processing report indicating how many docs where processed"""
+
+    n_docs: int = 0
+
+    def __add__(self, other: Self) -> Self:
+        return ProcessingReport(n_docs=other.n_docs + self.n_docs)
+
+
+class ProcessingReportWithPages(ProcessingReport):
+    """Processing report indicating how many pages where processed"""
+
+    n_pages: int = 0
+
+    def __add__(self, other: Self) -> Self:
+        return ProcessingReportWithPages(
+            n_docs=other.n_docs + self.n_docs, n_pages=other.n_pages + self.n_pages
+        )
+
+
+def _without_detail(errors: list[Error]) -> list[Error]:
+    if any(e.detail is not None for e in errors):
+        msg = "expected errors without details"
+        raise ValueError(msg)
+    return errors
+
+
+class DocProcessingErrors(BaseModel):
+    """All errors related to several processing happening on a single document.
+    If a source doc has many children (typically a document split into pages
+    processed separately), all children errors are merged under the older parent.
+    """
+
+    doc_id: str
+    root_document: str
+    project: str
+    errors: Annotated[
+        list[Error], AfterValidator(_without_detail), Field(default_factory=list)
+    ]
+
+
+class ErrorReport(ProcessingReport):
+    """Processing report indicated how many docs failed and listing errors per doc."""
+
+    errors: list[DocProcessingErrors] = Field(default_factory=list)
+
+    def __add__(self, other: Self) -> Self:
+        return ErrorReportWithPages(
+            n_docs=other.n_docs + self.n_docs, errors=self.errors + other.errors
+        )
+
+    @classmethod
+    def from_errors(cls, *errors: ProcessingError) -> Self:
+        """Aggregate errors by root source documents"""
+        roots = dict()
+        for error in errors:
+            root = error.source
+            while (parent := getattr(root, "parent", None)) is not None:
+                root = parent
+            root_errors = roots.get(root.doc_id)
+            if root_errors is not None:
+                _, root_errors = root_errors
+            else:
+                root_errors = []
+            root_errors.append(error)
+            roots[root.doc_id] = (root, root_errors)
+        errors = []
+        for _, (root, root_errors) in sorted(roots.items()):
+            # We keep details for errors stored as intermediate steps to allow
+            # debugging, we don't output them in the response to keep everything
+            # lightweight and more secure
+            root_errors = [err.error.without_detail() for err in root_errors]  # noqa: PLW2901
+            doc_errors = DocProcessingErrors(
+                doc_id=root.doc_id,
+                project=root.project,
+                root_document=root.root_document,
+                errors=root_errors,
+            )
+            errors.append(doc_errors)
+        return cls(n_docs=len(errors), errors=errors)
+
+
+class ErrorReportWithPages(ProcessingReportWithPages, ABC):
+    """Same as ErrorReport but providing details about how many pages failed"""
+
+    errors: list[DocProcessingErrors] = Field(default_factory=list)
+
+    def __add__(self, other: Self) -> Self:
+        return ErrorReportWithPages(
+            n_docs=other.n_docs + self.n_docs,
+            n_pages=other.n_pages + self.n_pages,
+            errors=self.errors + other.errors,
+        )
+
+    @classmethod
+    def from_errors(cls, *errors: ProcessingError) -> Self:
+        """Aggregate errors by root source documents"""
+        roots = dict()
+        n_pages = 0
+        for error in errors:
+            root = error.source
+            while (parent := getattr(root, "parent", None)) is not None:
+                root = parent
+            root_errors = roots.get(root.doc_id)
+            if root_errors is not None:
+                _, root_errors = root_errors
+            else:
+                root_errors = []
+            n_pages += error.source.n_pages
+            root_errors.append(error)
+            roots[root.doc_id] = (root, root_errors)
+        errors = []
+        for _, (root, root_errors) in sorted(roots.items()):
+            # We keep details for errors stored as intermediate steps to allow
+            # debugging, we don't output them in the response to keep everything
+            # lightweight and more secure
+            root_errors = [err.error.without_detail() for err in root_errors]  # noqa: PLW2901
+            doc_errors = DocProcessingErrors(
+                doc_id=root.doc_id,
+                project=root.project,
+                root_document=root.root_document,
+                errors=root_errors,
+            )
+            errors.append(doc_errors)
+        return cls(n_docs=len(errors), n_pages=n_pages, errors=errors)
+
+
+class ProcessingResult[V](BaseModel):
+    value: V
+
+
+class WithParent[P](FromParent[P]):
+    """Result produced from another processing result
+
+    Tracking parent allows track errors from one intermediate results to another back
+     to the initial DS Document from the index. It also allows error aggregation.
+    """
+
+    parent: P
+
+    @property
+    def doc_id(self) -> str:
+        return self.parent.doc_id
+
+    @property
+    def root_document(self) -> str:
+        return self.parent.root_document
+
+    @property
+    def project(self) -> str:
+        return self.parent.project
+
+    @property
+    def route(self) -> DocRoute:
+        return self.parent.route
+
+
+class DatashareFile(
+    WithParent[Document], ProcessingResult[WorkerFilePath], ErrorSourceWithPages
+):
+    """A datashare document file ready to be processed and located somewhere on the
+    filesystem or artifact directory.
+    """
+
+    parent: Document
+
+    def locate(self, paths: WorkerPaths) -> Path:
+        return self.value.locate(paths)
+
+    @classmethod
+    def from_parent(cls, parent: Document) -> Self:
+        from .utils import artifacts_dir  # noqa: PLC0415
+
+        if parent.metadata is None:
+            raise ValueError(
+                "can't compute filesystem path for document withtout metadata"
+            )
+        if parent.is_root_document:
+            path = parent.path
+            if path is None:
+                msg = (
+                    "can't create a Datashare file from ES document without path, "
+                    "path is needed to locate the file, retrieve it from ES"
+                )
+                raise ValueError(msg)
+            location = FileLocation.FILESYSTEM
+        else:
+            path = artifacts_dir(doc_id=parent.id, project=parent.project) / "raw"
+            location = FileLocation.ARTIFACTS
+        # The filesystem dod is alway relative to the base location, let's make sure
+        # we store a relative path otherwise joining with the location will fail
+        if path.parts and path.parts[0] == os.path.sep:
+            path = Path(*path.parts[1:])
+        n_pages = 1
+        if parent.metadata:
+            n_pages = parent.metadata.get("tika_metadata_xmptpg_npages", n_pages)
+        path = WorkerFilePath(path=path, location=location)
+        return cls(
+            doc_id=parent.id,
+            parent=parent,
+            value=path,
+            n_pages=n_pages,
+            project=parent.project,
+        )
+
+
+class WorkerFile(
+    WithParent["ProcessedFile"], ProcessingResult[WorkerFilePath], ErrorSourceWithPages
+):
+    """An intermediate results produce by the worker either from a Datashare doc on the
+    filesystem or from another intermediate result
+    """
+
+    parent: "ProcessedFile"
+
+    def locate(self, paths: WorkerPaths) -> Path:
+        return self.value.locate(paths)
+
+    @classmethod
+    def from_parent(
+        cls, parent: "ProcessedFile", path: Path, paths: WorkerPaths
+    ) -> Self:
+        path = WorkerFilePath.relative_to_workdir(path, paths)
+        return cls(
+            doc_id=parent.doc_id,
+            project=parent.project,
+            parent=parent,
+            value=path,
+            n_pages=parent.n_pages,
+        )
+
+
+def _file_discriminator(v: Any) -> str:
+    parent = v.get("parent") if isinstance(v, dict) else getattr(v, "parent", None)
+    if parent is None:
+        raise ValueError(f"{v} has no parent")
+    if isinstance(parent, dict):
+        is_document = parent.get("type") == "Document"
+    else:
+        is_document = isinstance(parent, Document)
+    return "datashare" if is_document else "worker"
+
+
+ProcessedFile = Annotated[
+    Annotated[DatashareFile, Tag("datashare")] | Annotated[WorkerFile, Tag("worker")],
+    Discriminator(_file_discriminator),
+]
+WorkerFile.model_rebuild()
+PROCESSED_FILE_TA = TypeAdapter(ProcessedFile, config=BaseModel.model_config)

datashare_python/utils.py

@@ -21,18 +21,17 @@ from collections.abc import (
 )
 from copy import deepcopy
 from dataclasses import dataclass
-from datetime import timedelta
 from functools import cache, wraps
 from hashlib import sha256
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Self, TypeVar
+from typing import Any, Protocol, Self, TypeVar
 from uuid import uuid4
 
 import temporalio
 from aiofile import async_open
 from lru import LRU
-from pydantic import ValidationError
+from pydantic import TypeAdapter, ValidationError
 from pydantic.alias_generators import to_snake
 from temporalio import activity, workflow
 from temporalio.api.common.v1 import Payload
@@ -47,6 +46,7 @@ from temporalio.converter import (
 )
 from temporalio.exceptions import ApplicationError
 
+from datashare_python.config import ActivityTimeouts
 from datashare_python.types_ import (
     AsyncProgressRateHandler,
     RawSyncProgressHandler,
@@ -57,9 +57,13 @@ from .constants import MANIFEST_JSON, METADATA_JSON
 from .mimetypes_ import types_map
 from .objects import (
     BaseModel,
+    DatashareFile,
     DocArtifact,
-    DocumentLocation,
+    Document,
+    ErrorSource,
     ProcessedFile,
+    ProcessingError,
+    WorkerFile,
     WorkerPaths,
 )
 from .types_ import RawAsyncProgressHandler
@@ -162,10 +166,11 @@ async def execute_activity(
     arg: Any = temporalio.common._arg_unset,
     *,
     args: list | None = None,
-    start_to_close_timeout: timedelta | None = None,
-    heartbeat_timeout: timedelta = timedelta(minutes=1),
+    timeouts: ActivityTimeouts | None = None,
     retry_policy: temporalio.common.RetryPolicy | None = None,
 ) -> Any:
+    if timeouts is None:
+        timeouts = ActivityTimeouts()
     if args is None:
         args = []
     retry_policy = _retry_policy_with_default(retry_policy)
@@ -173,10 +178,10 @@ async def execute_activity(
         activity,
         arg=arg,
         args=args,
-        start_to_close_timeout=start_to_close_timeout,
         task_queue=task_queue,
         retry_policy=retry_policy,
-        heartbeat_timeout=heartbeat_timeout,
+        start_to_close_timeout=timeouts.start_to_close,
+        heartbeat_timeout=timeouts.heartbeat,
     )
 
 
@@ -389,8 +394,7 @@ def to_scaled_async_progress(
 def safe_dir(doc_id: str) -> Path:
     if len(doc_id) < 4:
         raise ValueError(f"expected doc_id to be at least 4, found {doc_id}")
-    parts = (p for p in (doc_id[:2], doc_id[2:4]) if p)
-    return Path(*parts)
+    return Path(doc_id[:2], doc_id[2:4])
 
 
 def artifacts_dir(doc_id: str, *, project: str) -> Path:
@@ -608,31 +612,18 @@ def activity_workdir(
 
 
 def symlink_embedded_document_to_workdir(
-    doc: ProcessedFile, paths: WorkerPaths
+    doc: Document, paths: WorkerPaths
 ) -> ProcessedFile:
-    match doc.location:
-        case DocumentLocation.ARTIFACTS:
-            symlinks_dir = paths.workdir / doc.project / "symlinks"
-            symlinks_dir.mkdir(parents=True, exist_ok=True)
-            symlink_path = Path(*doc.path.parts[1:-1], doc.resource_name)
-            symlink_path = symlinks_dir / symlink_path
-            # Replace the "raw" with the resource name
-            symlink_path.parent.mkdir(parents=True, exist_ok=True)
-            artifact_path = paths.artifacts / doc.path
-            with contextlib.suppress(FileExistsError):
-                os.symlink(artifact_path, symlink_path)
-            return ProcessedFile(
-                path=symlink_path.relative_to(paths.workdir),
-                id=doc.id,
-                location=DocumentLocation.WORKDIR,
-                project=doc.project,
-                resource_name=doc.resource_name,
-                n_pages=doc.n_pages,
-            )
-        case DocumentLocation.FILESYSTEM:
-            return doc
-        case _:
-            raise ValueError(f"unsupported location {doc.location}")
+    ds_file = DatashareFile.from_parent(doc)
+    if doc.is_root_document:
+        return ds_file
+    symlinks_dir = paths.workdir / doc.project / "symlinks"
+    symlink_path = symlinks_dir / safe_dir(doc.id) / doc.id / doc.resource_name
+    symlink_path.parent.mkdir(parents=True, exist_ok=True)
+    raw_path = ds_file.locate(paths)
+    with contextlib.suppress(FileExistsError):
+        os.symlink(raw_path, symlink_path)
+    return WorkerFile.from_parent(ds_file, symlink_path, paths)
 
 
 def artifact_path(
@@ -645,7 +636,7 @@ def artifact_path(
     return root / artifacts_dir(doc_id, project=project) / artifact_type.filename
 
 
-def _read_jsonl(path: Path) -> Iterable[dict]:
+def read_jsonl(path: Path) -> Iterable[dict]:
     with path.open() as f:
         for line in f:
             line = line.strip()  # noqa: PLW2901
@@ -656,18 +647,82 @@ def _read_jsonl(path: Path) -> Iterable[dict]:
 M = TypeVar("M", bound=BaseModel)
 
 
-def read_jsonl_as[M](path: Path, cls: type[M]) -> Iterable[M]:
-    return (cls.model_validate(d) for d in _read_jsonl(path))
+def read_jsonl_as[M](
+    path: Path, processed_file_cls: type[M] | TypeAdapter[M]
+) -> Iterable[M]:
+    if isinstance(processed_file_cls, TypeAdapter):
+        deser_fn = processed_file_cls.validate_python
+    else:
+        deser_fn = processed_file_cls.model_validate
+    return (deser_fn(d) for d in read_jsonl(path))
 
 
 async def async_read_jsonl_as[M](
-    path: Path, processed_file_cls: type[M]
+    path: Path, processed_file_cls: type[M] | TypeAdapter[M]
 ) -> AsyncIterable[M]:
+    if isinstance(processed_file_cls, TypeAdapter):
+        deser_fn = processed_file_cls.validate_json
+    else:
+        deser_fn = processed_file_cls.model_validate_json
     async with async_open(path, "r") as f:
         async for line in f:
             line = line.strip()  # noqa: PLW2901
             if line:
-                yield processed_file_cls.model_validate_json(line)
+                yield deser_fn(line)
+
+
+async def write_batches(
+    batches: AsyncIterable[list[ProcessedFile]] | Iterable[list[ProcessedFile]],
+    root: Path,
+    batch_offset: int = 0,
+    prefix: str = "batch_",
+) -> AsyncIterable[Path]:
+    if hasattr(batches, "__aiter__"):
+        async for b in _async_write_batches(batches, root, batch_offset, prefix):
+            yield b
+        return
+    async for b in _write_batches(batches, root, batch_offset, prefix):
+        yield b
+
+
+async def _async_write_batches(
+    batches: AsyncIterable[list[ProcessedFile]],
+    root: Path,
+    batch_offset: int,
+    prefix: str,
+) -> AsyncIterable[Path]:
+    batch_id = batch_offset
+    async for batch in batches:
+        batch_path = root / f"{batch_id // 1000}" / f"{prefix}{batch_id}.jsonl"
+        batch_path.parent.mkdir(parents=True, exist_ok=True)
+        async with async_open(batch_path, "w") as f:
+            for fs_doc in batch:
+                serialized = fs_doc.model_dump_json(
+                    exclude_none=True, polymorphic_serialization=True
+                )
+                await f.write(f"{serialized}\n")
+        yield batch_path
+        batch_id += 1
+
+
+async def _write_batches(
+    batches: Iterable[list[ProcessedFile]],
+    root: Path,
+    batch_offset: int,
+    prefix: str,
+) -> AsyncIterable[Path]:
+    batch_id = batch_offset
+    for batch in batches:
+        batch_path = root / f"{batch_id // 1000}" / f"{prefix}{batch_id}.jsonl"
+        batch_path.parent.mkdir(parents=True, exist_ok=True)
+        async with async_open(batch_path, "w") as f:
+            for fs_doc in batch:
+                serialized = fs_doc.model_dump_json(
+                    exclude_none=True, polymorphic_serialization=True
+                )
+                await f.write(f"{serialized}\n")
+        yield batch_path
+        batch_id += 1
 
 
 @cache
@@ -690,6 +745,8 @@ def ext_to_mime_types(ext: str) -> set[str]:
             return ext_to_mime_types(".xml")
         case ".dclg.xml":
             return ext_to_mime_types(".xml")
+        case ".msg":
+            return ext_to_mime_types(".eml")
     try:
         return types_map()[ext]
     except KeyError as e:
@@ -907,3 +964,43 @@ def config_cache_key(config: BaseModel) -> str:
         .replace("_", "-")
     )
     return f"{config_label}-{str(hash(config))}"
+
+
+class _PreprocessingFunction[S: ErrorSource, R](Protocol):
+    def __call__(self, doc: S, *args, **kwargs) -> R: ...
+
+
+def reports_errors[S: ErrorSource, R](
+    errors: tuple[type[Exception]], exc_cls: type[ProcessingError[S]]
+) -> Callable[
+    [_PreprocessingFunction[S, R]], _PreprocessingFunction[S, R | ProcessingError[S]]
+]:
+
+    def parent_wrapper(
+        f: _PreprocessingFunction[S, R],
+    ) -> _PreprocessingFunction[S, R | ProcessingError[S]]:
+        if inspect.iscoroutinefunction(f):
+
+            @wraps(f)  # noqa: F821
+            async def wrapper(doc: S, *args, **kwargs) -> R | ProcessingError[S]:
+                try:
+                    return await f(doc, *args, **kwargs)
+                except errors as e:
+                    logger.exception("error while processing doc %s", doc)
+                    report = exc_cls.from_exception(doc, e)
+                    return report
+        else:
+
+            @wraps(f)
+            def wrapper(doc: ProcessedFile, *args, **kwargs) -> R | ProcessingError[S]:
+
+                try:
+                    return f(doc, *args, **kwargs)
+                except errors as e:
+                    logger.exception("error while processing doc %s", doc)
+                    report = exc_cls.from_exception(doc, e)
+                    return report
+
+        return wrapper
+
+    return parent_wrapper

{datashare_python-0.11.4.dist-info → datashare_python-0.12.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datashare-python
-Version: 0.11.4
+Version: 0.12.1
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.11.4.dist-info → datashare_python-0.12.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,22
 datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
-datashare_python/config.py,sha256=YybzH9QoILIMbUKGPgbgZj_ZwLFQqCRQ3mq4YSD6NWU,4158
+datashare_python/config.py,sha256=ODJrwrOTZK9D90cUbNYqSUnu5vsyMWw_IVgXpSdDYJs,4464
 datashare_python/conftest.py,sha256=hgBfg5Df9Kf0tMPvrqsrgjR6LZx1jG8WsZiVLQLHhuI,13824
 datashare_python/constants.py,sha256=-Irwd2bbnFGIkLVJi_SiQ7PM2eNW3Ia_co4SE6z4j-A,352
 datashare_python/dependencies.py,sha256=Q-BIhCwWu-GGp9omYNhENZ88cF0QJ8spfaMZX40mmNU,4386
@@ -9,20 +9,20 @@ datashare_python/discovery.py,sha256=3mUck2GUSKu2DSP9-MM1OdVM4FqFKqfBeSQDQZxgh2g
 datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
 datashare_python/interceptors.py,sha256=-XC-y_MvGyHlZQ6jItfMRPAmKwQkQzzncpAGxceKlwM,16485
 datashare_python/logging_.py,sha256=96KNXhP-JR5ybu9dld4VLtF7v7PfiphOCR1Nio-mmUs,4483
-datashare_python/mimetypes_.py,sha256=mjQgaqnSFZEcl3LoLDGBaBBjTTzwYfSNZND9Fmkk2zs,49682
-datashare_python/objects.py,sha256=RuDEC4dZPndMYwFKhmMlToY8jjwjupG7mGGCG5tzc6g,18788
+datashare_python/mimetypes_.py,sha256=YAtNKewLJIXuTxdYYkprK237IOOw1vUGUVNiyZRdpQU,49874
+datashare_python/objects.py,sha256=Fy9LVFMTSWsgjJBhZyJrwnWFP0PKcsw3arJ8JwpumGc,26878
 datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
 datashare_python/template.py,sha256=N1uM8TRXh5tQPV7BC1oJj-Pdm2Gs_SkjaXv50RnNYso,4682
 datashare_python/types_.py,sha256=cPcCkIk6ecILrKetPnAsrb0njHYuWJU3eZicUXy2lF8,1278
-datashare_python/utils.py,sha256=MU_laix9TclXQ7Pt0mqqYQylYR3OKU-MItjQupcG3c8,30183
-datashare_python/worker-template.tar.gz,sha256=5U1vAMg7xtJSKAz20kqz7QLknvzmREN5ddqK7RvAYyI,236093
+datashare_python/utils.py,sha256=Y0rNipGGW4yhOK6GrhXP9cdmHWMis2oFUxRsNK9eBMU,33254
+datashare_python/worker-template.tar.gz,sha256=A0dXeN0Y4zZghAltznv5SvhXjpOdsWJDU1wny87uUlY,281066
 datashare_python/worker.py,sha256=ad36_yJOgwtj8KBheWndGkqaP_2aXTPSrYaHrbCXTvg,8389
 datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
 datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
 datashare_python/cli/task.py,sha256=GsyR5o0fN4Pd8tJKAUgivAvCBKT3L4Ay5j_BCW1XBHk,5828
 datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
 datashare_python/cli/worker.py,sha256=x0p3a1_NsZPbmV7-WSVMRyW3_iadYcgyZKbDvWgymQc,5790
-datashare_python-0.11.4.dist-info/METADATA,sha256=wesn34A1Y6Omeo3gftg-Jd4jiXo4a4hktN1blM3vCJI,1012
-datashare_python-0.11.4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-datashare_python-0.11.4.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
-datashare_python-0.11.4.dist-info/RECORD,,
+datashare_python-0.12.1.dist-info/METADATA,sha256=FiCE6tiVkAW0xz8za0fBqlE_VttoPwOxEhsnRk2-Mxk,1012
+datashare_python-0.12.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+datashare_python-0.12.1.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
+datashare_python-0.12.1.dist-info/RECORD,,