datashare-python 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

datashare_python/cli/task.py

@@ -10,8 +10,7 @@ import typer
 from alive_progress import alive_bar
 
 from datashare_python.cli.utils import AsyncTyper, eprint
-from datashare_python.constants import PYTHON_TASK_GROUP
-from datashare_python.objects import READY_STATES, Task, TaskError, TaskState
+from datashare_python.objects import READY_STATES, Task, TaskError, TaskGroup, TaskState
 from datashare_python.task_client import DatashareTaskClient
 
 logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ async def start(
     group: Annotated[
         str | None,
         typer.Option("--group", "-g", help=_GROUP_HELP),
-    ] = PYTHON_TASK_GROUP.name,
+    ] = TaskGroup.python,  # noqa: F821
     ds_address: Annotated[
         str, typer.Option("--ds-address", "-a", help=_DS_URL_HELP)
     ] = DEFAULT_DS_ADDRESS,

datashare_python/config.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import ClassVar
 
 from icij_common.es import ESClient
@@ -87,6 +88,10 @@ class WorkerConfig(ICIJSettings, LogWithWorkerIDMixin, BaseModel):
 
     max_concurrent_io_activities: int = 5
 
+    docs_root: Path | None = None
+    artifacts_root: Path | None = None
+    workdir: Path | None = None
+
     def to_es_client(self) -> ESClient:
         return self.elasticsearch.to_es_client(self.datashare.api_key)
 

datashare_python/conftest.py

@@ -262,6 +262,7 @@ def doc_2() -> Document:
 def doc_3() -> Document:
     return Document(
         id="doc-3",
+        index=TEST_PROJECT,
         root_document="root-3",
         language="SPANISH",
         content="traduce este texto al inglés",

datashare_python/constants.py

@@ -1,11 +1,8 @@
 from pathlib import Path
 
-from .objects import TaskGroup
-
 PACKAGE_DIR = Path(__file__).parent
 PACKAGE_ROOT = PACKAGE_DIR.parent
 
-PYTHON_TASK_GROUP = TaskGroup(name="PYTHON")
 
 DEFAULT_TEMPORAL_ADDRESS = "temporal:7233"
 
@@ -14,3 +11,5 @@ DEFAULT_DS_ADDRESS = "http://localhost:8080"
 DEFAULT_NAMESPACE = "datashare-default"
 
 METADATA_JSON = "metadata.json"
+
+TIKA_METADATA_RESOURCENAME = "tika_metadata_resourcename"

datashare_python/objects.py

@@ -5,12 +5,24 @@ from datetime import UTC, datetime
 from enum import StrEnum, unique
 from io import BytesIO
 from pathlib import Path
-from typing import Any, Literal, Self, TypeVar
+from typing import Any, Literal, Self, TypeVar, cast
 
 from temporalio import workflow
 
+from .constants import TIKA_METADATA_RESOURCENAME
+
 with workflow.unsafe.imports_passed_through():
-    from icij_common.es import DOC_CONTENT, DOC_LANGUAGE, DOC_ROOT_ID, ID_, SOURCE
+    from icij_common.es import (
+        DOC_CONTENT,
+        DOC_CONTENT_TRANSLATED,
+        DOC_LANGUAGE,
+        DOC_METADATA,
+        DOC_PATH,
+        DOC_ROOT_ID,
+        ID_,
+        INDEX_,
+        SOURCE,
+    )
 
 from icij_common.pydantic_utils import (
     icij_config,
@@ -137,11 +149,48 @@ class Task(Message):
 class TaskGroup:
     name: str
 
+    @property
+    @classmethod
+    def python(cls) -> Self:
+        return cls(name="PYTHON")
+
+
+@unique
+class DocumentLocation(StrEnum):
+    ORIGINAL = "original"
+    ARTIFACTS = "artifacts"
+    WORKDIR = "workdir"
+
+
+class FilesystemDocument(DatashareModel):
+    id: str
+    path: Path
+    index: str
+    location: DocumentLocation
+    resource_name: str
+
+    def locate(
+        self, original_root: Path, *, artifacts_root: Path, workdir: Path
+    ) -> Path:
+        from datashare_python.utils import artifacts_dir  # noqa: PLC0415
+
+        project = self.index
+        match self.location:
+            case DocumentLocation.ORIGINAL:
+                return original_root / self.path
+            case DocumentLocation.ARTIFACTS:
+                return artifacts_root / artifacts_dir(self.id, project=project) / "raw"
+            case DocumentLocation.WORKDIR:
+                return workdir / self.path
+            case _:
+                raise ValueError(f"invalid location: {self.path}")
+
 
 class Document(DatashareModel):
     id: str
-    root_document: str
     language: str
+    index: str | None = None
+    root_document: str | None = None
     content: str | None = None
     content_type: str | None = None
     path: Path | None = None
@@ -149,6 +198,7 @@ class Document(DatashareModel):
     content_translated: dict[str, str] = Field(
         default_factory=dict, alias="content_translated"
     )
+    metadata: dict[str, Any] | None = None
     type: str = Field(default="Document", frozen=True)
 
     @classmethod
@@ -156,11 +206,42 @@ class Document(DatashareModel):
         sources = es_doc[SOURCE]
         return cls(
             id=es_doc[ID_],
-            content=sources[DOC_CONTENT],
-            content_translated=sources.get("content_translated", dict()),
+            index=es_doc.get(INDEX_),
+            content=sources.get(DOC_CONTENT),
+            content_translated=sources.get(DOC_CONTENT_TRANSLATED, dict()),
             language=sources[DOC_LANGUAGE],
-            root_document=sources[DOC_ROOT_ID],
+            root_document=sources.get(DOC_ROOT_ID),
             tags=sources.get("tags", []),
+            path=sources.get(DOC_PATH),
+            metadata=sources.get(DOC_METADATA),
+        )
+
+    def to_filesystem(self) -> FilesystemDocument:
+        from .utils import artifacts_dir  # noqa: PLC0415
+
+        if self.metadata is None:
+            raise ValueError(
+                "can't compute filesyste path for document withtout metadata"
+            )
+        resource_name = cast(str, self.metadata[TIKA_METADATA_RESOURCENAME])
+        if self.root_document is None:
+            path = self.path
+            location = DocumentLocation.ORIGINAL
+        else:
+            if self.index is None:
+                msg = (
+                    f"can't compute filesystem path for embedded doc {self.id} without"
+                    f" index"
+                )
+                raise ValueError(msg)
+            path = artifacts_dir(doc_id=self.id, project=self.index) / "raw"
+            location = DocumentLocation.ARTIFACTS
+        return FilesystemDocument(
+            id=self.id,
+            path=path,
+            index=self.index,
+            location=location,
+            resource_name=resource_name,
         )
 
 

datashare_python/utils.py

@@ -440,12 +440,12 @@ def safe_dir(doc_id: str) -> Path:
     return Path(*parts)
 
 
-def _artifacts_dir(doc_id: str, *, project: str) -> Path:
+def artifacts_dir(doc_id: str, *, project: str) -> Path:
     return Path(project, safe_dir(doc_id), doc_id)
 
 
 def _metadata_path(doc_id: str, *, project: str) -> Path:
-    metadata_path = _artifacts_dir(doc_id, project=project) / METADATA_JSON
+    metadata_path = artifacts_dir(doc_id, project=project) / METADATA_JSON
     return metadata_path
 
 
@@ -455,7 +455,7 @@ def _read_artifact_metadata(root: Path, artifact: DocArtifact) -> dict:
 
 
 def write_artifact(root: Path, artifact: DocArtifact) -> Path:
-    artif_dir = root / _artifacts_dir(artifact.doc_id, project=artifact.project)
+    artif_dir = root / artifacts_dir(artifact.doc_id, project=artifact.project)
     artif_dir.mkdir(exist_ok=True, parents=True)
     # TODO: if transcriptions are too large we could also serialize them
     #  as jsonl
@@ -479,17 +479,23 @@ def debuggable_name(
     displayable_file_name = [c[:component_size_limit] for c in path.parts]
     uuid = sha256(str(path).encode()).hexdigest() if deterministic else uuid4().hex
     uuid = uuid[:20]
-    return f"{uuid}-{'__'.join(displayable_file_name)}"
+    return f"{uuid}-{'--'.join(displayable_file_name)}"
 
 
-def activity_contextual_id(*, wf_context: bool = False) -> str:
+def activity_contextual_id(
+    *, wf_context: bool = True, act_context: bool = False, run_context: bool = False
+) -> str:
+    contextual_id = []
     act_info = activity.info()
-    act_id = act_info.activity_id
-    act_run_id = act_info.activity_id
-    act_type = act_info.activity_type
-    contextual_id = f"{act_type}-{act_id}-{act_run_id}"
+    if not wf_context and not act_context:
+        raise ValueError("at least one of wf_context and act_context must be True")
     if wf_context:
-        wf_id = act_info.workflow_id
-        wf_run_id = act_info.workflow_run_id
-        contextual_id += f"-{wf_id}-{wf_run_id}"
-    return contextual_id
+        contextual_id.append(act_info.workflow_id)
+        if run_context:
+            contextual_id.append(act_info.workflow_run_id)
+    if act_context:
+        contextual_id.append(act_info.activity_type)
+        contextual_id.append(act_info.activity_id)
+        if run_context:
+            contextual_id.append(act_info.activity_run_id)
+    return "-".join(contextual_id)

{datashare_python-0.6.0.dist-info → datashare_python-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datashare-python
-Version: 0.6.0
+Version: 0.6.2
 Summary: Manage Pythoœn tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.6.0.dist-info → datashare_python-0.6.2.dist-info}/RECORD

@@ -1,25 +1,25 @@
 datashare_python/.gitignore,sha256=e-SRgnvGGdsjRrqgKsTzALz6Obx8IYiOjr0yaAxT6v8,22
 datashare_python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datashare_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
-datashare_python/config.py,sha256=p-uBTle30kbUdqrj8rXcYv2gHiNwtqcYMnGi4Kctumk,3683
-datashare_python/conftest.py,sha256=JfVoi2MXx5XVAyQV4XxRQ6bHh-26vj56Wtjn-Zrgefk,8484
-datashare_python/constants.py,sha256=e6Px11OUee9GSHwTgsgFMszGCMwpW-OznHSMgINvepc,338
+datashare_python/config.py,sha256=Mn43zAqUH8bmVpPQ4DK42ffg5LmRtC7ZHHgBlIINlD4,3814
+datashare_python/conftest.py,sha256=zUEB9d3CcYVVfsuEXkvF1G5JCEchTknB6pFUiM8FbXA,8512
+datashare_python/constants.py,sha256=a8-ceZKBVMXydcoNQ35fSjFjxeJ7dt-N6eAvqtPpf9g,320
 datashare_python/dependencies.py,sha256=4UsVFKRjd2Q0ghg_fUU24P26tFYhg_SnAENj2mKErrY,4060
 datashare_python/discovery.py,sha256=UsfIb_pL56BQ5i5xvfuhOO0bDefjA_oToxrCeJQQbcU,6925
 datashare_python/exceptions.py,sha256=bVHEAXxDPKfxeeMC0hJXEsrJkgsKO2ESAhxWU96GA4M,496
-datashare_python/objects.py,sha256=hTtkmuQIlQycJsBFgK3wU9emrIXzWmc3sOuEIGpniNI,4581
+datashare_python/objects.py,sha256=mz025_6DogMMXSk5PqEctp0slvY4EKkuMDm80Kpm5_A,7020
 datashare_python/task_client.py,sha256=oTmP8bvZW0UyhLNMi1AV3XIAx7hrdbxNRss2Mw2azEc,8435
 datashare_python/template.py,sha256=RxKTYLXoS_EQ8Jc41JkBXppPdbCFqDWfP3BmC0gvB5o,4024
 datashare_python/types_.py,sha256=9Hk1XqpdXbM1TnEzwvJ5G9ABbaCZW9KgBTtiPBVn_7k,649
-datashare_python/utils.py,sha256=qnBrK-oefv6OiL9XUBImsuTUdpwgi2lE97kVv4M5X94,16974
-datashare_python/worker-template.tar.gz,sha256=cH5LwVFfdhzooE81PtC2sL6NFThLS2274Wt63XRv5TE,274869
+datashare_python/utils.py,sha256=ZGZKO9Q4_aLVVilZUCkmHQ21M_37hVOCr7G-qZPOflU,17234
+datashare_python/worker-template.tar.gz,sha256=f_9jyCcZDTgQvzl7K3vD5Ved9MaHXUV9Ox_uZWzub10,274849
 datashare_python/worker.py,sha256=UdSCWZw8qhkhzok89oU7J35VTDZwxRAqef-5Z8yt95A,6333
 datashare_python/cli/__init__.py,sha256=9BPWtssDgsVfWMsZ1TtZCla0EC_kai4RHttr8oNLYOE,1401
 datashare_python/cli/project.py,sha256=w32Gy9AOL5B00uDT4in7YUCt2g68FnNbvwg2M3a8G6o,946
-datashare_python/cli/task.py,sha256=9If5OC7loG4C4gWWl4iOeqPJ4GOLlCWXQfuNLUHORrQ,5860
+datashare_python/cli/task.py,sha256=8mvKGS21bZ14BgZ0Uo-dfameljkaI2ZBha80ywCy-E8,5822
 datashare_python/cli/utils.py,sha256=p69CQb0zfixuyBkiZprhdMCc_NuYwXyAn6vC9H1UzAw,911
 datashare_python/cli/worker.py,sha256=I4KTpFIpXFowioFn72Rm6LBCYlY-Dhp4NBIPvtRgUXE,5283
-datashare_python-0.6.0.dist-info/METADATA,sha256=JIEU1lFTYPVZA9chJpchkMc5ZalduhrI89v2uT8ZwIk,907
-datashare_python-0.6.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-datashare_python-0.6.0.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
-datashare_python-0.6.0.dist-info/RECORD,,
+datashare_python-0.6.2.dist-info/METADATA,sha256=-tF3-ZSn0AA_xOJe-25-cJUnE-am9-yqDBRZqeNWbFs,907
+datashare_python-0.6.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+datashare_python-0.6.2.dist-info/entry_points.txt,sha256=ILE7auxabHWiu3GC-AunWnzjhOI_SbZp7D4GqZHlLw4,68
+datashare_python-0.6.2.dist-info/RECORD,,