datashare-python 0.10.10__tar.gz → 0.11.1__tar.gz

{datashare_python-0.10.10 → datashare_python-0.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datashare-python
-Version: 0.10.10
+Version: 0.11.1
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.10.10 → datashare_python-0.11.1}/datashare_python/conftest.py

@@ -233,14 +233,22 @@ def text_1() -> str:
 @pytest.fixture(scope="session")
 def doc_0(text_0: str) -> Document:
     return Document(
-        id="doc-0", root_document="root-0", language="ENGLISH", content=text_0
+        id="doc-0",
+        root_document="root-0",
+        extraction_level=1,
+        language="ENGLISH",
+        content=text_0,
     )
 
 
 @pytest.fixture(scope="session")
 def doc_1(text_1: str) -> Document:
     return Document(
-        id="doc-1", root_document="root-1", language="ENGLISH", content=text_1
+        id="doc-1",
+        root_document="root-1",
+        extraction_level=1,
+        language="ENGLISH",
+        content=text_1,
     )
 
 
@@ -249,6 +257,7 @@ def doc_2() -> Document:
     return Document(
         id="doc-2",
         root_document="root-2",
+        extraction_level=1,
         language="FRENCH",
         content="traduis ce texte en anglais",
     )
@@ -260,6 +269,7 @@ def doc_3() -> Document:
         id="doc-3",
         index=TEST_PROJECT,
         root_document="root-3",
+        extraction_level=1,
         language="SPANISH",
         content="traduce este texto al inglés",
     )

{datashare_python-0.10.10 → datashare_python-0.11.1}/datashare_python/objects.py

@@ -24,6 +24,7 @@ with workflow.unsafe.imports_passed_through():
     from icij_common.es import (
         DOC_CONTENT,
         DOC_CONTENT_TRANSLATED,
+        DOC_EXTRACTION_LEVEL,
         DOC_LANGUAGE,
         DOC_METADATA,
         DOC_PATH,
@@ -222,6 +223,8 @@ class Document(DatashareModel):
     language: DatashareLanguage
     index: str | None = None
     root_document: str | None = None
+    content: Annotated[str, BeforeValidator(_from_sentences)]
+    extraction_level: int = 0
     content: str | None = None
     content_text_length: int | None = None
     content_type: str | None = None
@@ -244,6 +247,37 @@ class Document(DatashareModel):
                 data["content_text_length"] = len(content)
         return data
 
+    @model_validator(mode="before")
+    @classmethod
+    def _set_root_document_to_self_when_not_provided(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            root = data.get("root_document")
+            if root is None:
+                data["root_document"] = data.get("id")
+        return data
+
+    @model_validator(mode="after")
+    def _validate_root_document(self) -> Any:
+        if self.root_document is None:
+            msg = (
+                "inconsistent state, root_document cannot be after validation, "
+                "there's a bug in this object's implementation."
+                " Please report to a developer."
+            )
+            raise ValueError(msg)
+        if self.is_root_document:
+            if self.extraction_level > 0:
+                msg = "extraction_level should be <= 0 for root documents"
+                raise ValueError(msg)
+        elif self.extraction_level <= 0:
+            msg = "extraction_level should be > 0 for embedded documents"
+            raise ValueError(msg)
+        return self
+
+    @property
+    def is_root_document(self) -> bool:
+        return self.root_document == self.id and self.extraction_level <= 0
+
     @classmethod
     def from_es(cls, es_doc: dict) -> Self:
         sources = es_doc[SOURCE]
@@ -255,6 +289,7 @@ class Document(DatashareModel):
             content_text_length=sources.get("content_text_length"),
             language=DatashareLanguage(sources[DOC_LANGUAGE]),
             root_document=sources.get(DOC_ROOT_ID),
+            extraction_level=sources.get(DOC_EXTRACTION_LEVEL),
             tags=sources.get("tags", []),
             path=sources.get(DOC_PATH),
             metadata=sources.get(DOC_METADATA),
@@ -269,13 +304,17 @@ class Document(DatashareModel):
             )
         resource_name = self.metadata.get(TIKA_METADATA_RESOURCENAME)
         if resource_name is None:
-            msg = (
-                f"doc {self} can't turn ES document into a {ProcessedFile.__name__} "
-                f"without tika metadata resources name in metadata: {self.metadata}"
-            )
-            raise ValueError(msg)
+            # Backward compat for project before resourcename was mandatory
+            if self.is_root_document:
+                resource_name = self.path.name
+            else:
+                msg = (
+                    f"doc {self} can't turn ES document into a {ProcessedFile.__name__} "
+                    f"without tika metadata resources name in metadata: {self.metadata}"
+                )
+                raise ValueError(msg)
         resource_name = cast(str, resource_name)
-        if self.root_document is None:
+        if self.is_root_document:
             path = self.path
             location = DocumentLocation.FILESYSTEM
         else:

{datashare_python-0.10.10 → datashare_python-0.11.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.10.10"
+version = "0.11.1"
 description = "Manage Python tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },