mmar-mapi 1.0.18__tar.gz → 1.0.20__tar.gz

{mmar_mapi-1.0.18 → mmar_mapi-1.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mmar-mapi
-Version: 1.0.18
+Version: 1.0.20
 Summary: Common pure/IO utilities for multi-modal architectures team
 Keywords: 
 Author: Eugene Tagin

{mmar_mapi-1.0.18 → mmar_mapi-1.0.20}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mmar-mapi"
 # dynamic version is not supported yet on uv_build
-version = "1.0.18"
+version = "1.0.20"
 description = "Common pure/IO utilities for multi-modal architectures team"
 authors = [{name = "Eugene Tagin", email = "tagin@airi.net"}]
 license = "MIT"

mmar_mapi-1.0.20/src/mmar_mapi/api.py

@@ -0,0 +1,151 @@
+from enum import StrEnum
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel
+
+from mmar_mapi.file_storage import ResourceId
+from mmar_mapi.models.chat import Chat, ChatMessage
+from mmar_mapi.models.tracks import DomainInfo, TrackInfo
+
+Value = str
+Interpretation = str
+
+
+class ChatManagerAPI:
+    def get_domains(self, *, client_id: str, language_code: str = "ru") -> list[DomainInfo]:
+        raise NotImplementedError
+
+    def get_tracks(self, *, client_id: str, language_code: str = "ru") -> list[TrackInfo]:
+        raise NotImplementedError
+
+    def get_response(self, *, chat: Chat) -> list[ChatMessage]:
+        raise NotImplementedError
+
+
+class TextGeneratorAPI:
+    def process(self, *, chat: Chat) -> str:
+        raise NotImplementedError
+
+
+class ContentInterpreterRemoteResponse(BaseModel):
+    interpretation: str
+    resource_fname: str
+    resource: bytes
+
+
+class ContentInterpreterRemoteAPI:
+    def interpret_remote(
+        self, *, kind: str, query: str, resource: bytes, chat: Chat | None = None
+    ) -> ContentInterpreterRemoteResponse:
+        raise NotImplementedError
+
+
+class ClassifierAPI:
+    def get_values(self) -> list[Value]:
+        raise NotImplementedError
+
+    def evaluate(self, *, chat: Chat) -> Value:
+        raise NotImplementedError
+
+
+class CriticAPI:
+    def evaluate(self, *, text: str, chat: Chat | None = None) -> float:
+        raise NotImplementedError
+
+
+class ContentInterpreterAPI:
+    def interpret(
+        self, *, kind: str, query: str, resource_id: str = "", chat: Chat | None = None
+    ) -> tuple[Interpretation, ResourceId | None]:
+        raise NotImplementedError
+
+
+class TextProcessorAPI:
+    def process(self, *, text: str, chat: Chat | None = None) -> str:
+        raise NotImplementedError
+
+
+class TextExtractorAPI:
+    def extract(self, *, resource_id: ResourceId) -> ResourceId:
+        """returns file with text"""
+        raise NotImplementedError
+
+
+PageRange = Annotated[tuple[int, int], AfterValidator(lambda rng: rng[0] <= rng[1])]
+ForceOCR = StrEnum("ForceOCR", ["ENABLED", "DISABLED", "AUTO"])
+OutputType = StrEnum("OutputType", ["RAW", "PLAIN", "MARKDOWN"])
+
+
+class ExtractionEngineSpec(BaseModel):
+    output_type: OutputType = OutputType.MARKDOWN
+    force_ocr: ForceOCR = ForceOCR.AUTO
+    do_ocr: bool = False
+    do_table_structure: bool = False
+    do_cell_matching: bool = False
+    do_annotations: bool = False
+    do_image_extraction: bool = False
+    generate_page_images: bool = False
+    images_scale: float = 2.0
+
+
+class DocExtractionSpec(BaseModel):
+    page_range: PageRange | None = None
+    engine: ExtractionEngineSpec = ExtractionEngineSpec()
+
+    def _update(self, **update):
+        return self.model_copy(update=update)
+
+    def _update_engine(self, **engine_update):
+        return self._update(engine=self.engine.model_copy(update=engine_update))
+
+    # fmt: off
+    def with_output_type_raw(self): return self._update_engine(output_type=OutputType.RAW)
+    def with_output_type_plain(self): return self._update_engine(output_type=OutputType.PLAIN)
+    def with_ocr(self): return self._update_engine(do_ocr=True)
+    def with_tables(self): return self._update_engine(do_table_structure=True, do_cell_matching=True)
+    def with_images(self): return self._update_engine(do_image_extraction=True)
+    def with_annotations(self): return self._update_engine(do_annotations=True)
+    def with_force_ocr_enabled(self): return self._update_engine(force_ocr=ForceOCR.ENABLED)
+    def with_force_ocr_disabled(self): return self._update_engine(force_ocr=ForceOCR.DISABLED)
+    def with_page_images(self): return self._update_engine(generate_page_images=True)
+
+    def with_page_range(self, page_range: PageRange): return self._update(page_range=page_range)
+    # fmt: on
+
+
+class ExtractedImage(BaseModel):
+    page: int
+    image_resource_id: ResourceId | None = None
+
+
+class ExtractedImageMetadata(BaseModel):
+    annotation: str = ""
+    caption: str = ""
+    width: int | None = None
+    height: int | None = None
+
+
+class ExtractedPicture(ExtractedImage, ExtractedImageMetadata):
+    pass
+
+
+class ExtractedTable(ExtractedImage, ExtractedImageMetadata):
+    formatted_str: str
+
+
+class ExtractedPageImage(ExtractedImage):
+    pass
+
+
+class DocExtractionOutput(BaseModel):
+    spec: DocExtractionSpec
+    text: str = ""
+    tables: list[ExtractedTable] = []
+    pictures: list[ExtractedPicture] = []
+    page_images: list[ExtractedPageImage] = []
+
+
+class DocumentExtractorAPI:
+    def extract(self, *, resource_id: ResourceId, spec: DocExtractionSpec) -> ResourceId | None:
+        """returns file with DocExtractionOutput"""
+        raise NotImplementedError

{mmar_mapi-1.0.18 → mmar_mapi-1.0.20}/src/mmar_mapi/file_storage.py

@@ -45,7 +45,7 @@ class FileStorage:
         resource_id = self.upload(content, fname)
         return resource_id
 
-    def upload(self, content: bytes | str, fname: str) -> ResourceId:
+    def upload(self, content: bytes | str, fname: str, origin: str | None = None) -> ResourceId:
         if isinstance(content, str):
             content = content.encode()
 
@@ -56,7 +56,7 @@ class FileStorage:
 
         fpath_md = fpath.with_suffix(SUFFIX_METADATA)
         update_date = f"{datetime.now():%Y-%m-%d--%H-%M-%S}"
-        metadata = {"fname": fname, "update_date": update_date, "size": len(content)}
+        metadata = {"fname": fname, "update_date": update_date, "size": len(content), "origin": origin}
         fpath_md.write_text(json.dumps(metadata, ensure_ascii=False))
 
         return str(fpath)
@@ -67,6 +67,12 @@ class FileStorage:
             return None
         return json.loads(metadata_path.read_text())
 
+    def get_fname(self, resource_id: ResourceId) -> str | None:
+        metadata = self.get_metadata(resource_id)
+        if metadata is None:
+            return None
+        return metadata.get("fname")
+
     async def upload_async(self, content: bytes | str, fname: str) -> ResourceId:
         return self.upload(content, fname)
 
@@ -90,6 +96,9 @@ class FileStorage:
         res = self.download_text(resource_id).split("\n")
         return res
 
+    def get_path(self, resource_id: ResourceId | None) -> Path | None:
+        return self._get_path(resource_id)
+
     def _get_path(self, resource_id: ResourceId | None) -> Path | None:
         if not resource_id:
             return None

mmar_mapi-1.0.18/src/mmar_mapi/api.py

@@ -1,62 +0,0 @@
-from mmar_mapi.models.chat import Chat, ChatMessage
-from mmar_mapi.models.tracks import DomainInfo, TrackInfo
-from pydantic import BaseModel
-
-
-Value = str
-Interpretation = str
-ResourceId = str
-
-
-class ChatManagerAPI:
-    def get_domains(self, *, client_id: str, language_code: str = "ru") -> list[DomainInfo]:
-        raise NotImplementedError
-
-    def get_tracks(self, *, client_id: str, language_code: str = "ru") -> list[TrackInfo]:
-        raise NotImplementedError
-
-    def get_response(self, *, chat: Chat) -> list[ChatMessage]:
-        raise NotImplementedError
-
-
-class TextGeneratorAPI:
-    def process(self, *, chat: Chat) -> str:
-        raise NotImplementedError
-
-
-class ContentInterpreterRemoteResponse(BaseModel):
-    interpretation: str
-    resource_fname: str
-    resource: bytes
-
-
-class ContentInterpreterRemoteAPI:
-    def interpret_remote(
-        self, *, kind: str, query: str, resource: bytes, chat: Chat | None = None
-    ) -> ContentInterpreterRemoteResponse:
-        raise NotImplementedError
-
-
-class ClassifierAPI:
-    def get_values(self) -> list[Value]:
-        raise NotImplementedError
-
-    def evaluate(self, *, chat: Chat) -> Value:
-        raise NotImplementedError
-
-
-class CriticAPI:
-    def evaluate(self, *, text: str, chat: Chat | None = None) -> float:
-        raise NotImplementedError
-
-
-class ContentInterpreterAPI:
-    def interpret(
-        self, *, kind: str, query: str, resource_id: str = "", chat: Chat | None = None
-    ) -> tuple[Interpretation, ResourceId | None]:
-        raise NotImplementedError
-
-
-class TextProcessorAPI:
-    def process(self, *, text: str, chat: Chat | None = None) -> str:
-        raise NotImplementedError