mmar-mapi 1.0.18__py3-none-any.whl → 1.0.20__py3-none-any.whl

mmar_mapi/api.py

@@ -1,11 +1,14 @@
+from enum import StrEnum
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel
+
+from mmar_mapi.file_storage import ResourceId
 from mmar_mapi.models.chat import Chat, ChatMessage
 from mmar_mapi.models.tracks import DomainInfo, TrackInfo
-from pydantic import BaseModel
-
 
 Value = str
 Interpretation = str
-ResourceId = str
 
 
 class ChatManagerAPI:
@@ -60,3 +63,89 @@ class ContentInterpreterAPI:
 class TextProcessorAPI:
     def process(self, *, text: str, chat: Chat | None = None) -> str:
         raise NotImplementedError
+
+
+class TextExtractorAPI:
+    def extract(self, *, resource_id: ResourceId) -> ResourceId:
+        """returns file with text"""
+        raise NotImplementedError
+
+
+PageRange = Annotated[tuple[int, int], AfterValidator(lambda rng: rng[0] <= rng[1])]
+ForceOCR = StrEnum("ForceOCR", ["ENABLED", "DISABLED", "AUTO"])
+OutputType = StrEnum("OutputType", ["RAW", "PLAIN", "MARKDOWN"])
+
+
+class ExtractionEngineSpec(BaseModel):
+    output_type: OutputType = OutputType.MARKDOWN
+    force_ocr: ForceOCR = ForceOCR.AUTO
+    do_ocr: bool = False
+    do_table_structure: bool = False
+    do_cell_matching: bool = False
+    do_annotations: bool = False
+    do_image_extraction: bool = False
+    generate_page_images: bool = False
+    images_scale: float = 2.0
+
+
+class DocExtractionSpec(BaseModel):
+    page_range: PageRange | None = None
+    engine: ExtractionEngineSpec = ExtractionEngineSpec()
+
+    def _update(self, **update):
+        return self.model_copy(update=update)
+
+    def _update_engine(self, **engine_update):
+        return self._update(engine=self.engine.model_copy(update=engine_update))
+
+    # fmt: off
+    def with_output_type_raw(self): return self._update_engine(output_type=OutputType.RAW)
+    def with_output_type_plain(self): return self._update_engine(output_type=OutputType.PLAIN)
+    def with_ocr(self): return self._update_engine(do_ocr=True)
+    def with_tables(self): return self._update_engine(do_table_structure=True, do_cell_matching=True)
+    def with_images(self): return self._update_engine(do_image_extraction=True)
+    def with_annotations(self): return self._update_engine(do_annotations=True)
+    def with_force_ocr_enabled(self): return self._update_engine(force_ocr=ForceOCR.ENABLED)
+    def with_force_ocr_disabled(self): return self._update_engine(force_ocr=ForceOCR.DISABLED)
+    def with_page_images(self): return self._update_engine(generate_page_images=True)
+
+    def with_page_range(self, page_range: PageRange): return self._update(page_range=page_range)
+    # fmt: on
+
+
+class ExtractedImage(BaseModel):
+    page: int
+    image_resource_id: ResourceId | None = None
+
+
+class ExtractedImageMetadata(BaseModel):
+    annotation: str = ""
+    caption: str = ""
+    width: int | None = None
+    height: int | None = None
+
+
+class ExtractedPicture(ExtractedImage, ExtractedImageMetadata):
+    pass
+
+
+class ExtractedTable(ExtractedImage, ExtractedImageMetadata):
+    formatted_str: str
+
+
+class ExtractedPageImage(ExtractedImage):
+    pass
+
+
+class DocExtractionOutput(BaseModel):
+    spec: DocExtractionSpec
+    text: str = ""
+    tables: list[ExtractedTable] = []
+    pictures: list[ExtractedPicture] = []
+    page_images: list[ExtractedPageImage] = []
+
+
+class DocumentExtractorAPI:
+    def extract(self, *, resource_id: ResourceId, spec: DocExtractionSpec) -> ResourceId | None:
+        """returns file with DocExtractionOutput"""
+        raise NotImplementedError

mmar_mapi/file_storage.py

@@ -45,7 +45,7 @@ class FileStorage:
         resource_id = self.upload(content, fname)
         return resource_id
 
-    def upload(self, content: bytes | str, fname: str) -> ResourceId:
+    def upload(self, content: bytes | str, fname: str, origin: str | None = None) -> ResourceId:
         if isinstance(content, str):
             content = content.encode()
 
@@ -56,7 +56,7 @@ class FileStorage:
 
         fpath_md = fpath.with_suffix(SUFFIX_METADATA)
         update_date = f"{datetime.now():%Y-%m-%d--%H-%M-%S}"
-        metadata = {"fname": fname, "update_date": update_date, "size": len(content)}
+        metadata = {"fname": fname, "update_date": update_date, "size": len(content), "origin": origin}
         fpath_md.write_text(json.dumps(metadata, ensure_ascii=False))
 
         return str(fpath)
@@ -67,6 +67,12 @@ class FileStorage:
             return None
         return json.loads(metadata_path.read_text())
 
+    def get_fname(self, resource_id: ResourceId) -> str | None:
+        metadata = self.get_metadata(resource_id)
+        if metadata is None:
+            return None
+        return metadata.get("fname")
+
     async def upload_async(self, content: bytes | str, fname: str) -> ResourceId:
         return self.upload(content, fname)
 
@@ -90,6 +96,9 @@ class FileStorage:
         res = self.download_text(resource_id).split("\n")
         return res
 
+    def get_path(self, resource_id: ResourceId | None) -> Path | None:
+        return self._get_path(resource_id)
+
     def _get_path(self, resource_id: ResourceId | None) -> Path | None:
         if not resource_id:
             return None

{mmar_mapi-1.0.18.dist-info → mmar_mapi-1.0.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mmar-mapi
-Version: 1.0.18
+Version: 1.0.20
 Summary: Common pure/IO utilities for multi-modal architectures team
 Keywords: 
 Author: Eugene Tagin

{mmar_mapi-1.0.18.dist-info → mmar_mapi-1.0.20.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mmar_mapi/__init__.py,sha256=9Q5xsrj26uUnn7ZWvvJUvdVIuzC2oCIeNB4dEoqjF-o,1256
-mmar_mapi/api.py,sha256=C9Sr8dISvf51xfEznPjccI_odaG4coQE3HI_0jVpjMQ,1677
+mmar_mapi/api.py,sha256=LCWO4HN8mAmgpZI5KJ5MSZwI55Y7hWuplOo1e3EGC_I,4670
 mmar_mapi/decorators_maybe_lru_cache.py,sha256=eO2I6t1fHLUNRABClK1c8EZzHAmCeSK6O-hbJGb2c9E,444
-mmar_mapi/file_storage.py,sha256=RNPHKDV7JIo2ZlSOyi7UfE8q7kpMvv7ZzfpTdoVg1vM,4687
+mmar_mapi/file_storage.py,sha256=xJU59HmXFsfc53XALdx53IwyqV_k4218AzzXq1Q65Js,5052
 mmar_mapi/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mmar_mapi/models/base.py,sha256=mKtXV2x51XVj7W-et9tjGcPMDUUUMelW-BywMgFc2p0,411
 mmar_mapi/models/chat.py,sha256=-XilkiderIOFG1oSKRDG9NDOEN21sBpbTPHUrqVPjc4,15400
@@ -13,7 +13,7 @@ mmar_mapi/type_union.py,sha256=diwmzcnbqkpGFckPHNw9o8zyQ955mOGNvhTlcBJ0RMI,1905
 mmar_mapi/utils.py,sha256=FlW9n-84xz2zSHsahHzJ3Y4Wu5mjpFer6t9z6PF6lS0,488
 mmar_mapi/utils_import.py,sha256=pUyMFd8SItTxBKI-GO9JhRmy43jG_OQlUPr8QCBOSwg,1682
 mmar_mapi/xml_parser.py,sha256=VvLIX_XCZao9i0qqpTVx8nx0vbFXSe8pEbdJdXnj97g,568
-mmar_mapi-1.0.18.dist-info/licenses/LICENSE,sha256=2A90w8WjhOgQXnFuUijKJYazaqZ4_NTokYb9Po4y-9k,1061
-mmar_mapi-1.0.18.dist-info/WHEEL,sha256=-neZj6nU9KAMg2CnCY6T3w8J53nx1kFGw_9HfoSzM60,79
-mmar_mapi-1.0.18.dist-info/METADATA,sha256=EIQ57TyO_0Ur_a_04YCmNztCgtNjGmTlGfxXWWe9v2g,944
-mmar_mapi-1.0.18.dist-info/RECORD,,
+mmar_mapi-1.0.20.dist-info/licenses/LICENSE,sha256=2A90w8WjhOgQXnFuUijKJYazaqZ4_NTokYb9Po4y-9k,1061
+mmar_mapi-1.0.20.dist-info/WHEEL,sha256=-neZj6nU9KAMg2CnCY6T3w8J53nx1kFGw_9HfoSzM60,79
+mmar_mapi-1.0.20.dist-info/METADATA,sha256=IoMO8Hdd07-s3F4aC3OuA7dgYgZddbXiK7ES5Jwnyck,944
+mmar_mapi-1.0.20.dist-info/RECORD,,