mmar-mapi 1.2.6__tar.gz

mmar_mapi-1.2.6/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 AIRI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mmar_mapi-1.2.6/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: mmar-mapi
+Version: 1.2.6
+Summary: Common pure/IO utilities for multi-modal architectures team
+Keywords: 
+Author: Eugene Tagin
+Author-email: Eugene Tagin <tagin@airi.net>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Documentation
+Classifier: Topic :: Software Development
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Dist: pydantic~=2.11.7
+Requires-Dist: loguru~=0.7.3
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# mmar-mapi
+
+Multimodal architectures Maestro API

mmar_mapi-1.2.6/README.md

@@ -0,0 +1,3 @@
+# mmar-mapi
+
+Multimodal architectures Maestro API

mmar_mapi-1.2.6/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "mmar-mapi"
+# dynamic version is not supported yet on uv_build
+version = "1.2.6"
+description = "Common pure/IO utilities for multi-modal architectures team"
+authors = [{name = "Eugene Tagin", email = "tagin@airi.net"}]
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.11"
+keywords = []
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Documentation",
+    "Topic :: Software Development",
+    "Topic :: Utilities",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pydantic~=2.11.7",
+    "loguru~=0.7.3",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.14,<0.9.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "mmar_mapi"
+source-exclude = [".ruff_cache"]
+
+[dependency-groups]
+maintain = [
+    "build>=1.2",
+    "git-changelog>=2.5",
+    "twine>=5.1",
+    "yore>=0.3.3",
+]
+ci = [
+    "ruff>=0.4",
+    "pytest>=8.2",
+    "pytest-cov>=5.0",
+    "pytest-randomly>=3.15",
+    "pytest-asyncio>=1.0.0",
+    "mypy>=1.10",
+    "types-markdown>=3.6",
+    "types-pyyaml>=6.0",
+]
+
+[tool.uv]
+default-groups = ["maintain", "ci"]
+
+[tool.ruff]
+line-length = 120
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[pytest]
+pythonpath = "src"
+python_files = [ "test_*.py" ]
+testpaths = [ "tests" ]
+
+# action:message_regex:warning_class:module_regex:line
+filterwarnings = [ "error" ]
+
+[mypy]
+ignore_missing_imports = true
+exclude = "tests/fixtures/"
+warn_unused_ignores = true
+show_error_codes = true

mmar_mapi-1.2.6/src/mmar_mapi/__init__.py

@@ -0,0 +1,47 @@
+from .file_storage import FileStorage, ResourceId
+from .models.base import Base
+from .models.chat import (
+    Chat,
+    Context,
+    ChatMessage,
+    AIMessage,
+    HumanMessage,
+    MiscMessage,
+    make_content,
+    Content,
+    BaseMessage,
+)
+from .models.enums import MTRSLabelEnum, DiagnosticsXMLTagEnum, MTRSXMLTagEnum, DoctorChoiceXMLTagEnum
+from .models.tracks import TrackInfo, DomainInfo
+from .models.widget import Widget
+from .utils import make_session_id, chunked
+from .xml_parser import XMLParser
+from .utils_import import load_main_objects
+from .decorators_maybe_lru_cache import maybe_lru_cache
+
+__all__ = [
+    "AIMessage",
+    "Base",
+    "BaseMessage",
+    "Chat",
+    "ChatMessage",
+    "Content",
+    "Context",
+    "DiagnosticsXMLTagEnum",
+    "DoctorChoiceXMLTagEnum",
+    "DomainInfo",
+    "FileStorage",
+    "HumanMessage",
+    "MTRSLabelEnum",
+    "MTRSXMLTagEnum",
+    "MiscMessage",
+    "ResourceId",
+    "TrackInfo",
+    "Widget",
+    "XMLParser",
+    "chunked",
+    "load_main_objects",
+    "make_content",
+    "make_session_id",
+    "maybe_lru_cache",
+]

mmar_mapi-1.2.6/src/mmar_mapi/api/__init__.py

@@ -0,0 +1,134 @@
+from pydantic import BaseModel
+
+from mmar_mapi.api.document_extractor import (
+    DocExtractionOutput,
+    DocExtractionSpec,
+    DocumentExtractorAPI,
+    ExtractedImage,
+    ExtractedImageMetadata,
+    ExtractedPageImage,
+    ExtractedPicture,
+    ExtractedTable,
+    ExtractionEngineSpec,
+    ForceOCR,
+    OutputType,
+    PageRange,
+)
+from mmar_mapi.api.llm_accessor import (
+    LCP,
+    RESPONSE_EMPTY,
+    Attachments,
+    LLMAccessorAPI,
+    LLMCallProps,
+    Message,
+    Messages,
+    Payload,
+    Request,
+    ResponseExt,
+    EntrypointInfo,
+    EntrypointsConfig,
+)
+from mmar_mapi.models.chat import Chat, ChatMessage
+from mmar_mapi.models.tracks import DomainInfo, TrackInfo
+
+# variable to prevent removing unused imports
+__imported__ = [
+    # llm_accessor
+    LLMCallProps,
+    LCP,
+    Attachments,
+    Message,
+    Messages,
+    Payload,
+    Request,
+    ResponseExt,
+    RESPONSE_EMPTY,
+    LLMAccessorAPI,
+    # document_extractor
+    PageRange,
+    ForceOCR,
+    OutputType,
+    ExtractionEngineSpec,
+    DocExtractionSpec,
+    ExtractedImage,
+    ExtractedImageMetadata,
+    ExtractedPicture,
+    ExtractedTable,
+    ExtractedPageImage,
+    DocExtractionOutput,
+    DocumentExtractorAPI,
+    EntrypointInfo,
+    EntrypointsConfig,
+]
+
+
+Interpretation = str
+ResourceId = str
+
+
+class ChatManagerAPI:
+    def get_domains(self, *, client_id: str, language_code: str = "ru") -> list[DomainInfo]:
+        raise NotImplementedError
+
+    def get_tracks(self, *, client_id: str, language_code: str = "ru") -> list[TrackInfo]:
+        raise NotImplementedError
+
+    def get_response(self, *, chat: Chat) -> list[ChatMessage]:
+        raise NotImplementedError
+
+
+class TextGeneratorAPI:
+    def process(self, *, chat: Chat) -> str:
+        raise NotImplementedError
+
+
+class ContentInterpreterRemoteResponse(BaseModel):
+    interpretation: str
+    resource_fname: str
+    resource: bytes
+
+
+class ContentInterpreterRemoteAPI:
+    def interpret_remote(
+        self, *, kind: str, query: str, resource: bytes, chat: Chat | None = None
+    ) -> ContentInterpreterRemoteResponse:
+        raise NotImplementedError
+
+
+class BinaryClassifiersAPI:
+    def get_classifiers(self) -> list[str]:
+        raise NotImplementedError
+
+    def evaluate(self, *, classifier: str | None = None, text: str) -> bool:
+        raise NotImplementedError
+
+
+class TranslatorAPI:
+    def get_lang_codes(self) -> list[str]:
+        raise NotImplementedError
+
+    def translate(self, *, text: str, lang_code_from: str | None = None, lang_code_to: str) -> str:
+        raise NotImplementedError
+
+
+class CriticAPI:
+    def evaluate(self, *, text: str, chat: Chat | None = None) -> float:  # TODO replace float with bool
+        raise NotImplementedError
+
+
+class ContentInterpreterAPI:
+    def interpret(
+        self, *, kind: str, query: str, resource_id: str = "", chat: Chat | None = None
+    ) -> tuple[Interpretation, ResourceId | None]:
+        raise NotImplementedError
+
+
+class TextProcessorAPI:
+    def process(self, *, text: str, chat: Chat | None = None) -> str:
+        raise NotImplementedError
+
+
+class TextExtractorAPI:
+    def extract(self, *, resource_id: ResourceId) -> ResourceId:
+        """returns file with text"""
+        raise NotImplementedError

mmar_mapi-1.2.6/src/mmar_mapi/api/document_extractor.py

@@ -0,0 +1,95 @@
+from enum import StrEnum
+from typing import Annotated
+
+from pydantic import AfterValidator, BaseModel
+
+
+def _validate_page_range(v: tuple[int, int]) -> tuple[int, int]:
+    if v[0] < 1 or v[1] < v[0]:
+        raise ValueError("Invalid page range: start must be ≥ 1 and end must be ≥ start.")
+    return v
+
+
+PageRange = Annotated[tuple[int, int], AfterValidator(_validate_page_range)]
+ForceOCR = StrEnum("ForceOCR", ["ENABLED", "DISABLED", "AUTO"])
+OutputType = StrEnum("OutputType", ["RAW", "PLAIN", "MARKDOWN"])
+ResourceId = str
+
+
+class ExtractionEngineSpec(BaseModel, frozen=True):
+    output_type: OutputType = OutputType.MARKDOWN
+    force_ocr: ForceOCR = ForceOCR.AUTO
+    do_ocr: bool = False
+    do_table_structure: bool = False
+    do_cell_matching: bool = False
+    do_annotations: bool = False
+    do_image_extraction: bool = False
+    generate_page_images: bool = False
+    images_scale: float = 2.0
+
+
+class DocExtractionSpec(BaseModel, frozen=True):
+    page_range: PageRange | None = None
+    engine: ExtractionEngineSpec = ExtractionEngineSpec()
+
+    def _update(self, **update):
+        return self.model_copy(update=update)
+
+    def _update_engine(self, **engine_update):
+        return self._update(engine=self.engine.model_copy(update=engine_update))
+
+    # fmt: off
+    def with_output_type_raw(self): return self._update_engine(output_type=OutputType.RAW)
+    def with_output_type_plain(self): return self._update_engine(output_type=OutputType.PLAIN)
+    def with_ocr(self): return self._update_engine(do_ocr=True)
+    def with_tables(self): return self._update_engine(do_table_structure=True, do_cell_matching=True)
+    def with_images(self): return self._update_engine(do_image_extraction=True)
+    def with_annotations(self): return self._update_engine(do_annotations=True)
+    def with_force_ocr_enabled(self): return self._update_engine(force_ocr=ForceOCR.ENABLED)
+    def with_force_ocr_disabled(self): return self._update_engine(force_ocr=ForceOCR.DISABLED)
+    def with_page_images(self): return self._update_engine(generate_page_images=True)
+
+    def with_page_range(self, page_range: PageRange): return self._update(page_range=page_range)
+    # fmt: on
+
+
+class ExtractedImage(BaseModel):
+    page: int
+    image_resource_id: ResourceId | None = None
+
+
+class ExtractedImageMetadata(BaseModel):
+    annotation: str = ""
+    caption: str = ""
+    width: int | None = None
+    height: int | None = None
+
+
+class ExtractedPicture(ExtractedImage, ExtractedImageMetadata):
+    "Image of part of page"
+
+    ...
+
+
+class ExtractedTable(ExtractedImage, ExtractedImageMetadata):
+    formatted_str: str
+
+
+class ExtractedPageImage(ExtractedImage):
+    "Image of all page"
+
+    pass
+
+
+class DocExtractionOutput(BaseModel):
+    spec: DocExtractionSpec
+    text: str = ""
+    tables: list[ExtractedTable] = []
+    pictures: list[ExtractedPicture] = []
+    page_images: list[ExtractedPageImage] = []
+
+
+class DocumentExtractorAPI:
+    def extract(self, *, resource_id: ResourceId, spec: DocExtractionSpec) -> ResourceId | None:
+        """returns file with DocExtractionOutput"""
+        raise NotImplementedError

mmar_mapi-1.2.6/src/mmar_mapi/api/llm_accessor.py

@@ -0,0 +1,93 @@
+from typing import Literal
+
+from pydantic import BaseModel
+
+from mmar_mapi import ChatMessage
+
+
+class EntrypointInfo(BaseModel):
+    entrypoint_key: str
+    caption: str
+
+class EntrypointsConfig(BaseModel):
+    entrypoints: list[EntrypointInfo]
+    default_entrypoint_key: str
+
+
+class LLMCallProps(BaseModel, frozen=True):
+    entrypoint_key: str = ""
+    attempts: int = 1
+
+
+LCP = LLMCallProps()
+ResourceId = str
+FileId = str
+Attachments = list[list[ResourceId]]
+
+
+class Message(BaseModel, frozen=True):
+    role: Literal["system", "assistant", "user"]
+    content: str
+
+    @staticmethod
+    def create(message: ChatMessage) -> "Message":
+        return _create_message(message=message)
+
+
+def _create_message(message: ChatMessage) -> Message | None:
+    role = "assistant" if message.is_ai else "user" if message.is_human else None
+    return Message(role=role, content=message.text) if role else None
+
+
+class Messages(BaseModel, frozen=True):
+    messages: list[Message]
+
+
+class Payload(Messages, frozen=True):
+    attachments: Attachments | None = None
+
+    def with_attachments(self, attachments: Attachments) -> "Payload":
+        return self.model_copy(update=dict(attachments=attachments))
+
+    def __repr__(self):
+        parts = [f"messages: {len(self.messages)}", self.attachments and "has attachments"]
+        payload_pretty = ", ".join(filter(None, parts))
+        return f"Payload({payload_pretty})"
+
+    @staticmethod
+    def create(user_text: str, resource_id: ResourceId = "") -> "Payload":
+        return _create_payload(user_text=user_text, resource_id=resource_id)
+
+
+def _create_payload(user_text: str, resource_id: ResourceId = ""):
+    payload = Payload(messages=[Message(role="user", content=user_text)])
+    if not resource_id:
+        return payload
+    else:
+        return payload.with_attachments(attachments=[[resource_id]])
+
+
+class ResponseExt(BaseModel):
+    text: str
+    resource_id: ResourceId | None = None
+
+
+RESPONSE_EMPTY = ResponseExt(text="")
+Request = str | Messages | Payload
+
+
+class LLMAccessorAPI:
+    def get_entrypoints_config(self) -> EntrypointsConfig:
+        raise NotImplementedError
+
+    def get_entrypoint_keys(self) -> list[str]:
+        raise NotImplementedError
+
+    def get_response(self, *, request: Request, props: LLMCallProps = LCP) -> str:
+        raise NotImplementedError
+
+    def get_response_ext(self, *, request: Request, props: LLMCallProps = LCP) -> ResponseExt:
+        raise NotImplementedError
+
+    def get_embedding(self, *, prompt: str, props: LLMCallProps = LCP) -> list[float] | None:
+        raise NotImplementedError

mmar_mapi-1.2.6/src/mmar_mapi/decorators_maybe_lru_cache.py

@@ -0,0 +1,14 @@
+from collections.abc import Callable
+from functools import lru_cache
+
+from loguru import logger
+
+
+def maybe_lru_cache(maxsize: int, func: Callable) -> tuple[str, Callable]:
+    if maxsize >= 0:
+        maxsize = maxsize or None
+        logger.info(f"Caching for {func.__name__}: enabled: maxsize={maxsize}")
+        func = lru_cache(maxsize=maxsize)(func)
+    else:
+        logger.info(f"Caching for {func.__name__}: disabled")
+    return func

mmar_mapi-1.2.6/src/mmar_mapi/file_storage.py

@@ -0,0 +1,141 @@
+import json
+import string
+from datetime import datetime
+from hashlib import md5
+from pathlib import Path
+from zipfile import ZipFile, is_zipfile
+
+ResourceId = str
+ASCII_DIGITS_SPECIAL = set(string.ascii_lowercase + string.digits + "-")
+SUFFIX_DIR = ".dir"
+SUFFIX_METADATA = ".metadata"
+
+
+def _validate_exist(files_dir):
+    if not files_dir.exists():
+        err = f"Failed to access file-storage directory: {files_dir}"
+        raise OSError(err)
+
+
+def _validate_dtype(dtype: str):
+    if all(map(ASCII_DIGITS_SPECIAL.__contains__, dtype)):
+        return
+    raise ValueError(f"Bad dtype: {dtype}")
+
+
+def generate_fname(content, dtype):
+    fname_hash = md5(content).hexdigest()
+    fname = f"{fname_hash}.{dtype}"
+    return fname
+
+
+class FileStorage:
+    def __init__(self, files_dir):
+        self.files_dir = Path(files_dir)
+        self.files_dir.mkdir(exist_ok=True, parents=True)
+        _validate_exist(self.files_dir)
+
+    def _generate_fname_path(self, content: bytes, dtype: str):
+        fpath = self.files_dir / generate_fname(content, dtype)
+        return fpath
+
+    def upload_maybe(self, content: bytes | str | None, fname: str) -> ResourceId | None:
+        if not content:
+            return None
+        resource_id = self.upload(content, fname)
+        return resource_id
+
+    def upload(self, content: bytes | str, fname: str, origin: str | None = None) -> ResourceId:
+        if isinstance(content, str):
+            content = content.encode()
+
+        dtype = fname.rsplit(".", 1)[-1]
+        _validate_dtype(dtype)
+        fpath = self._generate_fname_path(content, dtype)
+        fpath.write_bytes(content)
+
+        fpath_md = fpath.with_suffix(SUFFIX_METADATA)
+        update_date = f"{datetime.now():%Y-%m-%d--%H-%M-%S}"
+        metadata = {"fname": fname, "update_date": update_date, "size": len(content), "origin": origin}
+        fpath_md.write_text(json.dumps(metadata, ensure_ascii=False))
+
+        return str(fpath)
+
+    def get_metadata(self, resource_id: ResourceId) -> dict | None:
+        metadata_path = Path(resource_id).with_suffix(SUFFIX_METADATA)
+        if not metadata_path.exists():
+            return None
+        return json.loads(metadata_path.read_text())
+
+    def get_fname(self, resource_id: ResourceId) -> str | None:
+        metadata = self.get_metadata(resource_id)
+        if metadata is None:
+            return None
+        return metadata.get("fname")
+
+    async def upload_async(self, content: bytes | str, fname: str) -> ResourceId:
+        return self.upload(content, fname)
+
+    def upload_dir(self, resource_ids: list[ResourceId]) -> ResourceId:
+        content = "\n".join(resource_ids)
+        res = self.upload(content=content, fname=".dir")
+        return res
+
+    def download(self, resource_id: ResourceId) -> bytes:
+        return Path(resource_id).read_bytes()
+
+    async def download_async(self, resource_id: ResourceId) -> bytes:
+        return self.download(resource_id)
+
+    def download_text(self, resource_id: ResourceId) -> str:
+        return Path(resource_id).read_text(encoding="utf-8")
+
+    def read_dir_or_none(self, resource_id: ResourceId) -> list[ResourceId] | None:
+        if not self.is_dir(resource_id):
+            return None
+        res = self.download_text(resource_id).split("\n")
+        return res
+
+    def get_path(self, resource_id: ResourceId | None) -> Path | None:
+        return self._get_path(resource_id)
+
+    def _get_path(self, resource_id: ResourceId | None) -> Path | None:
+        if not resource_id:
+            return None
+        path = Path(resource_id)
+        return path if (path.exists() and path.is_file()) else None
+
+    def is_valid(self, resource_id: ResourceId | None) -> bool:
+        path = self._get_path(resource_id)
+        return path is not None
+
+    def is_file(self, resource_id: ResourceId | None) -> bool:
+        path = self._get_path(resource_id)
+        return bool(path and path.suffix != SUFFIX_DIR)
+
+    def is_dir(self, resource_id: ResourceId | None) -> bool:
+        path = self._get_path(resource_id)
+        return bool(path and path.suffix == SUFFIX_DIR)
+
+    def get_dtype(self, resource_id: ResourceId | None) -> str | None:
+        return resource_id and resource_id.rsplit(".")[-1].lower()
+
+    def unzip_file(self, resource_id: str) -> ResourceId:
+        """takes resource_id which refer to zip-archive, unpacks it and returns directory ResourceId with content of zip-archive"""
+        path = self._get_path(resource_id)
+        if not path:
+            raise ValueError(f"Not found path: {resource_id}")
+        if not is_zipfile(resource_id):
+            raise ValueError(f"Expected zip archive but found: {resource_id}")
+
+        resource_ids = []
+
+        with ZipFile(path, mode="r") as zip_file:
+            for file_info in zip_file.filelist:
+                file_dtype = file_info.filename.rsplit(".")[-1]
+                file_bytes = zip_file.read(file_info)
+                rid = self.upload(file_bytes, file_dtype)
+                resource_ids.append(rid)
+
+        res = self.upload_dir(resource_ids)
+        return res

mmar_mapi-1.2.6/src/mmar_mapi/models/base.py

@@ -0,0 +1,15 @@
+import json
+from typing import Any
+
+from pydantic import ConfigDict, BaseModel, model_validator
+
+
+class Base(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, str_strip_whitespace=True)
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_to_json(cls, value: str | Any) -> Any:
+        if isinstance(value, str):
+            return cls(**json.loads(value))
+        return value