clichefactory 0.1.0__py3-none-any.whl

clichefactory/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

clichefactory/__init__.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from clichefactory.__about__ import __version__
+from clichefactory.client import Client, factory
+from clichefactory.cliche import Cliche
+from clichefactory.types import Endpoint, PartialExtraction, ParsingOptions, PostprocessFn
+from clichefactory.errors import (
+    AuthenticationError,
+    ClicheFactoryError,
+    ConfigurationError,
+    ExtractionError,
+    ParsingError,
+    ServiceUnavailableError,
+    TrainingError,
+    UnsupportedModeError,
+    UnsupportedParserError,
+    UploadError,
+    ValidationError,
+)
+
+__all__ = [
+    "__version__",
+    "AuthenticationError",
+    "Cliche",
+    "ClicheFactoryError",
+    "Client",
+    "ConfigurationError",
+    "Endpoint",
+    "ExtractionError",
+    "ParsingError",
+    "ParsingOptions",
+    "PartialExtraction",
+    "PostprocessFn",
+    "ServiceUnavailableError",
+    "TrainingError",
+    "UnsupportedModeError",
+    "UnsupportedParserError",
+    "UploadError",
+    "ValidationError",
+    "factory",
+]
+

clichefactory/_config.py

@@ -0,0 +1,164 @@
+"""
+CLI configuration file management.
+
+Reads/writes ~/.clichefactory/config.toml.  Config precedence (highest first):
+    1. CLI flags
+    2. Environment variables
+    3. Config file (~/.clichefactory/config.toml)
+    4. Defaults
+"""
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+_CONFIG_DIR = Path.home() / ".clichefactory"
+_CONFIG_FILE = _CONFIG_DIR / "config.toml"
+
+
+@dataclass
+class ServiceConfig:
+    api_key: str = ""
+    base_url: str = ""
+
+@dataclass
+class LocalConfig:
+    model: str = ""
+    api_key: str = ""
+    ocr_model: str = ""
+    ocr_api_key: str = ""
+
+@dataclass
+class CLIConfig:
+    default_mode: str = "service"
+    service: ServiceConfig = field(default_factory=ServiceConfig)
+    local: LocalConfig = field(default_factory=LocalConfig)
+
+
+def config_dir() -> Path:
+    return _CONFIG_DIR
+
+
+def config_file_path() -> Path:
+    return _CONFIG_FILE
+
+
+def load_config() -> CLIConfig:
+    """Load config from ~/.clichefactory/config.toml. Returns defaults if file doesn't exist."""
+    cfg = CLIConfig()
+    if not _CONFIG_FILE.is_file():
+        return cfg
+
+    with open(_CONFIG_FILE, "rb") as f:
+        data = tomllib.load(f)
+
+    cfg.default_mode = data.get("default_mode", cfg.default_mode)
+
+    if "service" in data:
+        s = data["service"]
+        cfg.service.api_key = s.get("api_key", "")
+        cfg.service.base_url = s.get("base_url", "")
+
+    if "local" in data:
+        lo = data["local"]
+        cfg.local.model = lo.get("model", "")
+        cfg.local.api_key = lo.get("api_key", "")
+        cfg.local.ocr_model = lo.get("ocr_model", "")
+        cfg.local.ocr_api_key = lo.get("ocr_api_key", "")
+
+    return cfg
+
+
+def save_config(cfg: CLIConfig) -> Path:
+    """Write config to ~/.clichefactory/config.toml. Returns the path written."""
+    _CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+    lines: list[str] = []
+    lines.append(f'default_mode = "{cfg.default_mode}"')
+    lines.append("")
+
+    lines.append("[service]")
+    lines.append(f'api_key = "{cfg.service.api_key}"')
+    if cfg.service.base_url:
+        lines.append(f'base_url = "{cfg.service.base_url}"')
+    lines.append("")
+
+    lines.append("[local]")
+    lines.append(f'model = "{cfg.local.model}"')
+    lines.append(f'api_key = "{cfg.local.api_key}"')
+    if cfg.local.ocr_model:
+        lines.append(f"")
+        lines.append(f"# Optional: separate model for OCR/VLM tasks (image-to-text).")
+        lines.append(f"# If not set, the main model is used for everything.")
+        lines.append(f"# Only needed when you want a cheaper/faster model for OCR")
+        lines.append(f"# while keeping a more capable model for extraction.")
+        lines.append(f'ocr_model = "{cfg.local.ocr_model}"')
+    if cfg.local.ocr_api_key and cfg.local.ocr_api_key != cfg.local.api_key:
+        lines.append(f'ocr_api_key = "{cfg.local.ocr_api_key}"')
+    lines.append("")
+
+    _CONFIG_FILE.write_text("\n".join(lines), encoding="utf-8")
+    return _CONFIG_FILE
+
+
+def resolve_api_key(*, cli_flag: str | None, cfg: CLIConfig) -> str:
+    """Resolve ClicheFactory service API key: CLI flag > env > config file."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_API_KEY", "")
+    if env:
+        return env
+    return cfg.service.api_key
+
+
+def resolve_base_url(*, cli_flag: str | None, cfg: CLIConfig) -> str | None:
+    """Resolve service base URL: CLI flag > env > config file > None."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_API_URL", "")
+    if env:
+        return env
+    return cfg.service.base_url or None
+
+
+def resolve_model(*, cli_flag: str | None, cfg: CLIConfig) -> str:
+    """Resolve LLM model name: CLI flag > env > config file."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_LLM_MODEL_NAME") or os.environ.get("LLM_MODEL_NAME", "")
+    if env:
+        return env
+    return cfg.local.model
+
+
+def resolve_model_api_key(*, cli_flag: str | None, cfg: CLIConfig) -> str:
+    """Resolve LLM API key: CLI flag > env > config file."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_LLM_API_KEY") or os.environ.get("LLM_API_KEY", "")
+    if env:
+        return env
+    return cfg.local.api_key
+
+
+def resolve_ocr_model(*, cli_flag: str | None, cfg: CLIConfig) -> str:
+    """Resolve OCR model name: CLI flag > env > config file > main model."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_OCR_MODEL_NAME") or os.environ.get("OCR_MODEL_NAME", "")
+    if env:
+        return env
+    return cfg.local.ocr_model
+
+
+def resolve_ocr_api_key(*, cli_flag: str | None, cfg: CLIConfig, model_api_key: str) -> str:
+    """Resolve OCR API key: CLI flag > env > config file > main model key."""
+    if cli_flag:
+        return cli_flag
+    env = os.environ.get("CLICHEFACTORY_OCR_API_KEY") or os.environ.get("OCR_API_KEY", "")
+    if env:
+        return env
+    return cfg.local.ocr_api_key or model_api_key

clichefactory/_engine/__init__.py

@@ -0,0 +1,6 @@
+"""
+Private SDK engine internals.
+
+This package contains local parsing and extraction implementation details and
+is not part of the public stable API surface.
+"""

clichefactory/_engine/adapters/csv_adapter.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+from typing import Optional, Sequence, List, Tuple
+
+from clichefactory._engine.models.normalized_doc import NormalizedDoc
+from clichefactory._engine.models.document_model import Page, Section, Heading, Table, Block
+
+
+class CsvNormalizedDoc(NormalizedDoc):
+    def __init__(self, filename: str, blocks: List[Block], markdown: str) -> None:
+        self.filename = filename
+        self.summary_text = ""
+        self.markdown = markdown
+
+        # CSV is inherently tabular and has no notion of pages. Use a single synthetic page.
+        self.pages = (Page(index=1, size=None, blocks=tuple(blocks)),)
+        self.sections = self._build_sections_from_headings(self.pages[0].blocks)
+
+        self.images = tuple()
+        self.tables = tuple(b for b in self.pages[0].blocks if isinstance(b, Table))
+
+    def get_plain_text(self) -> str:
+        return self.get_markdown()
+
+    def get_markdown(self) -> str:
+        return self.markdown
+
+    def get_json(self, table_index: int = 0, header: bool = True) -> Optional[list[dict[str, str]]]:
+        """
+        Convert the first table in this document to a list of row dicts.
+
+        - If header=True: uses row 0 as header keys.
+            - Missing/empty header cell => "col_{index}"
+            - Duplicate header names are disambiguated with _1, _2, ... in left-to-right order.
+        - If header=False: uses synthetic keys "col_{index}" and includes row 0 as data.
+        """
+        if not self.tables:
+            return None
+
+        t = self.tables[table_index]
+        if not getattr(t, "cells", None):
+            return None
+
+        max_row = max(c.row for c in t.cells)
+        max_col = max(c.col for c in t.cells)
+
+        grid: list[list[str]] = [[""] * (max_col + 1) for _ in range(max_row + 1)]
+        for c in t.cells:
+            if 0 <= c.row <= max_row and 0 <= c.col <= max_col:
+                grid[c.row][c.col] = (c.text or "").strip()
+
+        def make_unique_header(raw_keys: list[str]) -> list[str]:
+            base_keys: list[str] = []
+            counts: dict[str, int] = {}
+            for j, k in enumerate(raw_keys):
+                base = (k or "").strip()
+                if not base:
+                    base = f"col_{j}"
+                base_keys.append(base)
+                counts[base] = counts.get(base, 0) + 1
+
+            seen: dict[str, int] = {}
+            unique: list[str] = []
+            for base in base_keys:
+                if counts.get(base, 0) > 1:
+                    seen[base] = seen.get(base, 0) + 1
+                    unique.append(f"{base}_{seen[base]}")
+                else:
+                    unique.append(base)
+            return unique
+
+        if header and grid:
+            raw_header = grid[0]
+            keys = make_unique_header(raw_header)
+            start_row = 1
+        else:
+            keys = [f"col_{j}" for j in range(max_col + 1)]
+            start_row = 0
+
+        out: list[dict[str, str]] = []
+        for r in range(start_row, max_row + 1):
+            row_vals = grid[r]
+            if all(not (v or "").strip() for v in row_vals):
+                continue
+            out.append({keys[j]: (row_vals[j] or "") for j in range(len(keys))})
+
+        return out
+
+    def _build_sections_from_headings(self, blocks: Sequence[Block]) -> Sequence[Section]:
+        class _Builder:
+            def __init__(self, heading: Heading) -> None:
+                self.heading = heading
+                self.blocks: list[Block] = []
+                self.children: list[Section] = []
+
+            def finalize(self) -> Section:
+                return Section(
+                    heading=self.heading,
+                    blocks=tuple(self.blocks),
+                    subsections=tuple(self.children),
+                )
+
+        stack: list[Tuple[int, _Builder]] = []
+        roots: list[Section] = []
+
+        def push_section(h: Heading) -> None:
+            nonlocal stack, roots
+            b = _Builder(h)
+            lvl = max(1, int(h.level))
+            while stack and stack[-1][0] >= lvl:
+                _, closed = stack.pop()
+                sec = closed.finalize()
+                if stack:
+                    stack[-1][1].children.append(sec)
+                else:
+                    roots.append(sec)
+            stack.append((lvl, b))
+
+        def add_to_current(block: Block) -> None:
+            if not stack:
+                push_section(Heading(level=1, text="Document"))
+            stack[-1][1].blocks.append(block)
+
+        for blk in blocks:
+            if isinstance(blk, Heading):
+                push_section(blk)
+            else:
+                add_to_current(blk)
+
+        while stack:
+            _, b = stack.pop()
+            sec = b.finalize()
+            if stack:
+                stack[-1][1].children.append(sec)
+            else:
+                roots.append(sec)
+
+        return tuple(roots)
+

clichefactory/_engine/adapters/docling_adapter.py

@@ -0,0 +1,280 @@
+from collections import defaultdict
+from typing import DefaultDict, Dict, Literal, Optional, Sequence, Tuple
+
+from clichefactory._engine.models.normalized_doc import NormalizedDoc
+from clichefactory._engine.models.document_model import BBox, Block, Heading, Image, Page
+from clichefactory._engine.models.document_model import Paragraph, Section, Table, TableCell
+from docling_core.types.doc.base import BoundingBox
+from docling_core.types.doc.document import (
+    DocItem,
+    DoclingDocument,
+    PictureItem,
+    SectionHeaderItem,
+    TableItem,
+    NodeItem,
+    TextItem,
+    TitleItem,
+    ListItem,
+    CodeItem,
+    FormulaItem,
+    KeyValueItem
+)
+
+
+class DoclingNormalizedDoc(NormalizedDoc):
+    """
+    NormalizedDoc from Docling document.
+
+    output_mode: "markdown" uses docling_document.export_to_markdown();
+    "structured" builds markdown from pages/sections/images/tables.
+    """
+
+    def __init__(
+        self,
+        docling_document: DoclingDocument,
+        *,
+        output_mode: Literal["markdown", "structured"] = "markdown",
+    ) -> None:
+        self.docling_document = docling_document
+        self._output_mode = output_mode
+        self.pages: Sequence[Page] = self.build_pages()
+        self.sections: Sequence[Section] = self.build_sections()
+
+        self.images = tuple(
+            block
+            for page in self.pages
+            for block in page.blocks
+            if isinstance(block, Image)
+        )
+
+        self.tables = tuple(
+            block
+            for page in self.pages
+            for block in page.blocks
+            if isinstance(block, Table)
+        )
+
+    # ----- Implementation of the NormalizedDoc interface -----
+
+    def get_plain_text(self) -> str:
+        return self.docling_document.export_to_text()
+
+    def get_markdown(self) -> str:
+        if self._output_mode == "markdown":
+            return self.docling_document.export_to_markdown()
+        from clichefactory._engine.parsers.parser_utils.pdf.docling_helpers import blocks_to_markdown
+
+        return blocks_to_markdown(self.pages, self.sections, self.images, self.tables)
+    
+    # ----- Docling to document_model mapping functions -----
+
+    def build_pages(self) -> Sequence[Page]:
+        blocks_by_page = self._collect_blocks_by_page()
+
+        # If docling provides pages, respect them
+        if self.docling_document.pages:
+            pages: list[Page] = []
+            for index, page_item in self.docling_document.pages.items():
+                size_tuple = (page_item.size.width, page_item.size.height) if page_item.size else None
+                page_blocks = tuple(blocks_by_page.get(index, []))
+                pages.append(Page(index=index, size=size_tuple, blocks=page_blocks))
+            return tuple(pages)
+
+        # Otherwise: single synthetic page (DOCX pagination may be unavailable)
+        all_blocks: list[Block] = []
+        for page_idx in sorted(blocks_by_page.keys()):
+            all_blocks.extend(blocks_by_page[page_idx])
+        if not all_blocks:
+            # last resort: collect in reading order with no page assignment
+            for item, _ in self.docling_document.iterate_items(with_groups=False):
+                b = self._to_block(item)
+                if b is not None:
+                    all_blocks.append(b)
+
+        return (Page(index=1, size=None, blocks=tuple(all_blocks)),)
+
+ 
+    def build_sections(self) -> Sequence[Section]:
+        """
+        Build a semantic hierarchy from headings.
+
+        Strategy:
+        - Flatten blocks in reading order (page order, then block order).
+        - Start a new section when we hit a Heading.
+        - Nest based on heading level (H1 contains H2, etc.)
+        """
+        flat_blocks: list[Block] = []
+        # reading order: by page index then appearance
+        for page in sorted(self.pages, key=lambda p: p.index):
+            flat_blocks.extend(page.blocks)
+
+        # Collect top-level sections using a stack of (level, SectionBuilder)
+        class _Builder:
+            def __init__(self, heading: Heading) -> None:
+                self.heading = heading
+                self.blocks: list[Block] = []
+                self.children: list[Section] = []
+
+            def finalize(self) -> Section:
+                return Section(heading=self.heading, blocks=tuple(self.blocks), subsections=tuple(self.children))
+
+        stack: list[Tuple[int, _Builder]] = []
+        roots: list[Section] = []
+
+        def push_section(h: Heading) -> None:
+            nonlocal stack, roots
+            b = _Builder(h)
+            lvl = max(1, int(h.level))
+
+            # pop until parent is strictly lower level
+            while stack and stack[-1][0] >= lvl:
+                closed_lvl, closed = stack.pop()
+                sec = closed.finalize()
+                if stack:
+                    stack[-1][1].children.append(sec)
+                else:
+                    roots.append(sec)
+
+            stack.append((lvl, b))
+
+        def add_to_current(block: Block) -> None:
+            if not stack:
+                # No heading seen yet: create an implicit H1 section so content isn’t lost
+                push_section(Heading(level=1, text="Document"))
+            stack[-1][1].blocks.append(block)
+
+        for blk in flat_blocks:
+            if isinstance(blk, Heading):
+                push_section(blk)
+            else:
+                add_to_current(blk)
+
+        # close remaining
+        while stack:
+            lvl, b = stack.pop()
+            sec = b.finalize()
+            if stack:
+                stack[-1][1].children.append(sec)
+            else:
+                roots.append(sec)
+
+        return tuple(roots)
+
+    # Collect all blocks for all pages (poor man's provenance))
+    def _collect_blocks_by_page(self) -> Dict[int, list[Block]]:
+        blocks: DefaultDict[int, list[Block]] = defaultdict(list)
+        for item, _ in self.docling_document.iterate_items(with_groups=False):
+            block = self._to_block(item)
+            
+            if block is None:
+                continue
+
+            if not isinstance(item, DocItem):  # Could have been a GroupItem or NodeItem or similar semantic, non content descriptor
+                continue
+
+            page_no = self._get_primary_page(item)
+            if page_no is None:
+                continue
+
+            blocks[page_no].append(block)
+
+        return blocks
+
+
+    def _to_block(self, item: NodeItem) -> Optional[Block]:
+        if not isinstance(item, DocItem):
+            return None
+
+        bbox = self._item_bbox(item)
+
+        if isinstance(item, TableItem):
+            return self._table_to_block(item, bbox)
+        if isinstance(item, PictureItem):
+            return self._image_to_block(item, bbox)
+        if isinstance(item, SectionHeaderItem):
+            return Heading(level=item.level, text=item.text, bbox=bbox)
+        if isinstance(item, TitleItem):
+            return Heading(level=1, text=item.text, bbox=bbox)
+        if isinstance(item, TextItem):
+           return Paragraph(text=item.text, bbox=bbox)
+        if isinstance(item, ListItem):
+           return Paragraph(text=item.text, bbox=bbox)
+        if isinstance(item, CodeItem):
+           return Paragraph(text=item.text, bbox=bbox)
+        if isinstance(item, FormulaItem):
+           return Paragraph(text=item.text, bbox=bbox)
+        if isinstance(item, KeyValueItem):
+           return Paragraph(text=self._key_value_to_md(item), bbox=bbox)
+        # if isinstance(item, FormItem):
+        # Made of KeyValueItems, TextItems, etc., skipping for now
+        
+        return None
+
+    def _table_to_block(self, table_item: TableItem, bbox: Optional[BBox]) -> Optional[Table]:
+        cells: list[TableCell] = []
+        page_height = self._get_page_height(table_item)
+
+        for cell in table_item.data.table_cells:
+            cell_bbox = self._normalize_bbox(cell.bbox, page_height)
+            cells.append(
+                TableCell(
+                    text=cell.text,
+                    row=cell.start_row_offset_idx,
+                    col=cell.start_col_offset_idx,
+                    bbox=cell_bbox,
+                )
+            )
+
+        return Table(cells=tuple(cells), bbox=bbox)
+
+    def _image_to_block(self, picture_item: PictureItem, bbox: Optional[BBox]) -> Optional[Image]:
+        img_ref = picture_item.image
+        if img_ref is None:
+            ref = picture_item.self_ref
+            mime = "application/octet-stream"
+        else:
+            ref = str(img_ref.uri)
+            mime = img_ref.mimetype
+
+        alt_text = picture_item.caption_text(self.docling_document) or None
+        return Image(ref=ref, mime_type=mime, bbox=bbox, alt_text=alt_text)
+
+    def _get_primary_page(self, item: DocItem) -> Optional[int]:
+        if not item.prov:
+            return None
+        return item.prov[0].page_no
+
+    def _item_bbox(self, item: DocItem) -> Optional[BBox]:
+        if not item.prov:
+            return None
+        return self._normalize_bbox(item.prov[0].bbox, self._get_page_height(item))
+
+    def _normalize_bbox(self, bbox: Optional[BoundingBox], page_height: Optional[float]) -> Optional[BBox]:
+        if bbox is None:
+            return None
+        if page_height:
+            return bbox.to_top_left_origin(page_height).as_tuple()
+        return bbox.as_tuple()
+
+    def _get_page_height(self, item: DocItem) -> Optional[float]:
+        page_no = self._get_primary_page(item)
+        if page_no is None:
+            return None
+        page = self.docling_document.pages.get(page_no)
+        if page is None:
+            return None
+        return page.size.height
+    
+    def _key_value_to_md(self, item: KeyValueItem) -> str:
+        # Safely obtain the dumped mapping, then get the first key and its value without indexing dict_keys
+        key = ""
+        value = ""
+        dumped_kv = item.model_dump() if hasattr(item, "model_dump") else {}
+        key = next(iter(dumped_kv.keys()), "") or ""
+        value = dumped_kv.get(key, "")
+
+        key = (key or "").strip()
+        value = (str(value) if value is not None else "").strip()
+
+        return f"{key}: {value}"
+

clichefactory/_engine/adapters/eml_adapter.py

@@ -0,0 +1,45 @@
+from typing import Sequence
+
+from clichefactory._engine.models.document_model import Image, Page, Section, Table
+from clichefactory._engine.models.normalized_doc import NormalizedDoc
+
+
+class EmlDoc(NormalizedDoc):
+    summary_text: str
+    media_type: str
+    pages: Sequence[Page]
+    sections: Sequence[Section]
+
+    # Internal fields
+    _plain_text: str
+    _markdown: str
+    _images: Sequence[Image]
+    _tables: Sequence[Table]
+
+    def __init__(
+        self,
+        summary_text: str,
+        media_type: str,
+        pages: Sequence[Page],
+        sections: Sequence[Section],
+        images: Sequence[Image],
+        tables: Sequence[Table],
+        _plain_text: str,
+        _markdown: str
+        ) -> None:
+        
+        self.summary_text = summary_text
+        self.media_type = media_type
+        self.pages = pages
+        self.sections = sections
+        self.images = images
+        self.tables = tables
+        self._plain_text = _plain_text
+        self._markdown = _markdown
+
+
+    def get_plain_text(self) -> str:
+        return self._plain_text
+
+    def get_markdown(self) -> str:
+        return self._markdown