docqwise 0.1.0__py3-none-any.whl

docqwise/core/element.py

@@ -0,0 +1,163 @@
+"""Document element types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from docqwise.core.chunk import BoundingBox, DocqwiseChunk, ChunkMetadata, ElementType
+
+
+@dataclass
+class TableCell:
+    """A single cell in a table."""
+
+    text: str
+    row: int
+    col: int
+    row_span: int = 1
+    col_span: int = 1
+    is_header: bool = False
+    bbox: Optional[BoundingBox] = None
+    confidence: float = 1.0
+
+
+@dataclass
+class Table:
+    """Extracted table from a document."""
+
+    table_id: str
+    cells: list[TableCell]
+    rows: int
+    cols: int
+    page: int = 0
+    bbox: Optional[BoundingBox] = None
+    caption: str = ""
+    confidence: float = 1.0
+    context: str = ""
+    has_merged_cells: bool = False
+    is_cross_page: bool = False
+
+    def to_dataframe(self):
+        """Convert to pandas DataFrame."""
+        import pandas as pd
+
+        grid = [["" for _ in range(self.cols)] for _ in range(self.rows)]
+        headers = []
+
+        for cell in self.cells:
+            if cell.row < self.rows and cell.col < self.cols:
+                grid[cell.row][cell.col] = cell.text
+                if cell.is_header:
+                    headers.append(cell.text)
+
+        if headers:
+            return pd.DataFrame(grid[1:], columns=grid[0])
+        return pd.DataFrame(grid)
+
+    def to_csv(self, path: str = "") -> str:
+        """Export as CSV."""
+        df = self.to_dataframe()
+        if path:
+            df.to_csv(path, index=False)
+            return path
+        return df.to_csv(index=False)
+
+    def to_json(self) -> list[dict]:
+        """Export as list of row dicts."""
+        df = self.to_dataframe()
+        return df.to_dict(orient="records")
+
+    def to_excel(self, path: str) -> None:
+        """Export as Excel file."""
+        df = self.to_dataframe()
+        df.to_excel(path, index=False)
+
+    def to_markdown(self) -> str:
+        """Export as markdown table."""
+        df = self.to_dataframe()
+        return df.to_markdown(index=False)
+
+    def to_chunk(self) -> DocqwiseChunk:
+        """Convert to retrieval chunk with table context."""
+        text = f"Table: {self.caption}\n{self.to_markdown()}" if self.caption else self.to_markdown()
+        return DocqwiseChunk(
+            chunk_id=f"table_{self.table_id}",
+            text=text,
+            metadata=ChunkMetadata(
+                page_num=self.page,
+                element_type=ElementType.TABLE,
+                bbox=self.bbox,
+                confidence=self.confidence,
+            ),
+            table_data=self.to_json(),
+        )
+
+
+@dataclass
+class ExtractedImage:
+    """Image or figure extracted from a document."""
+
+    image_id: str
+    page: int = 0
+    bbox: Optional[BoundingBox] = None
+    caption: str = ""
+    alt_text: str = ""
+    image_type: str = ""  # photo | chart | diagram | logo | signature
+    chart_type: str = ""  # bar | line | pie | scatter
+    chart_data: Any = None
+    image_bytes: Optional[bytes] = None
+    image_path: Optional[str] = None
+    resolution: tuple[int, int] = (0, 0)
+    confidence: float = 1.0
+
+    def save(self, directory: str, filename: str = "") -> str:
+        """Save extracted image to file."""
+        import os
+
+        if not filename:
+            filename = f"image_{self.image_id}.png"
+        path = os.path.join(directory, filename)
+        if self.image_bytes:
+            os.makedirs(directory, exist_ok=True)
+            with open(path, "wb") as f:
+                f.write(self.image_bytes)
+        return path
+
+
+@dataclass
+class Entity:
+    """Named entity extracted from a document."""
+
+    text: str
+    entity_type: str  # PERSON, ORG, MONEY, DATE, LOCATION, custom
+    page: int = 0
+    bbox: Optional[BoundingBox] = None
+    confidence: float = 1.0
+    normalized_value: Any = None
+
+
+@dataclass
+class Relation:
+    """Relation between two entities."""
+
+    subject: str
+    predicate: str
+    object: str
+    confidence: float = 1.0
+    page: int = 0
+    source_entities: list[Entity] = field(default_factory=list)
+
+
+@dataclass
+class FormField:
+    """Form field extracted from a document."""
+
+    field_name: str
+    field_type: str  # text | checkbox | radio | dropdown | signature
+    value: Any = None
+    is_checked: Optional[bool] = None
+    options: list[str] = field(default_factory=list)
+    bbox: Optional[BoundingBox] = None
+    page: int = 0
+    confidence: float = 1.0

docqwise/core/field.py

@@ -0,0 +1,72 @@
+"""Field extraction result models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from docqwise.core.chunk import BoundingBox
+
+
+@dataclass
+class FieldResult:
+    """Result of a single field extraction."""
+
+    value: Any
+    confidence: float
+    raw_text: str = ""
+    bbox: Optional[BoundingBox] = None
+    page: int = 0
+    extraction_method: str = ""
+    is_corrected: bool = False
+
+
+@dataclass
+class ExtractionResult:
+    """Result of extracting fields from a document."""
+
+    doc_id: str
+    source_path: str
+    fields: dict[str, FieldResult]
+    confidence: float
+    strategy_used: str = ""
+    processing_time_ms: float = 0.0
+    warnings: list[str] = field(default_factory=list)
+
+    _engine: Any = field(default=None, repr=False)
+
+    def correct(self, corrections: dict[str, Any], scope: str = "global") -> None:
+        """Apply user corrections and store for future use."""
+        for field_name, correct_value in corrections.items():
+            if field_name in self.fields:
+                self.fields[field_name] = FieldResult(
+                    value=correct_value,
+                    confidence=1.0,
+                    raw_text=self.fields[field_name].raw_text,
+                    bbox=self.fields[field_name].bbox,
+                    page=self.fields[field_name].page,
+                    extraction_method="user_correction",
+                    is_corrected=True,
+                )
+            else:
+                self.fields[field_name] = FieldResult(
+                    value=correct_value,
+                    confidence=1.0,
+                    extraction_method="user_correction",
+                    is_corrected=True,
+                )
+
+        if self._engine is not None:
+            self._engine._correction_store.save(
+                doc_id=self.doc_id,
+                source_path=self.source_path,
+                corrections=corrections,
+                scope=scope,
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {k: v.value for k, v in self.fields.items()}
+
+    def to_json(self) -> str:
+        import json
+        return json.dumps(self.to_dict(), default=str, indent=2)

docqwise/engine.py

@@ -0,0 +1,377 @@
+"""Main Docqwise engine class."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional, Union
+
+from docqwise.config import DocqwiseConfig
+from docqwise.core.document import DocqwiseDocument
+from docqwise.core.chunk import DocqwiseChunk
+from docqwise.core.field import ExtractionResult
+from docqwise.core.element import Table, ExtractedImage, Entity, Relation
+
+logger = logging.getLogger("docqwise")
+
+
+class Docqwise:
+    """
+    DocQWise: Read. Extract. Retrieve.
+
+    Document intelligence engine that reads any format, extracts exact
+    structured data, and retrieves with semantic search.
+
+    Usage:
+        from docqwise import Docqwise
+
+        dq = Docqwise()
+        dq.ingest("documents/")
+        results = dq.retrieve("payment terms", top_k=5)
+        answer = dq.ask("What is the total amount?")
+    """
+
+    def __init__(
+        self,
+        # Strategy
+        strategy: str = "auto",
+        confidence_threshold: float = 0.85,
+        max_api_cost_per_doc: float = 0.05,
+        # Speed
+        workers: Union[int, str] = 1,
+        gpu_workers: int = 0,
+        threads: int = 4,
+        batch_size: int = 32,
+        prefetch: bool = False,
+        async_mode: bool = False,
+        # Models
+        ocr: Any = "easyocr",
+        layout_model: Any = "yolov8",
+        embedder: Any = "all-MiniLM-L6-v2",
+        llm: Any = None,
+        reranker: Any = None,
+        # Storage
+        vector_store: Any = "sqlite",
+        database: Any = None,
+        graph_store: Any = None,
+        store_path: str = "./docqwise_db",
+        # Search
+        search_mode: str = "vector",
+        # Incremental
+        incremental: bool = True,
+        cache: bool = True,
+        cache_dir: str = ".docqwise_cache",
+        # Learning
+        learning_mode: str = "exact",
+        # Config override
+        config: Optional[Union[str, DocqwiseConfig]] = None,
+    ):
+        if isinstance(config, str):
+            self._config = DocqwiseConfig.from_yaml(config)
+        elif isinstance(config, DocqwiseConfig):
+            self._config = config
+        else:
+            self._config = DocqwiseConfig()
+            self._config.strategy.mode = strategy
+            self._config.strategy.confidence_threshold = confidence_threshold
+            self._config.speed.workers = workers
+            self._config.speed.gpu_workers = gpu_workers
+            self._config.speed.threads = threads
+            self._config.speed.batch_size = batch_size
+            self._config.speed.prefetch = prefetch
+            self._config.speed.async_mode = async_mode
+            self._config.storage.vector_store = vector_store if isinstance(vector_store, str) else "custom"
+            self._config.storage.store_path = store_path
+            self._config.storage.cache = cache
+            self._config.storage.cache_dir = cache_dir
+            self._config.search.mode = search_mode
+            self._config.incremental.enabled = incremental
+
+        # Store raw component references for lazy init
+        self._ocr_ref = ocr
+        self._layout_ref = layout_model
+        self._embedder_ref = embedder
+        self._llm_ref = llm
+        self._reranker_ref = reranker
+        self._vector_store_ref = vector_store
+        self._database_ref = database
+        self._graph_store_ref = graph_store
+        self._learning_mode = learning_mode
+
+        # Lazy-initialized components
+        self._reader = None
+        self._ocr_engine = None
+        self._layout_model = None
+        self._embedder = None
+        self._llm = None
+        self._reranker = None
+        self._vector_store = None
+        self._database = None
+        self._graph_store = None
+        self._correction_store = None
+        self._strategy_router = None
+        self._change_detector = None
+
+        self._initialized = False
+        logger.info("DocQWise engine created (lazy init)")
+
+    def _ensure_initialized(self) -> None:
+        """Lazy initialization of components on first use."""
+        if self._initialized:
+            return
+        self._init_reader()
+        self._init_store()
+        self._init_correction_store()
+        self._initialized = True
+        logger.info("DocQWise engine initialized")
+
+    def _init_reader(self) -> None:
+        from docqwise.readers.auto_reader import AutoReader
+        self._reader = AutoReader()
+
+    def _init_store(self) -> None:
+        if isinstance(self._vector_store_ref, str) and self._vector_store_ref == "sqlite":
+            from docqwise.stores.sqlite_store import SQLiteVectorStore
+            self._vector_store = SQLiteVectorStore(self._config.storage.store_path)
+        elif hasattr(self._vector_store_ref, "insert"):
+            self._vector_store = self._vector_store_ref
+
+    def _init_embedder(self) -> None:
+        if self._embedder is not None:
+            return
+        if isinstance(self._embedder_ref, str):
+            from docqwise.embedders.sentence_transformer import SentenceTransformerEmbedder
+            self._embedder = SentenceTransformerEmbedder(self._embedder_ref)
+        elif hasattr(self._embedder_ref, "embed"):
+            self._embedder = self._embedder_ref
+
+    def _init_correction_store(self) -> None:
+        from docqwise.learning.correction import CorrectionStore
+        self._correction_store = CorrectionStore(self._config.storage.store_path)
+
+    # ═══════════════════════════════════════
+    # INGESTION
+    # ═══════════════════════════════════════
+
+    def ingest(self, source: str, **kwargs) -> dict:
+        """Ingest documents from file, folder, URL, or database connection string."""
+        self._ensure_initialized()
+        # TODO: implement full ingestion pipeline
+        logger.info(f"Ingesting from: {source}")
+        raise NotImplementedError("Ingestion pipeline coming in v0.1.0")
+
+    async def ingest_async(self, source: str, **kwargs) -> dict:
+        """Async ingestion with parallel processing."""
+        self._ensure_initialized()
+        raise NotImplementedError("Async ingestion coming in v0.1.0")
+
+    def sync(self, source: str = None, mode: str = "incremental",
+             interval: int = 300) -> None:
+        """Sync with database — incremental by default."""
+        raise NotImplementedError("Database sync coming in v0.1.0")
+
+    def watch(self, path: str, poll_interval: int = 60,
+              on_new: Any = None) -> None:
+        """Watch folder for new documents."""
+        raise NotImplementedError("File watching coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # EXTRACTION
+    # ═══════════════════════════════════════
+
+    def extract_all(self, source: str, **kwargs) -> DocqwiseDocument:
+        """Extract everything from a document."""
+        self._ensure_initialized()
+        raise NotImplementedError("Full extraction coming in v0.1.0")
+
+    def extract_text(self, source: str, **kwargs) -> str:
+        """Extract text from a document."""
+        self._ensure_initialized()
+        raise NotImplementedError("Text extraction coming in v0.1.0")
+
+    def extract_fields(self, source: str, schema: dict = None,
+                       template: str = None, **kwargs) -> ExtractionResult:
+        """Extract fields — schema-driven or auto-detect."""
+        self._ensure_initialized()
+        raise NotImplementedError("Field extraction coming in v0.1.0")
+
+    def extract_fields_batch(self, source: str, **kwargs) -> list[ExtractionResult]:
+        """Batch field extraction across multiple documents."""
+        self._ensure_initialized()
+        raise NotImplementedError("Batch extraction coming in v0.1.0")
+
+    def extract_tables(self, source: str, **kwargs) -> list[Table]:
+        """Extract tables from a document."""
+        self._ensure_initialized()
+        raise NotImplementedError("Table extraction coming in v0.1.0")
+
+    def extract_images(self, source: str, **kwargs) -> list[ExtractedImage]:
+        """Extract images and figures from a document."""
+        self._ensure_initialized()
+        raise NotImplementedError("Image extraction coming in v0.1.0")
+
+    def extract_form(self, source: str) -> dict:
+        """Extract form fields."""
+        self._ensure_initialized()
+        raise NotImplementedError("Form extraction coming in v0.1.0")
+
+    def extract_entities(self, source: str, types: list[str] = None,
+                         custom_types: dict = None) -> list[Entity]:
+        """Extract named entities."""
+        self._ensure_initialized()
+        raise NotImplementedError("Entity extraction coming in v0.1.0")
+
+    def extract_relations(self, source: str) -> list[Relation]:
+        """Extract entity relations."""
+        self._ensure_initialized()
+        raise NotImplementedError("Relation extraction coming in v0.1.0")
+
+    def extract_layout(self, source: str) -> dict:
+        """Extract full layout map with bounding boxes."""
+        self._ensure_initialized()
+        raise NotImplementedError("Layout extraction coming in v0.1.0")
+
+    def auto_extract(self, source: str) -> ExtractionResult:
+        """Auto-detect document type and extract all relevant fields."""
+        self._ensure_initialized()
+        raise NotImplementedError("Auto extraction coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # RETRIEVAL + Q&A
+    # ═══════════════════════════════════════
+
+    def retrieve(self, query: str, top_k: int = 5,
+                 mode: str = None, **kwargs) -> list[dict]:
+        """Semantic search across ingested corpus."""
+        self._ensure_initialized()
+        raise NotImplementedError("Retrieval coming in v0.1.0")
+
+    def ask(self, question: str, source: str = None, **kwargs) -> str:
+        """Natural language Q&A."""
+        self._ensure_initialized()
+        raise NotImplementedError("Q&A coming in v0.1.0")
+
+    def ask_batch(self, question: str = None, questions: list = None,
+                  documents: str = None, **kwargs) -> list:
+        """Batch Q&A across documents."""
+        raise NotImplementedError("Batch Q&A coming in v0.1.0")
+
+    def sql(self, query: str) -> list[dict]:
+        """Run SQL directly against ingested structured data."""
+        raise NotImplementedError("SQL queries coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # CLASSIFICATION + COMPARISON
+    # ═══════════════════════════════════════
+
+    def classify(self, source: str, **kwargs) -> list[dict]:
+        """Classify document type."""
+        raise NotImplementedError("Classification coming in v0.1.0")
+
+    def classify_batch(self, source: str, **kwargs) -> list:
+        """Batch classification."""
+        raise NotImplementedError("Batch classification coming in v0.1.0")
+
+    def set_taxonomy(self, labels: list[str]) -> None:
+        """Set custom classification taxonomy."""
+        raise NotImplementedError("Taxonomy coming in v0.1.0")
+
+    def train_classifier(self, examples: dict) -> None:
+        """Few-shot classifier training."""
+        raise NotImplementedError("Classifier training coming in v0.1.0")
+
+    def compare(self, source_a: str, source_b: str) -> dict:
+        """Compare two documents."""
+        raise NotImplementedError("Comparison coming in v0.1.0")
+
+    def compare_batch(self, pairs: list, **kwargs) -> list:
+        """Batch comparison."""
+        raise NotImplementedError("Batch comparison coming in v0.1.0")
+
+    def find_duplicates(self, source: str, threshold: float = 0.85) -> list:
+        """Find near-duplicate documents."""
+        raise NotImplementedError("Deduplication coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # SCHEMA
+    # ═══════════════════════════════════════
+
+    def detect_schema(self, source: str) -> dict:
+        """Auto-detect schema from structured data."""
+        raise NotImplementedError("Schema detection coming in v0.1.0")
+
+    def validate(self, source: str, schema: dict) -> dict:
+        """Validate data against schema."""
+        raise NotImplementedError("Validation coming in v0.1.0")
+
+    def discover_schema(self, connection_string: str) -> dict:
+        """Discover database schema."""
+        raise NotImplementedError("Schema discovery coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # GRAPH
+    # ═══════════════════════════════════════
+
+    def build_graph(self, source: str = None) -> Any:
+        """Build document graph from ingested corpus."""
+        raise NotImplementedError("GraphRAG coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # LEARNING
+    # ═══════════════════════════════════════
+
+    def learn_from_labels(self, documents: str, labels: str) -> None:
+        """Bulk learning from labeled dataset."""
+        raise NotImplementedError("Bulk learning coming in v0.1.0")
+
+    def learning_report(self) -> dict:
+        """Report on corrections and accuracy improvements."""
+        raise NotImplementedError("Learning report coming in v0.1.0")
+
+    def export_learnings(self, path: str) -> None:
+        """Export learned corrections."""
+        raise NotImplementedError("Export learnings coming in v0.1.0")
+
+    def import_learnings(self, path: str) -> None:
+        """Import corrections from another instance."""
+        raise NotImplementedError("Import learnings coming in v0.1.0")
+
+    # ═══════════════════════════════════════
+    # UTILITIES
+    # ═══════════════════════════════════════
+
+    def to_dataframe(self, source: str, **kwargs):
+        """Convert structured source to DataFrame."""
+        raise NotImplementedError("DataFrame conversion coming in v0.1.0")
+
+    def detect_pii(self, source: str) -> list:
+        """Detect PII in document."""
+        raise NotImplementedError("PII detection coming in v0.1.0")
+
+    def redact(self, source: str, output: str, redact_types: list = None) -> None:
+        """Redact PII."""
+        raise NotImplementedError("PII redaction coming in v0.1.0")
+
+    def checkpoint(self) -> None:
+        """Save current processing state."""
+        raise NotImplementedError("Checkpointing coming in v0.1.0")
+
+    def metrics(self) -> dict:
+        """Pipeline health and performance metrics."""
+        return {
+            "version": "0.1.0",
+            "store_path": self._config.storage.store_path,
+            "strategy": self._config.strategy.mode,
+            "workers": self._config.speed.workers,
+            "initialized": self._initialized,
+        }
+
+    def export(self, path: str, format: str = "json", **kwargs) -> None:
+        """Export processed data."""
+        raise NotImplementedError("Export coming in v0.1.0")
+
+    def __repr__(self) -> str:
+        return (
+            f"Docqwise(strategy='{self._config.strategy.mode}', "
+            f"store='{self._config.storage.vector_store}', "
+            f"workers={self._config.speed.workers})"
+        )

docqwise/exceptions.py

@@ -0,0 +1,49 @@
+"""DocQWise exceptions."""
+
+
+class DocqwiseError(Exception):
+    """Base exception for docqwise."""
+
+
+class ReaderError(DocqwiseError):
+    """Error reading a document."""
+
+
+class UnsupportedFormatError(ReaderError):
+    """Document format is not supported."""
+
+
+class ExtractionError(DocqwiseError):
+    """Error extracting content from a document."""
+
+
+class OCRError(ExtractionError):
+    """Error during OCR processing."""
+
+
+class StoreError(DocqwiseError):
+    """Error with vector/database store operations."""
+
+
+class ConnectionError(DocqwiseError):
+    """Error connecting to external service."""
+
+
+class SchemaError(DocqwiseError):
+    """Error with schema detection or validation."""
+
+
+class PipelineError(DocqwiseError):
+    """Error in pipeline execution."""
+
+
+class CycleDetectedError(PipelineError):
+    """Pipeline DAG contains a cycle."""
+
+
+class ConfigError(DocqwiseError):
+    """Invalid configuration."""
+
+
+class CorrectionError(DocqwiseError):
+    """Error applying or storing corrections."""