docqwise 0.1.0__py3-none-any.whl

docqwise/learning/correction.py

@@ -0,0 +1,94 @@
+"""Simple correction store — exact overrides, no ML."""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from datetime import datetime
+from typing import Any, Optional
+
+
+class CorrectionStore:
+    """Stores user corrections as exact overrides. No RL. No weights. Deterministic."""
+
+    def __init__(self, store_path: str = "./docqwise_db"):
+        os.makedirs(store_path, exist_ok=True)
+        self._db_path = os.path.join(store_path, "corrections.db")
+        self._init_db()
+
+    def _init_db(self) -> None:
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS corrections (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    doc_id TEXT,
+                    source_path TEXT,
+                    field_name TEXT NOT NULL,
+                    wrong_value TEXT,
+                    correct_value TEXT NOT NULL,
+                    scope TEXT DEFAULT 'global',
+                    vendor TEXT,
+                    doc_type TEXT,
+                    created_at TEXT NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_corrections_field
+                ON corrections(field_name, scope, vendor)
+            """)
+
+    def save(self, doc_id: str, source_path: str,
+             corrections: dict[str, Any], scope: str = "global") -> None:
+        """Save corrections as exact overrides."""
+        with sqlite3.connect(self._db_path) as conn:
+            for field_name, correct_value in corrections.items():
+                conn.execute(
+                    """INSERT INTO corrections
+                    (doc_id, source_path, field_name, correct_value, scope, created_at)
+                    VALUES (?, ?, ?, ?, ?, ?)""",
+                    (doc_id, source_path, field_name,
+                     json.dumps(correct_value, default=str),
+                     scope, datetime.now().isoformat()),
+                )
+
+    def apply(self, doc_type: str, fields: dict[str, Any],
+              vendor: Optional[str] = None) -> dict[str, Any]:
+        """Apply stored corrections as exact overrides. No scoring. No probability."""
+        corrected = dict(fields)
+        with sqlite3.connect(self._db_path) as conn:
+            for field_name in fields:
+                rows = conn.execute(
+                    """SELECT correct_value FROM corrections
+                    WHERE field_name = ? AND (scope = 'global' OR vendor = ?)
+                    ORDER BY created_at DESC LIMIT 1""",
+                    (field_name, vendor or ""),
+                ).fetchall()
+                if rows:
+                    corrected[field_name] = json.loads(rows[0][0])
+        return corrected
+
+    def list_corrections(self, doc_type: Optional[str] = None) -> list[dict]:
+        """List all stored corrections."""
+        with sqlite3.connect(self._db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            if doc_type:
+                rows = conn.execute(
+                    "SELECT * FROM corrections WHERE doc_type = ? ORDER BY created_at DESC",
+                    (doc_type,),
+                ).fetchall()
+            else:
+                rows = conn.execute(
+                    "SELECT * FROM corrections ORDER BY created_at DESC"
+                ).fetchall()
+            return [dict(row) for row in rows]
+
+    def delete_correction(self, correction_id: int) -> None:
+        """Remove a correction."""
+        with sqlite3.connect(self._db_path) as conn:
+            conn.execute("DELETE FROM corrections WHERE id = ?", (correction_id,))
+
+    def count(self) -> int:
+        """Count total corrections."""
+        with sqlite3.connect(self._db_path) as conn:
+            return conn.execute("SELECT COUNT(*) FROM corrections").fetchone()[0]

docqwise/readers/__init__.py

@@ -0,0 +1,3 @@
+"""Document readers — one reader per format."""
+from docqwise.readers.base import BaseReader
+from docqwise.readers.auto_reader import AutoReader

docqwise/readers/auto_reader.py

@@ -0,0 +1,127 @@
+"""Auto-detect document format and dispatch to appropriate reader."""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from docqwise.core.document import DocqwiseDocument, DocumentType, SourceType
+from docqwise.readers.base import BaseReader
+
+
+FORMAT_MAP = {
+    ".pdf": DocumentType.PDF,
+    ".doc": DocumentType.DOCX,
+    ".docx": DocumentType.DOCX,
+    ".rtf": DocumentType.RTF,
+    ".txt": DocumentType.TEXT,
+    ".md": DocumentType.MARKDOWN,
+    ".rst": DocumentType.TEXT,
+    ".log": DocumentType.TEXT,
+    ".html": DocumentType.HTML,
+    ".htm": DocumentType.HTML,
+    ".xhtml": DocumentType.HTML,
+    ".jpg": DocumentType.IMAGE,
+    ".jpeg": DocumentType.IMAGE,
+    ".png": DocumentType.IMAGE,
+    ".tiff": DocumentType.IMAGE,
+    ".tif": DocumentType.IMAGE,
+    ".bmp": DocumentType.IMAGE,
+    ".webp": DocumentType.IMAGE,
+    ".heic": DocumentType.IMAGE,
+    ".xlsx": DocumentType.EXCEL,
+    ".xls": DocumentType.EXCEL,
+    ".xlsm": DocumentType.EXCEL,
+    ".ods": DocumentType.EXCEL,
+    ".csv": DocumentType.CSV,
+    ".tsv": DocumentType.CSV,
+    ".json": DocumentType.JSON,
+    ".jsonl": DocumentType.JSON,
+    ".xml": DocumentType.XML,
+    ".yaml": DocumentType.YAML,
+    ".yml": DocumentType.YAML,
+    ".toml": DocumentType.YAML,
+    ".parquet": DocumentType.PARQUET,
+    ".pptx": DocumentType.PPTX,
+    ".ppt": DocumentType.PPTX,
+    ".eml": DocumentType.EMAIL,
+    ".msg": DocumentType.EMAIL,
+}
+
+
+class AutoReader(BaseReader):
+    """Auto-detects document format and dispatches to the correct reader."""
+
+    def __init__(self):
+        self._readers: dict[DocumentType, BaseReader] = {}
+
+    def can_read(self, path: str) -> bool:
+        return True
+
+    def supported_formats(self) -> list[str]:
+        return list(FORMAT_MAP.keys())
+
+    def detect_type(self, path: str) -> DocumentType:
+        """Detect document type from file extension and magic bytes."""
+        if path.startswith(("postgresql://", "mysql://", "sqlite:///",
+                           "mongodb://", "mongodb+srv://")):
+            return DocumentType.DATABASE
+
+        ext = os.path.splitext(path)[1].lower()
+        return FORMAT_MAP.get(ext, DocumentType.UNKNOWN)
+
+    def detect_source_type(self, path: str) -> SourceType:
+        """Detect source type from path."""
+        if path.startswith(("http://", "https://")):
+            return SourceType.URL
+        if path.startswith(("postgresql://", "mysql://", "sqlite:///", "mongodb://")):
+            return SourceType.DATABASE
+        if path.startswith(("s3://", "gs://", "az://")):
+            return SourceType.CLOUD_STORAGE
+        if path.startswith(("imap://", "pop3://")):
+            return SourceType.EMAIL
+        return SourceType.FILE
+
+    def _get_reader(self, doc_type: DocumentType) -> BaseReader:
+        """Get or lazy-load the reader for a document type."""
+        if doc_type in self._readers:
+            return self._readers[doc_type]
+
+        reader: Optional[BaseReader] = None
+
+        if doc_type == DocumentType.PDF:
+            from docqwise.readers.pdf_reader import PDFReader
+            reader = PDFReader()
+        elif doc_type == DocumentType.DOCX:
+            from docqwise.readers.docx_reader import DOCXReader
+            reader = DOCXReader()
+        elif doc_type == DocumentType.TEXT or doc_type == DocumentType.MARKDOWN:
+            from docqwise.readers.text_reader import TextReader
+            reader = TextReader()
+        elif doc_type == DocumentType.IMAGE:
+            from docqwise.readers.image_reader import ImageReader
+            reader = ImageReader()
+        elif doc_type in (DocumentType.CSV, DocumentType.EXCEL):
+            from docqwise.readers.csv_reader import CSVReader
+            reader = CSVReader()
+        elif doc_type in (DocumentType.JSON, DocumentType.XML, DocumentType.YAML):
+            from docqwise.readers.json_reader import JSONReader
+            reader = JSONReader()
+
+        if reader:
+            self._readers[doc_type] = reader
+
+        return reader
+
+    def read(self, path: str, **kwargs) -> DocqwiseDocument:
+        """Read a document by auto-detecting its format."""
+        doc_type = self.detect_type(path)
+        reader = self._get_reader(doc_type)
+
+        if reader is None:
+            from docqwise.exceptions import UnsupportedFormatError
+            raise UnsupportedFormatError(
+                f"No reader available for {doc_type.value} format: {path}"
+            )
+
+        return reader.read(path, **kwargs)

docqwise/readers/base.py

@@ -0,0 +1,21 @@
+"""Base reader abstract class."""
+
+from abc import ABC, abstractmethod
+
+from docqwise.core.document import DocqwiseDocument
+
+
+class BaseReader(ABC):
+    """Abstract base class for document readers. Implement to add new format support."""
+
+    @abstractmethod
+    def can_read(self, path: str) -> bool:
+        """Return True if this reader can handle the given path or format."""
+
+    @abstractmethod
+    def read(self, path: str, **kwargs) -> DocqwiseDocument:
+        """Read a document and return a DocqwiseDocument."""
+
+    @abstractmethod
+    def supported_formats(self) -> list[str]:
+        """Return list of supported file extensions (e.g. ['.pdf', '.PDF'])."""

docqwise/readers/csv_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/docx_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/email_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/excel_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/html_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/image_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/json_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/parquet_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/pdf_reader.py

@@ -0,0 +1,79 @@
+"""PDF reader using PyMuPDF."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+
+import fitz  # PyMuPDF
+
+from docqwise.core.document import (
+    DocqwiseDocument, DocumentMetadata, DocumentType,
+    PageContent, SourceType,
+)
+from docqwise.readers.base import BaseReader
+
+
+class PDFReader(BaseReader):
+    """Read PDF documents using PyMuPDF (fitz)."""
+
+    def can_read(self, path: str) -> bool:
+        return path.lower().endswith((".pdf",))
+
+    def supported_formats(self) -> list[str]:
+        return [".pdf"]
+
+    def read(self, path: str, **kwargs) -> DocqwiseDocument:
+        """Read a PDF and return DocqwiseDocument with text and metadata."""
+        doc = fitz.open(path)
+        file_size = os.path.getsize(path)
+
+        with open(path, "rb") as f:
+            file_hash = hashlib.sha256(f.read()).hexdigest()[:16]
+
+        pages = []
+        full_text_parts = []
+        total_words = 0
+        has_text = False
+
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text = page.get_text("text")
+            if text.strip():
+                has_text = True
+
+            words = text.split()
+            total_words += len(words)
+            full_text_parts.append(text)
+
+            pages.append(PageContent(
+                page_num=page_num + 1,
+                text=text,
+                width=page.rect.width,
+                height=page.rect.height,
+                is_scanned=not bool(text.strip()),
+            ))
+
+        metadata = DocumentMetadata(
+            title=doc.metadata.get("title", ""),
+            author=doc.metadata.get("author", ""),
+            page_count=len(doc),
+            word_count=total_words,
+            file_size_bytes=file_size,
+            file_hash=file_hash,
+            is_scanned=not has_text,
+            has_selectable_text=has_text,
+        )
+
+        doc_id = DocqwiseDocument.generate_id(path, file_hash)
+        doc.close()
+
+        return DocqwiseDocument(
+            doc_id=doc_id,
+            source_path=path,
+            source_type=SourceType.FILE,
+            doc_type=DocumentType.PDF,
+            metadata=metadata,
+            text="\n".join(full_text_parts),
+            pages=pages,
+        )

docqwise/readers/pptx_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument

docqwise/readers/text_reader.py

@@ -0,0 +1,3 @@
+"""Stub reader — implementation coming in v0.1.0."""
+from docqwise.readers.base import BaseReader
+from docqwise.core.document import DocqwiseDocument