docqwise 0.1.0__py3-none-any.whl

docqwise/__init__.py

@@ -0,0 +1,9 @@
+"""
+DocQWise: Read. Extract. Retrieve.
+Document intelligence that adapts, accelerates, and scales.
+"""
+
+from docqwise._version import __version__
+from docqwise.engine import Docqwise
+
+__all__ = ["Docqwise", "__version__"]

docqwise/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

docqwise/cli/main.py

@@ -0,0 +1,79 @@
+"""DocQWise CLI."""
+import click
+from docqwise._version import __version__
+
+@click.group()
+@click.version_option(__version__, prog_name="docqwise")
+def app():
+    """DocQWise: Read. Extract. Retrieve."""
+
+@app.command()
+def info():
+    """Show system info."""
+    click.echo(f"DocQWise v{__version__}")
+    click.echo("Read. Extract. Retrieve.")
+    click.echo("https://github.com/VK-Ant/docqwise")
+
+@app.command()
+@click.argument("source")
+@click.option("--workers", default=1, help="Number of CPU workers")
+@click.option("--progress/--no-progress", default=True)
+def ingest(source, workers, progress):
+    """Ingest documents from a source."""
+    from docqwise import Docqwise
+    dq = Docqwise(workers=workers)
+    click.echo(f"Ingesting from: {source}")
+    dq.ingest(source)
+
+@app.command()
+@click.argument("source")
+@click.option("--template", default=None, help="Extraction template name")
+@click.option("--tables", is_flag=True, help="Extract tables only")
+def extract(source, template, tables):
+    """Extract fields or tables from a document."""
+    from docqwise import Docqwise
+    dq = Docqwise()
+    if tables:
+        result = dq.extract_tables(source)
+    else:
+        result = dq.extract_fields(source, template=template)
+    click.echo(result)
+
+@app.command()
+@click.argument("query")
+@click.option("--top-k", default=5)
+def query(query, top_k):
+    """Semantic search across ingested documents."""
+    from docqwise import Docqwise
+    dq = Docqwise()
+    results = dq.retrieve(query, top_k=top_k)
+    click.echo(results)
+
+@app.command()
+@click.argument("question")
+@click.option("--source", default=None)
+def ask(question, source):
+    """Ask a question about your documents."""
+    from docqwise import Docqwise
+    dq = Docqwise()
+    answer = dq.ask(question, source=source)
+    click.echo(answer)
+
+@app.command()
+@click.option("--host", default="0.0.0.0")
+@click.option("--port", default=8000)
+@click.option("--workers", default=4)
+def serve(host, port, workers):
+    """Start REST API server."""
+    click.echo(f"Starting DocQWise server on {host}:{port}")
+    click.echo("REST API server coming in v0.1.0")
+
+@app.command()
+@click.option("--port", default=8080)
+def mcp(port):
+    """Start MCP server."""
+    click.echo(f"Starting DocQWise MCP server on port {port}")
+    click.echo("MCP server coming in v0.1.0")
+
+if __name__ == "__main__":
+    app()

docqwise/config.py

@@ -0,0 +1,130 @@
+"""Configuration system for docqwise."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+
+@dataclass
+class StrategyConfig:
+    mode: str = "auto"
+    confidence_threshold: float = 0.85
+    max_api_cost_per_doc: float = 0.05
+
+
+@dataclass
+class SpeedConfig:
+    workers: Union[int, str] = 1
+    gpu_workers: int = 0
+    threads: int = 4
+    batch_size: int = 32
+    prefetch: bool = False
+    async_mode: bool = False
+    max_memory_gb: float = 0
+
+
+@dataclass
+class StorageConfig:
+    vector_store: str = "sqlite"
+    database: Optional[str] = None
+    graph_store: Optional[str] = None
+    store_path: str = "./docqwise_db"
+    cache: bool = True
+    cache_dir: str = ".docqwise_cache"
+
+
+@dataclass
+class ModelConfig:
+    ocr: str = "easyocr"
+    layout_model: str = "yolov8"
+    embedder: str = "all-MiniLM-L6-v2"
+    llm: Optional[str] = None
+    reranker: Optional[str] = None
+
+
+@dataclass
+class SearchConfig:
+    mode: str = "vector"
+    default_top_k: int = 5
+    rerank: bool = False
+    rerank_top_k: int = 5
+
+
+@dataclass
+class IncrementalConfig:
+    enabled: bool = True
+    change_detection: str = "hash"
+    checkpoint_interval: int = 100
+
+
+@dataclass
+class ChunkerConfig:
+    strategy: str = "structure"
+    max_chunk_tokens: int = 512
+    overlap_tokens: int = 50
+    preserve_tables: bool = True
+    attach_headings: bool = True
+    attach_page_context: bool = True
+
+
+@dataclass
+class DocqwiseConfig:
+    """Master configuration."""
+
+    strategy: StrategyConfig = field(default_factory=StrategyConfig)
+    speed: SpeedConfig = field(default_factory=SpeedConfig)
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    models: ModelConfig = field(default_factory=ModelConfig)
+    search: SearchConfig = field(default_factory=SearchConfig)
+    incremental: IncrementalConfig = field(default_factory=IncrementalConfig)
+    chunker: ChunkerConfig = field(default_factory=ChunkerConfig)
+
+    @classmethod
+    def from_yaml(cls, path: str) -> DocqwiseConfig:
+        import yaml
+
+        with open(path) as f:
+            data = yaml.safe_load(f)
+        return cls._from_dict(data)
+
+    @classmethod
+    def from_json(cls, path: str) -> DocqwiseConfig:
+        import json
+
+        with open(path) as f:
+            data = json.load(f)
+        return cls._from_dict(data)
+
+    @classmethod
+    def _from_dict(cls, data: dict) -> DocqwiseConfig:
+        config = cls()
+        if "strategy" in data:
+            config.strategy = StrategyConfig(**data["strategy"])
+        if "speed" in data:
+            config.speed = SpeedConfig(**data["speed"])
+        if "storage" in data:
+            config.storage = StorageConfig(**data["storage"])
+        if "models" in data:
+            config.models = ModelConfig(**data["models"])
+        if "search" in data:
+            config.search = SearchConfig(**data["search"])
+        if "incremental" in data:
+            config.incremental = IncrementalConfig(**data["incremental"])
+        if "chunker" in data:
+            config.chunker = ChunkerConfig(**data["chunker"])
+        return config
+
+    def to_yaml(self, path: str) -> None:
+        import yaml
+        from dataclasses import asdict
+
+        with open(path, "w") as f:
+            yaml.dump(asdict(self), f, default_flow_style=False)
+
+    def to_json(self, path: str) -> None:
+        import json
+        from dataclasses import asdict
+
+        with open(path, "w") as f:
+            json.dump(asdict(self), f, indent=2)

docqwise/core/__init__.py

@@ -0,0 +1,13 @@
+"""Core data models for docqwise."""
+
+from docqwise.core.document import DocqwiseDocument, DocumentMetadata, DocumentType, SourceType
+from docqwise.core.chunk import DocqwiseChunk, ChunkMetadata, ElementType, BoundingBox
+from docqwise.core.field import FieldResult, ExtractionResult
+from docqwise.core.element import Table, TableCell, ExtractedImage, Entity, Relation, FormField
+
+__all__ = [
+    "DocqwiseDocument", "DocumentMetadata", "DocumentType", "SourceType",
+    "DocqwiseChunk", "ChunkMetadata", "ElementType", "BoundingBox",
+    "FieldResult", "ExtractionResult",
+    "Table", "TableCell", "ExtractedImage", "Entity", "Relation", "FormField",
+]

docqwise/core/bases.py

@@ -0,0 +1,180 @@
+"""All pluggable base classes for docqwise.
+
+Every component is a Base* class. Swap anything. Test anything. Deploy anywhere.
+Import from the specific module (e.g. docqwise.ocr.base) for implementation.
+This file serves as a single reference for all interfaces.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+import numpy as np
+
+from docqwise.core.document import DocqwiseDocument
+from docqwise.core.chunk import DocqwiseChunk, BoundingBox
+from docqwise.core.field import ExtractionResult
+from docqwise.core.element import Table, Entity, Relation, FormField
+
+
+# ── PROCESSING LAYER ──
+
+
+class BaseReader(ABC):
+    @abstractmethod
+    def can_read(self, path: str) -> bool: ...
+    @abstractmethod
+    def read(self, path: str, **kwargs) -> DocqwiseDocument: ...
+    @abstractmethod
+    def supported_formats(self) -> list[str]: ...
+
+
+class BaseOCREngine(ABC):
+    @abstractmethod
+    def ocr_page(self, image: np.ndarray, language: str = "en") -> dict: ...
+    @abstractmethod
+    def ocr_batch(self, images: list[np.ndarray], **kwargs) -> list[dict]: ...
+    @abstractmethod
+    def supported_languages(self) -> list[str]: ...
+
+
+class BaseLayoutModel(ABC):
+    @abstractmethod
+    def detect_layout(self, image: np.ndarray) -> list[dict]: ...
+    @abstractmethod
+    def detect_batch(self, images: list[np.ndarray]) -> list[list[dict]]: ...
+
+
+class BaseExtractor(ABC):
+    @abstractmethod
+    def extract(self, document: DocqwiseDocument, **kwargs) -> DocqwiseDocument: ...
+
+
+class BaseTableExtractor(ABC):
+    @abstractmethod
+    def extract_tables(self, document: DocqwiseDocument, **kwargs) -> list[Table]: ...
+
+
+class BaseFormExtractor(ABC):
+    @abstractmethod
+    def extract_form(self, document: DocqwiseDocument) -> dict[str, FormField]: ...
+
+
+class BaseChunker(ABC):
+    @abstractmethod
+    def chunk(self, document: DocqwiseDocument, **kwargs) -> list[DocqwiseChunk]: ...
+
+
+class BaseEmbedder(ABC):
+    @abstractmethod
+    def embed(self, text: str) -> np.ndarray: ...
+    @abstractmethod
+    def embed_batch(self, texts: list[str]) -> np.ndarray: ...
+    @abstractmethod
+    def dimension(self) -> int: ...
+
+
+# ── STORAGE LAYER ──
+
+
+class BaseVectorStore(ABC):
+    @abstractmethod
+    def insert(self, chunks: list[DocqwiseChunk]) -> None: ...
+    @abstractmethod
+    def search(self, query_embedding: np.ndarray, top_k: int = 5,
+               filters: Optional[dict] = None) -> list[dict]: ...
+    @abstractmethod
+    def delete(self, doc_id: str) -> None: ...
+    @abstractmethod
+    def update(self, doc_id: str, chunks: list[DocqwiseChunk]) -> None: ...
+    @abstractmethod
+    def count(self) -> int: ...
+
+
+class BaseDatabaseStore(ABC):
+    @abstractmethod
+    def connect(self, connection_string: str, **kwargs) -> None: ...
+    @abstractmethod
+    def insert(self, data: dict, table: str) -> None: ...
+    @abstractmethod
+    def query(self, query: str, params: Optional[dict] = None) -> list[dict]: ...
+    @abstractmethod
+    def schema(self, table: Optional[str] = None) -> dict: ...
+    @abstractmethod
+    def tables(self) -> list[str]: ...
+
+
+class BaseGraphStore(ABC):
+    @abstractmethod
+    def add_node(self, node_id: str, node_type: str, properties: dict) -> None: ...
+    @abstractmethod
+    def add_edge(self, from_id: str, to_id: str, relation: str,
+                 properties: Optional[dict] = None) -> None: ...
+    @abstractmethod
+    def query(self, query: str) -> list[dict]: ...
+    @abstractmethod
+    def neighbors(self, node_id: str, hops: int = 1) -> list[dict]: ...
+    @abstractmethod
+    def communities(self) -> list[list[str]]: ...
+
+
+# ── INTELLIGENCE LAYER ──
+
+
+class BaseLLM(ABC):
+    @abstractmethod
+    def generate(self, prompt: str, **kwargs) -> str: ...
+    @abstractmethod
+    def generate_structured(self, prompt: str, schema: dict) -> dict: ...
+
+
+class BaseReranker(ABC):
+    @abstractmethod
+    def rerank(self, query: str, documents: list[str],
+               top_k: int = 5) -> list[dict]: ...
+
+
+class BaseClassifier(ABC):
+    @abstractmethod
+    def classify(self, document: DocqwiseDocument,
+                 labels: Optional[list[str]] = None) -> list[dict]: ...
+
+
+class BaseComparator(ABC):
+    @abstractmethod
+    def compare(self, doc_a: DocqwiseDocument,
+                doc_b: DocqwiseDocument) -> dict: ...
+
+
+# ── INFRASTRUCTURE LAYER ──
+
+
+class BaseConnector(ABC):
+    @abstractmethod
+    def list_files(self, path: str, **kwargs) -> list[str]: ...
+    @abstractmethod
+    def read_file(self, path: str) -> bytes: ...
+    @abstractmethod
+    def watch(self, path: str, callback: Any, interval: int = 60) -> None: ...
+
+
+class BaseCacheLayer(ABC):
+    @abstractmethod
+    def get(self, key: str) -> Optional[Any]: ...
+    @abstractmethod
+    def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None: ...
+    @abstractmethod
+    def invalidate(self, key: str) -> None: ...
+
+
+class BaseExporter(ABC):
+    @abstractmethod
+    def export(self, documents: list[DocqwiseDocument], path: str, **kwargs) -> None: ...
+
+
+class BaseTemplate(ABC):
+    @abstractmethod
+    def schema(self) -> dict: ...
+    @abstractmethod
+    def extract(self, document: DocqwiseDocument) -> ExtractionResult: ...
+    @abstractmethod
+    def validate(self, result: ExtractionResult) -> dict: ...

docqwise/core/chunk.py

@@ -0,0 +1,80 @@
+"""Chunk representation for retrieval."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Optional
+
+import numpy as np
+
+
+class ElementType(str, Enum):
+    TEXT = "text"
+    TABLE = "table"
+    IMAGE = "image"
+    HEADING = "heading"
+    LIST = "list"
+    CAPTION = "caption"
+    FORM_FIELD = "form_field"
+    CODE = "code"
+    EQUATION = "equation"
+
+
+@dataclass
+class BoundingBox:
+    """Bounding box coordinates on a page."""
+
+    x1: float
+    y1: float
+    x2: float
+    y2: float
+    page: int = 0
+
+    @property
+    def width(self) -> float:
+        return self.x2 - self.x1
+
+    @property
+    def height(self) -> float:
+        return self.y2 - self.y1
+
+    @property
+    def area(self) -> float:
+        return self.width * self.height
+
+    def to_list(self) -> list[float]:
+        return [self.x1, self.y1, self.x2, self.y2]
+
+
+@dataclass
+class ChunkMetadata:
+    """Metadata attached to a chunk."""
+
+    doc_id: str = ""
+    source_path: str = ""
+    page_num: int = 0
+    chunk_index: int = 0
+    element_type: ElementType = ElementType.TEXT
+    heading_context: str = ""
+    section_path: list[str] = field(default_factory=list)
+    bbox: Optional[BoundingBox] = None
+    confidence: float = 1.0
+    custom: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DocqwiseChunk:
+    """A retrieval-ready piece of a document."""
+
+    chunk_id: str
+    text: str
+    embedding: Optional[np.ndarray] = None
+    metadata: ChunkMetadata = field(default_factory=ChunkMetadata)
+
+    # For table chunks
+    table_data: Optional[dict] = None
+
+    # For image chunks
+    image_path: Optional[str] = None
+    image_description: str = ""

docqwise/core/document.py

@@ -0,0 +1,101 @@
+"""Core document representation."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+
+class DocumentType(str, Enum):
+    PDF = "pdf"
+    DOCX = "docx"
+    IMAGE = "image"
+    EXCEL = "excel"
+    CSV = "csv"
+    JSON = "json"
+    XML = "xml"
+    YAML = "yaml"
+    HTML = "html"
+    PPTX = "pptx"
+    EMAIL = "email"
+    TEXT = "text"
+    PARQUET = "parquet"
+    DATABASE = "database"
+    RTF = "rtf"
+    MARKDOWN = "markdown"
+    UNKNOWN = "unknown"
+
+
+class SourceType(str, Enum):
+    FILE = "file"
+    URL = "url"
+    DATABASE = "database"
+    CLOUD_STORAGE = "cloud_storage"
+    EMAIL = "email"
+    STREAM = "stream"
+
+
+@dataclass
+class DocumentMetadata:
+    """Metadata extracted from a document."""
+
+    title: Optional[str] = None
+    author: Optional[str] = None
+    created_at: Optional[datetime] = None
+    modified_at: Optional[datetime] = None
+    page_count: int = 0
+    word_count: int = 0
+    language: Optional[str] = None
+    file_size_bytes: int = 0
+    file_hash: str = ""
+    is_scanned: bool = False
+    has_selectable_text: bool = True
+    custom: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class PageContent:
+    """Content of a single page."""
+
+    page_num: int
+    text: str = ""
+    width: float = 0.0
+    height: float = 0.0
+    elements: list[Any] = field(default_factory=list)
+    is_scanned: bool = False
+
+
+@dataclass
+class DocqwiseDocument:
+    """Core document representation output of reader and extractors."""
+
+    doc_id: str
+    source_path: str
+    source_type: SourceType
+    doc_type: DocumentType
+    metadata: DocumentMetadata = field(default_factory=DocumentMetadata)
+
+    # Extracted content
+    text: str = ""
+    pages: list[PageContent] = field(default_factory=list)
+    tables: list[Any] = field(default_factory=list)
+    images: list[Any] = field(default_factory=list)
+    fields: dict[str, Any] = field(default_factory=dict)
+    entities: list[Any] = field(default_factory=list)
+    relations: list[Any] = field(default_factory=list)
+    form_fields: dict[str, Any] = field(default_factory=dict)
+
+    # Processing state
+    extraction_strategy: str = ""
+    extraction_confidence: float = 0.0
+    processing_time_ms: float = 0.0
+    extracted_at: Optional[datetime] = None
+
+    @staticmethod
+    def generate_id(source_path: str, content_hash: str = "") -> str:
+        """Generate a unique document ID."""
+        raw = f"{source_path}:{content_hash}"
+        return hashlib.sha256(raw.encode()).hexdigest()[:16]