doclex-cli 0.1.1__py3-none-any.whl

doclex/cache/store.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+
+from doclex.metadata.models import ExtractedDoc, Segment
+
+
+class CacheStore:
+    """JSON file cache: one file per document, keyed by sha256 of the path."""
+
+    def __init__(self, cache_dir: Path) -> None:
+        self._dir = cache_dir
+        self._dir.mkdir(parents=True, exist_ok=True)
+
+    def _cache_path(self, doc_path: str) -> Path:
+        key = hashlib.sha256(doc_path.encode()).hexdigest()
+        return self._dir / f"{key}.json"
+
+    def get(self, doc_path: str) -> tuple[ExtractedDoc, list[Segment]] | None:
+        """Return cached (doc, segments) or None if not present."""
+        cache_file = self._cache_path(doc_path)
+        if not cache_file.exists():
+            return None
+        try:
+            data = json.loads(cache_file.read_text(encoding="utf-8"))
+            doc = ExtractedDoc(
+                path=data["path"],
+                text="",  # content lives in index nodes
+                mime_type=data["mime_type"],
+                metadata=data.get("metadata", {}),
+            )
+            segments = _flatten_index(data.get("index", {}), data["path"])
+            return doc, segments
+        except (KeyError, json.JSONDecodeError):
+            return None
+
+    def put(self, doc: ExtractedDoc, segments: list[Segment]) -> None:
+        """Write extracted doc and its segments as a unified index tree."""
+        cache_file = self._cache_path(doc.path)
+        data = {
+            "path": doc.path,
+            "mime_type": doc.mime_type,
+            "metadata": doc.metadata,
+            "index": _build_index(Path(doc.path).stem, segments),
+        }
+        cache_file.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    def delete(self, doc_path: str) -> None:
+        cache_file = self._cache_path(doc_path)
+        if cache_file.exists():
+            cache_file.unlink()
+
+
+# ------------------------------------------------------------------
+# Index tree builder
+# ------------------------------------------------------------------
+
+def _build_index(title: str, segments: list[Segment]) -> dict:
+    """Build a unified PageIndex-style tree from flat segments."""
+    root: dict = {"title": title, "nodes": []}
+
+    has_hierarchy = any(s.level > 0 for s in segments)
+    if not has_hierarchy:
+        root["nodes"] = [_seg_to_node(s) for s in segments]
+        return root
+
+    stack: list[tuple[int, dict]] = [(0, root)]
+    for seg in segments:
+        node = _seg_to_node(seg)
+        if seg.level > 0:
+            # heading node: pop continuations (level=0) and same/deeper headings
+            while len(stack) > 1 and (stack[-1][0] == 0 or stack[-1][0] >= seg.level):
+                stack.pop()
+            stack[-1][1]["nodes"].append(node)
+            stack.append((seg.level, node))
+        else:
+            if stack[-1][0] == 0:
+                # previous was also a continuation — nest inside it
+                stack[-1][1]["nodes"].append(node)
+            else:
+                # first continuation after a heading — create a parent node from heading
+                title, _ = _extract_heading(node.get("content", ""))
+                if title:
+                    _, body = _extract_heading(node.get("content", ""))
+                    group_id = node["id"]
+                    node["id"] = hashlib.sha256((group_id + "|body").encode()).hexdigest()[:32]
+                    node["content"] = body
+                    parent_node: dict = {
+                        "id": group_id,
+                        "title": title,
+                        "location_type": node.get("location_type", "section"),
+                        "content": "",
+                        "bm25_doc_id": None,
+                        "nodes": [node],
+                    }
+                    stack[-1][1]["nodes"].append(parent_node)
+                    stack.append((0, parent_node))
+                else:
+                    stack[-1][1]["nodes"].append(node)
+                    stack.append((0, node))
+
+    _assign_navigation_links(root)
+    _clean_nodes(root)
+    return root
+
+
+def _extract_heading(content: str) -> tuple[str, str]:
+    """Extract the first markdown heading from content, return (title, remaining_body)."""
+    lines = content.split("\n", 1)
+    first_line = lines[0].strip()
+    m = re.match(r"^#{1,3} (.+)", first_line)
+    if not m:
+        return "", content
+    title = m.group(1).strip()
+    body = lines[1].strip() if len(lines) > 1 else ""
+    return title, body
+
+
+def _assign_navigation_links(root: dict) -> None:
+    """Assign prev_id, next_id, parent_id to every node in pre-order (reading order)."""
+    flat: list[tuple[dict, str | None]] = []
+
+    def collect(nodes: list, parent_id: str | None) -> None:
+        for node in nodes:
+            flat.append((node, parent_id))
+            collect(node.get("nodes", []), node["id"])
+
+    collect(root.get("nodes", []), None)
+
+    for i, (node, parent_id) in enumerate(flat):
+        node["parent_id"] = parent_id
+        node["prev_id"] = flat[i - 1][0]["id"] if i > 0 else None
+        node["next_id"] = flat[i + 1][0]["id"] if i < len(flat) - 1 else None
+
+
+def _clean_nodes(node: dict) -> None:
+    """Clean node titles and content to remove redundancy:
+    - Leaf nodes: drop title (content is sufficient)
+    - Non-leaf heading nodes: keep title, strip heading line from content
+    """
+    for child in node.get("nodes", []):
+        _clean_nodes(child)
+    if not node.get("nodes"):
+        node.pop("title", None)
+    elif node.get("level", 0) > 0 and node.get("title"):
+        _, body = _extract_heading(node.get("content", ""))
+        node["content"] = body
+
+
+def _seg_to_node(seg: Segment) -> dict:
+    node: dict = {
+        "id": seg.id,
+        "title": seg.location_label,
+        "location_type": seg.location_type,
+        "content": seg.content,
+        "bm25_doc_id": seg.bm25_doc_id,
+        "nodes": [],
+    }
+    if seg.level > 0:
+        node["level"] = seg.level
+    return node
+
+
+# ------------------------------------------------------------------
+# Index tree flattener (get)
+# ------------------------------------------------------------------
+
+def _flatten_index(index: dict, file_path: str) -> list[Segment]:
+    """Flatten a unified index tree back to a list of Segment objects."""
+    segments: list[Segment] = []
+
+    def walk(nodes: list, section_id: str | None, index: int) -> int:
+        for node in nodes:
+            node_id = node["id"]
+            level = node.get("level", 0)
+            seg = Segment(
+                id=node_id,
+                file_path=file_path,
+                content=node["content"],
+                location_type=node.get("location_type", "block"),
+                location_label=node.get("title") or node["content"].splitlines()[0].lstrip("#").strip()[:80],
+                location_index=index,
+                char_count=len(node["content"]),
+                bm25_doc_id=node.get("bm25_doc_id"),
+                section_id=node_id if level > 0 else section_id,
+                level=level,
+            )
+            segments.append(seg)
+            index = walk(node.get("nodes", []), seg.section_id, index + 1)
+        return index
+
+    walk(index.get("nodes", []), None, 0)
+    return segments

doclex/cli.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.progress import BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from rich.table import Table
+
+from doclex.config import Config
+from doclex.pipeline import Pipeline
+from doclex.registry import (
+    Project,
+    ProjectAlreadyExistsError,
+    ProjectFolderNotFoundError,
+    ProjectNotFoundError,
+    ProjectOverlapError,
+    ProjectRegistry,
+)
+
+app = typer.Typer(
+    name="doclex",
+    add_completion=False,
+    help="Local document search — BM25-based, no embeddings, no cloud.",
+)
+console = Console(width=220)
+
+
+def _resolve_project(name: str) -> Project:
+    try:
+        return ProjectRegistry().get(name)
+    except ProjectNotFoundError as exc:
+        console.print(
+            f"[red]Error:[/red] {exc}\n\n"
+            "Run [bold]doclex list[/bold] to see registered projects."
+        )
+        raise typer.Exit(1)
+
+
+def _build_pipeline(project: Project, *, create: bool = False) -> Pipeline:
+    os.environ["DOCLEX_FOLDER"] = str(project.folder)
+    config = Config()
+    if create:
+        config.ensure_dirs()
+    elif not config.cache_dir.exists():
+        console.print(
+            f"[yellow]No index found for project[/yellow] [bold]{project.name}[/bold] "
+            f"({project.folder}). Run [bold]doclex index --project {project.name}[/bold] first."
+        )
+        raise typer.Exit(0)
+    return Pipeline(config)
+
+
+# ------------------------------------------------------------------
+# add / remove / projects
+# ------------------------------------------------------------------
+
+@app.command()
+def add(
+    name: str = typer.Argument(..., help="Unique name for the project."),
+    folder: Path = typer.Argument(..., help="Folder to register."),
+) -> None:
+    """Register a folder under a project name.
+
+    \b
+    Examples:
+      doclex add docs ./docs
+    """
+    try:
+        project = ProjectRegistry().add(name, folder)
+    except (ProjectAlreadyExistsError, ProjectFolderNotFoundError, ProjectOverlapError) as exc:
+        console.print(f"[red]Error:[/red] {exc}")
+        raise typer.Exit(1)
+    console.print(f"[green]✓[/green] Registered project '{project.name}' → {project.folder}")
+
+
+@app.command("list")
+def list_projects() -> None:
+    """List registered projects."""
+    all_projects = ProjectRegistry().list()
+    if not all_projects:
+        console.print(
+            "[yellow]No projects registered.[/yellow] "
+            "Use [bold]doclex add <name> <folder>[/bold]."
+        )
+        return
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Name")
+    table.add_column("Folder")
+    for p in all_projects:
+        table.add_row(p.name, str(p.folder))
+    console.print(table)
+
+
+@app.command()
+def remove(
+    name: str = typer.Argument(..., help="Project name to remove."),
+) -> None:
+    """Remove a project and purge its cache/index data."""
+    try:
+        project = ProjectRegistry().remove(name)
+    except ProjectNotFoundError as exc:
+        console.print(f"[red]Error:[/red] {exc}")
+        raise typer.Exit(1)
+
+    cache_dir = project.folder / ".doclex"
+    shutil.rmtree(cache_dir, ignore_errors=True)
+    console.print(f"[green]✓[/green] Removed project '{name}' and its cache ({cache_dir}).")
+
+
+# ------------------------------------------------------------------
+# index
+# ------------------------------------------------------------------
+
+@app.command()
+def index(
+    project: str = typer.Option(
+        ..., "--project", "-p",
+        help="Registered project name.",
+    ),
+    files: list[Path] | None = typer.Option(
+        None, "--file",
+        help="One or more specific files to (re)index. Must be inside the project's folder.",
+    ),
+    force: bool = typer.Option(
+        False, "--force",
+        help="Force re-extraction of every file, ignoring cached state.",
+    ),
+) -> None:
+    """Index a project or re-index specific files.
+
+    \b
+    Examples:
+      doclex index --project docs                        # incremental scan
+      doclex index --project docs --force                # reindex everything
+      doclex index --project docs --file report.pdf      # reindex one file
+      doclex index --project docs --file a.pdf --file b.docx
+    """
+    if files and force:
+        console.print("[red]Error:[/red] --file and --force are mutually exclusive.")
+        raise typer.Exit(1)
+
+    proj = _resolve_project(project)
+
+    if files:
+        for f in files:
+            if not f.exists():
+                console.print(f"[red]Error:[/red] File not found: {f}")
+                raise typer.Exit(1)
+            if not f.resolve().is_relative_to(proj.folder):
+                console.print(
+                    f"[red]Error:[/red] {f} is not inside project '{proj.name}' ({proj.folder})"
+                )
+                raise typer.Exit(1)
+
+    pipeline = _build_pipeline(proj, create=True)
+
+    async def run() -> None:
+        await pipeline.startup()
+
+        # ── Specific files reindex ───────────────────────────────────
+        if files:
+            with Progress(
+                SpinnerColumn(), BarColumn(), MofNCompleteColumn(),
+                TimeElapsedColumn(), TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task("", total=len(files))
+                for f in files:
+                    progress.update(task, advance=1, description=f"[cyan]reindexing[/cyan] {f.name}")
+                    await pipeline.reindex_file(f)
+            console.print(f"[green]✓[/green] Reindexed {len(files)} file(s).")
+            await pipeline.shutdown()
+            return
+
+        # ── Force full reindex ───────────────────────────────────────
+        if force:
+            all_files = await pipeline.list_files()
+            total = len(all_files)
+            if total == 0:
+                console.print("[yellow]No files in index to reindex.[/yellow]")
+                await pipeline.shutdown()
+                return
+
+            console.print(f"[bold]Force reindexing[/bold] {total} files…")
+            with Progress(
+                SpinnerColumn(), BarColumn(), MofNCompleteColumn(),
+                TimeElapsedColumn(), TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task("", total=total)
+
+                def on_progress_force(path: Path, action: str) -> None:
+                    progress.update(task, advance=1, description=f"[cyan]{action}[/cyan] {path.name}")
+
+                await pipeline.reindex_all(on_progress=on_progress_force)
+            console.print("[green]✓[/green] Done.")
+            await pipeline.shutdown()
+            return
+
+        # ── Incremental reconciliation ───────────────────────────────
+        with Progress(SpinnerColumn(), TextColumn("{task.description}"), console=console, transient=True) as spin:
+            spin.add_task("Scanning folder…")
+            changes = await pipeline.reconcile()
+
+        total = len(changes.new) + len(changes.modified) + len(changes.deleted)
+
+        if total == 0:
+            console.print("[green]✓[/green] Index is up to date. No changes detected.")
+        else:
+            console.print(
+                f"[bold]Processing:[/bold] {len(changes.new)} new, "
+                f"{len(changes.modified)} modified, "
+                f"{len(changes.deleted)} deleted"
+            )
+            with Progress(
+                SpinnerColumn(), BarColumn(), MofNCompleteColumn(),
+                TimeElapsedColumn(), TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task("", total=total)
+
+                def on_progress(path: Path, action: str) -> None:
+                    progress.update(task, advance=1, description=f"[cyan]{action}[/cyan] {path.name}")
+
+                await pipeline.apply_changes(changes, on_progress=on_progress)
+
+            console.print("[green]✓[/green] Done.")
+
+        await pipeline.shutdown()
+
+    asyncio.run(run())
+
+    # ONNX Runtime (layout detection) crashes during Python interpreter shutdown
+    # when the GC tries to free native session objects. os._exit bypasses GC
+    # teardown; all work is already saved to disk.
+    os._exit(0)
+
+
+# ------------------------------------------------------------------
+# search
+# ------------------------------------------------------------------
+
+@app.command()
+def search(
+    query: str = typer.Argument(..., help="Search query (natural language or keywords)."),
+    project: str = typer.Option(
+        ..., "--project", "-p",
+        help="Registered project name.",
+    ),
+    top_k: int = typer.Option(10, "--top", "-k", help="Maximum number of results to return."),
+    full: bool = typer.Option(
+        False, "--full", "-F",
+        help="Show the full segment text instead of a short snippet.",
+    ),
+) -> None:
+    """Search indexed documents and show results with citations.
+
+    \b
+    Examples:
+      doclex search "GT/DPM" --project docs
+      doclex search "presupuesto 2024" --project docs --top 5 --full
+    """
+    proj = _resolve_project(project)
+    pipeline = _build_pipeline(proj)
+
+    async def run() -> None:
+        with Progress(SpinnerColumn(), TextColumn("{task.description}"), console=console, transient=True) as spin:
+            spin.add_task("Searching…")
+            await pipeline.startup()
+            results = await pipeline.search(query, top_k)
+            await pipeline.shutdown()
+
+        if not results:
+            console.print("[yellow]No results found.[/yellow]")
+            return
+
+        console.print()
+        for i, r in enumerate(results, 1):
+            file_name = Path(r.file_path).name
+            console.print(
+                f"[bold cyan][{i}][/bold cyan] [bold]{file_name}[/bold] — "
+                f"[dim]{r.location_label}[/dim]  "
+                f"[yellow](score: {r.score:.3f})[/yellow]"
+            )
+            text = r.content if full else r.snippet
+            console.print(f'    [italic]"{text}"[/italic]')
+            console.print()
+
+    asyncio.run(run())
+
+
+# ------------------------------------------------------------------
+# status
+# ------------------------------------------------------------------
+
+@app.command()
+def status(
+    project: str = typer.Option(
+        ..., "--project", "-p",
+        help="Registered project name.",
+    ),
+    files: list[Path] | None = typer.Option(
+        None, "--file",
+        help="Inspect one or more specific files.",
+    ),
+    all_files: bool = typer.Option(
+        False, "--all",
+        help="List all indexed files with their status.",
+    ),
+    filter_status: str | None = typer.Option(
+        None, "--filter",
+        help="Filter file list by status: indexed | error | pending.",
+    ),
+) -> None:
+    """Show corpus status, optionally listing files or inspecting specific files.
+
+    \b
+    Examples:
+      doclex status --project docs                                   # corpus summary
+      doclex status --project docs --all                             # list all files
+      doclex status --project docs --all --filter error              # filtered list
+      doclex status --project docs --file report.pdf --file r3.pdf   # inspect specific files
+    """
+    proj = _resolve_project(project)
+    pipeline = _build_pipeline(proj)
+
+    async def run() -> None:
+        await pipeline.startup()
+
+        # ── Specific-file inspect ────────────────────────────────────
+        if files:
+            for f in files:
+                info = await pipeline.get_file_status(f)
+                rec = info["record"]
+                console.print()
+
+                t = Table(show_header=False, box=None, padding=(0, 2))
+                t.add_column(style="bold dim")
+                t.add_column()
+                t.add_row("File", str(f))
+                t.add_row("On disk", "[green]yes[/green]" if info["on_disk"] else "[red]no[/red]")
+                t.add_row("In index", "[green]yes[/green]" if info["in_index"] else "[yellow]no[/yellow]")
+
+                if rec:
+                    modified_label = "[yellow]yes — run index to update[/yellow]" if info["is_modified"] else "[green]no[/green]"
+                    t.add_row("Modified on disk", modified_label)
+                    t.add_row("Status", rec.status)
+                    t.add_row("MIME type", rec.mime_type)
+                    t.add_row("Segments", str(len(info["segments"])))
+                    if rec.error_msg:
+                        t.add_row("Error", f"[red]{rec.error_msg}[/red]")
+
+                console.print(t)
+
+            await pipeline.shutdown()
+            return
+
+        # ── File list ────────────────────────────────────────────────
+        if all_files:
+            file_list = await pipeline.list_files(filter_status)
+            await pipeline.shutdown()
+
+            if not file_list:
+                console.print("[yellow]No files found.[/yellow]")
+                return
+
+            table = Table(show_header=True, header_style="bold")
+            table.add_column("Status", width=10)
+            table.add_column("File")
+            table.add_column("MIME type", style="dim")
+
+            style_map = {"indexed": "green", "error": "red", "pending": "yellow"}
+            icon_map = {"indexed": "✓", "error": "✗", "pending": "…"}
+
+            for f in file_list:
+                color = style_map.get(f.status, "white")
+                icon = icon_map.get(f.status, "?")
+                table.add_row(
+                    f"[{color}]{icon} {f.status}[/{color}]",
+                    f.path,
+                    f.mime_type,
+                )
+            console.print(table)
+            return
+
+        # ── Corpus summary ───────────────────────────────────────────
+        info = await pipeline.corpus_status()
+        await pipeline.shutdown()
+
+        t = Table(show_header=False, box=None, padding=(0, 2))
+        t.add_column(style="bold dim")
+        t.add_column()
+
+        t.add_row("Folder", str(info["folder"]))
+        t.add_row(
+            "Indexed",
+            f"[green]{info['files_indexed']} files[/green]"
+            + (f"  [red]{info['files_error']} errors[/red]" if info["files_error"] else "")
+            + (f"  [yellow]{info['files_pending']} pending[/yellow]" if info["files_pending"] else ""),
+        )
+        t.add_row("Segments", str(info["segments_total"]))
+        t.add_row("BM25 index", str(info["index_size"]) + " docs")
+
+        if not info["has_index"]:
+            t.add_row("Disk state", "[yellow]not indexed — run doclex index --project <name>[/yellow]")
+        else:
+            pending_total = info["pending_new"] + info["pending_modified"] + info["pending_deleted"]
+            if pending_total:
+                t.add_row(
+                    "Needs reindex",
+                    f"[yellow]{info['pending_new']} new  "
+                    f"{info['pending_modified']} modified  "
+                    f"{info['pending_deleted']} deleted[/yellow]"
+                    "  → run [bold]doclex index[/bold]",
+                )
+            else:
+                t.add_row("Disk state", "[green]up to date[/green]")
+
+        console.print(t)
+
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    app()

doclex/config.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+class Config:
+    def __init__(self) -> None:
+        folder_env = os.environ.get("DOCLEX_FOLDER")
+        if not folder_env:
+            raise ValueError("DOCLEX_FOLDER environment variable is required")
+
+        self.folder = Path(folder_env).expanduser().resolve()
+        if not self.folder.is_dir():
+            raise ValueError(f"DOCLEX_FOLDER is not a directory: {self.folder}")
+
+        cache_env = os.environ.get("DOCLEX_CACHE_DIR")
+        self.cache_dir = (
+            Path(cache_env).expanduser().resolve()
+            if cache_env
+            else self.folder / ".doclex"
+        )
+
+        db_env = os.environ.get("DOCLEX_DB_PATH")
+        self.db_path = (
+            Path(db_env).expanduser().resolve()
+            if db_env
+            else self.cache_dir / "doclex.db"
+        )
+
+        self.bm25_index_dir = self.cache_dir / "bm25_index"
+
+        self.top_k = int(os.environ.get("DOCLEX_TOP_K", "10"))
+        self.debounce_secs = float(os.environ.get("DOCLEX_DEBOUNCE", "1.0"))
+
+    def ensure_dirs(self) -> None:
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+
+
+SUPPORTED_EXTENSIONS: frozenset[str] = frozenset(
+    {".pdf", ".txt", ".md", ".docx", ".xlsx", ".csv"}
+)