uridx 0.1.0__tar.gz

uridx-0.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: uridx
+Version: 0.1.0
+Summary: uridx - Personal Reference Index
+Author: Justyn Shull
+Requires-Dist: fastmcp>=2.0,<3
+Requires-Dist: sqlmodel>=0.0.16
+Requires-Dist: httpx>=0.27
+Requires-Dist: typer>=0.12
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: rich>=13
+Requires-Dist: ruff ; extra == 'dev'
+Requires-Dist: docling>=2.66 ; extra == 'docling'
+Requires-Dist: pdfplumber>=0.10 ; extra == 'pdf'
+Requires-Python: >=3.12
+Provides-Extra: dev
+Provides-Extra: docling
+Provides-Extra: pdf

uridx-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "uridx"
+version = "0.1.0"
+description = "uridx - Personal Reference Index"
+authors = [
+    { name = "Justyn Shull" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "fastmcp>=2.0,<3",
+    "sqlmodel>=0.0.16",
+    "httpx>=0.27",
+    "typer>=0.12",
+    "sqlite-vec>=0.1.6",
+    "rich>=13",
+]
+
+[project.scripts]
+uridx = "uridx.cli.main:app"
+
+[project.optional-dependencies]
+dev = ["ruff"]
+docling = ["docling>=2.66"]
+pdf = ["pdfplumber>=0.10"]
+
+[tool.ruff]
+line-length = 120
+
+[build-system]
+requires = ["uv_build>=0.9.22,<0.10.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.14.10",
+]

uridx-0.1.0/src/uridx/__init__.py

@@ -0,0 +1,7 @@
+__version__ = "0.1.0"
+
+
+def main():
+    from uridx.cli.main import app
+
+    app()

uridx-0.1.0/src/uridx/cli/__init__.py

@@ -0,0 +1,3 @@
+from uridx.cli.main import app
+
+__all__ = ["app"]

uridx-0.1.0/src/uridx/cli/extract/__init__.py

@@ -0,0 +1,41 @@
+"""Extract subcommands with plugin support.
+
+Built-in extractors and plugin discovery via entry points.
+Plugins register under 'uridx.extractors' entry point group.
+"""
+
+import sys
+from importlib.metadata import entry_points
+
+import typer
+
+from . import claude_code, docling, image, markdown, pdf
+
+app = typer.Typer(help="Extract content to JSONL for ingestion")
+
+app.command("claude-code")(claude_code.extract)
+app.command("docling")(docling.extract)
+app.command("markdown")(markdown.extract)
+app.command("pdf")(pdf.extract)
+app.command("image")(image.extract)
+
+
+def load_plugins():
+    """Load extractor plugins from entry points."""
+    try:
+        eps = entry_points(group="uridx.extractors")
+    except TypeError:
+        eps = entry_points().get("uridx.extractors", [])
+
+    for ep in eps:
+        try:
+            extractor = ep.load()
+            if isinstance(extractor, typer.Typer):
+                app.add_typer(extractor, name=ep.name)
+            elif callable(extractor):
+                app.command(ep.name)(extractor)
+        except Exception as e:
+            print(f"Failed to load extractor plugin '{ep.name}': {e}", file=sys.stderr)
+
+
+load_plugins()

uridx-0.1.0/src/uridx/cli/extract/base.py

@@ -0,0 +1,30 @@
+"""Shared utilities for extractors."""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def get_file_mtime(path: Path) -> str:
+    """Get file modification time as ISO8601 string."""
+    mtime = path.stat().st_mtime
+    return datetime.fromtimestamp(mtime, tz=timezone.utc).isoformat()
+
+
+def output(record: dict) -> None:
+    """Output a single JSONL record to stdout."""
+    print(json.dumps(record))
+
+
+def resolve_paths(paths: list[Path], extensions: set[str]) -> list[Path]:
+    """Resolve a list of paths to matching files."""
+    if not paths:
+        paths = [Path.cwd()]
+
+    files = []
+    for p in paths:
+        if p.is_file():
+            files.append(p)
+        elif p.is_dir():
+            files.extend(f for f in p.rglob("*") if f.suffix.lower() in extensions and f.is_file())
+    return files

uridx-0.1.0/src/uridx/cli/extract/claude_code.py

@@ -0,0 +1,172 @@
+"""Extract Claude Code conversations from ~/.claude/projects/"""
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from .base import output
+
+
+def _extract_content(message: dict) -> str:
+    content = message.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        texts = []
+        for block in content:
+            if isinstance(block, dict):
+                if block.get("type") == "text":
+                    texts.append(block.get("text", ""))
+                elif block.get("type") == "tool_use":
+                    texts.append(f"[Tool: {block.get('name')}]")
+        return "\n".join(texts)
+    return ""
+
+
+def _is_tool_result(msg: dict) -> bool:
+    content = msg.get("message", {}).get("content", [])
+    if isinstance(content, list) and content:
+        first = content[0] if content else {}
+        return isinstance(first, dict) and first.get("type") == "tool_result"
+    return False
+
+
+def _build_turns(messages: list[dict]) -> list[dict]:
+    turns = []
+    current_user = None
+    current_assistant = []
+    turn_index = 0
+
+    for msg in messages:
+        msg_type = msg.get("type")
+        if msg_type not in ("user", "assistant"):
+            continue
+
+        content = _extract_content(msg.get("message", {}))
+        if not content:
+            continue
+
+        if msg_type == "user" and not _is_tool_result(msg):
+            if current_user or current_assistant:
+                text_parts = []
+                if current_user:
+                    text_parts.append(f"User: {current_user}")
+                if current_assistant:
+                    text_parts.append(f"Assistant: {' '.join(current_assistant)}")
+                if text_parts:
+                    turns.append(
+                        {
+                            "text": "\n\n".join(text_parts),
+                            "key": f"turn-{turn_index}",
+                            "meta": {"turn_index": turn_index},
+                        }
+                    )
+                    turn_index += 1
+            current_user = content
+            current_assistant = []
+        elif msg_type == "assistant":
+            current_assistant.append(content)
+
+    if current_user or current_assistant:
+        text_parts = []
+        if current_user:
+            text_parts.append(f"User: {current_user}")
+        if current_assistant:
+            text_parts.append(f"Assistant: {' '.join(current_assistant)}")
+        if text_parts:
+            turns.append(
+                {
+                    "text": "\n\n".join(text_parts),
+                    "key": f"turn-{turn_index}",
+                    "meta": {"turn_index": turn_index},
+                }
+            )
+
+    return turns
+
+
+def _parse_conversation(jsonl_path: Path) -> dict | None:
+    messages = []
+    metadata = {}
+    first_timestamp = None
+    last_timestamp = None
+
+    with open(jsonl_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                msg = json.loads(line)
+                messages.append(msg)
+                if first_timestamp is None:
+                    first_timestamp = msg.get("timestamp")
+                last_timestamp = msg.get("timestamp")
+                if not metadata and msg.get("cwd"):
+                    metadata = {
+                        "project_path": msg.get("cwd"),
+                        "agent_id": msg.get("agentId"),
+                        "session_id": msg.get("sessionId"),
+                        "git_branch": msg.get("gitBranch"),
+                        "slug": msg.get("slug"),
+                    }
+            except json.JSONDecodeError:
+                continue
+
+    if not messages:
+        return None
+
+    chunks = _build_turns(messages)
+    if not chunks:
+        return None
+
+    title = metadata.get("slug") or jsonl_path.stem
+    metadata["started_at"] = first_timestamp
+    metadata["ended_at"] = last_timestamp
+
+    return {"chunks": chunks, "metadata": metadata, "title": title}
+
+
+def extract(
+    path: Annotated[Optional[Path], typer.Argument(help="Projects directory")] = None,
+):
+    """Extract Claude Code conversations from ~/.claude/projects/"""
+    projects_dir = path or (Path.home() / ".claude" / "projects")
+
+    if not projects_dir.exists():
+        print(f"Projects directory not found: {projects_dir}", file=sys.stderr)
+        raise typer.Exit(1)
+
+    for project_dir in projects_dir.iterdir():
+        if not project_dir.is_dir():
+            continue
+
+        project_hash = project_dir.name
+
+        for jsonl_file in project_dir.glob("*.jsonl"):
+            if jsonl_file.stat().st_size == 0:
+                continue
+
+            try:
+                result = _parse_conversation(jsonl_file)
+            except Exception as e:
+                print(f"Error parsing {jsonl_file}: {e}", file=sys.stderr)
+                continue
+
+            if not result or not result["chunks"]:
+                continue
+
+            output(
+                {
+                    "source_uri": f"claude-code://{project_hash}/{jsonl_file.stem}",
+                    "chunks": result["chunks"],
+                    "tags": ["claude-code", "conversation"],
+                    "title": result["title"],
+                    "source_type": "claude-code",
+                    "context": json.dumps(result["metadata"]),
+                    "replace": True,
+                }
+            )

uridx-0.1.0/src/uridx/cli/extract/docling.py

@@ -0,0 +1,84 @@
+"""Extract documents using docling (PDF, DOCX, XLSX, PPTX, HTML, images)."""
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+from urllib.parse import urlparse
+
+import typer
+
+from .base import get_file_mtime, output, resolve_paths
+
+SUPPORTED_EXTENSIONS = {
+    ".pdf", ".docx", ".xlsx", ".pptx",
+    ".html", ".xhtml", ".htm",
+    ".png", ".jpg", ".jpeg", ".tiff", ".bmp", ".webp",
+    ".md", ".adoc", ".csv",
+}
+
+
+def extract(
+    sources: Annotated[Optional[list[str]], typer.Argument(help="Files, directories, or URLs")] = None,
+):
+    """Extract documents using docling (requires docling)."""
+    try:
+        from docling.document_converter import DocumentConverter
+        from docling_core.transforms.chunker import HybridChunker
+    except ImportError:
+        print("docling not installed. Install with: uv pip install 'uridx[docling]'", file=sys.stderr)
+        raise typer.Exit(1)
+
+    converter = DocumentConverter()
+    chunker = HybridChunker()
+
+    sources = sources or []
+    urls = [s for s in sources if s.startswith(("http://", "https://"))]
+    local_paths = [Path(s) for s in sources if not s.startswith(("http://", "https://"))]
+
+    for url in urls:
+        _convert_source(converter, chunker, url, url, created_at=None)
+
+    for file_path in resolve_paths(local_paths, SUPPORTED_EXTENSIONS):
+        _convert_source(converter, chunker, str(file_path), f"file://{file_path.resolve()}", created_at=get_file_mtime(file_path))
+
+
+def _convert_source(converter, chunker, source: str, source_uri: str, created_at: str | None = None):
+    """Convert a single source and output JSONL."""
+    try:
+        result = converter.convert(source)
+        doc = result.document
+        chunk_iter = chunker.chunk(dl_doc=doc)
+    except Exception as e:
+        print(f"Error processing {source}: {e}", file=sys.stderr)
+        return
+
+    chunks = []
+    for i, chunk in enumerate(chunk_iter):
+        text = chunk.text.strip() if hasattr(chunk, "text") else str(chunk).strip()
+        if text:
+            chunks.append({"text": text, "key": f"chunk-{i}"})
+
+    if not chunks:
+        return
+
+    parsed = urlparse(source)
+    if parsed.scheme in ("http", "https"):
+        title = Path(parsed.path).stem or parsed.netloc
+        ext = Path(parsed.path).suffix.lstrip(".").lower() or "html"
+    else:
+        title = Path(source).stem
+        ext = Path(source).suffix.lstrip(".").lower()
+
+    record = {
+        "source_uri": source_uri,
+        "chunks": chunks,
+        "tags": ["document", ext] if ext else ["document"],
+        "title": title,
+        "source_type": "document",
+        "context": json.dumps({"source": source}),
+        "replace": True,
+    }
+    if created_at:
+        record["created_at"] = created_at
+    output(record)

uridx-0.1.0/src/uridx/cli/extract/image.py

@@ -0,0 +1,67 @@
+"""Extract image descriptions via Ollama vision model."""
+
+import base64
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import httpx
+import typer
+
+from .base import get_file_mtime, output, resolve_paths
+
+EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"}
+
+
+def extract(
+    paths: Annotated[Optional[list[Path]], typer.Argument(help="Files or directories")] = None,
+    model: Annotated[str, typer.Option("--model", "-m", help="Vision model")] = "",
+    base_url: Annotated[str, typer.Option("--base-url", help="Ollama URL")] = "",
+):
+    """Extract image descriptions via Ollama vision model."""
+    ollama_url = base_url or os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+    vision_model = model or os.getenv("OLLAMA_VISION_MODEL", "llama3.2-vision")
+
+    for img_file in resolve_paths(paths or [], EXTENSIONS):
+        try:
+            with open(img_file, "rb") as f:
+                image_data = base64.b64encode(f.read()).decode("utf-8")
+
+            with httpx.Client(timeout=120.0) as client:
+                response = client.post(
+                    f"{ollama_url}/api/generate",
+                    json={
+                        "model": vision_model,
+                        "prompt": "Describe this image in detail. Include any text visible in the image.",
+                        "images": [image_data],
+                        "stream": False,
+                    },
+                )
+                response.raise_for_status()
+                description = response.json()["response"]
+        except httpx.ConnectError:
+            print(f"Cannot connect to Ollama at {ollama_url}", file=sys.stderr)
+            raise typer.Exit(1)
+        except Exception as e:
+            print(f"Error describing {img_file}: {e}", file=sys.stderr)
+            continue
+
+        if not description or not description.strip():
+            continue
+
+        output(
+            {
+                "source_uri": f"file://{img_file.resolve()}",
+                "chunks": [
+                    {"text": description.strip(), "key": "description", "meta": {"original_filename": img_file.name}}
+                ],
+                "tags": ["image"],
+                "title": img_file.stem,
+                "source_type": "image",
+                "context": json.dumps({"path": str(img_file), "vision_model": vision_model}),
+                "replace": True,
+                "created_at": get_file_mtime(img_file),
+            }
+        )

uridx-0.1.0/src/uridx/cli/extract/markdown.py

@@ -0,0 +1,100 @@
+"""Extract markdown files, splitting by headings."""
+
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from .base import get_file_mtime, output, resolve_paths
+
+EXTENSIONS = {".md", ".markdown"}
+
+
+def _slugify(text: str) -> str:
+    if not text:
+        return "untitled"
+    text = re.sub(r"^#+\s*", "", text)
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9]+", "-", text)
+    return text.strip("-")[:50] or "untitled"
+
+
+def _parse(path: Path) -> list[dict]:
+    content = path.read_text(encoding="utf-8")
+    heading_pattern = r"^(#{1,6}\s+.+)$"
+    parts = re.split(heading_pattern, content, flags=re.MULTILINE)
+
+    chunks = []
+    current_heading = None
+    current_content = []
+
+    for part in parts:
+        part = part.strip()
+        if not part:
+            continue
+        if re.match(r"^#{1,6}\s+", part):
+            if current_heading or current_content:
+                text_parts = [current_heading] if current_heading else []
+                text_parts.extend(current_content)
+                chunk_text = "\n\n".join(text_parts)
+                if chunk_text.strip():
+                    chunks.append(
+                        {
+                            "text": chunk_text,
+                            "key": _slugify(current_heading) if current_heading else f"section-{len(chunks)}",
+                            "meta": {"heading": current_heading},
+                        }
+                    )
+            current_heading = part
+            current_content = []
+        else:
+            current_content.append(part)
+
+    if current_heading or current_content:
+        text_parts = [current_heading] if current_heading else []
+        text_parts.extend(current_content)
+        chunk_text = "\n\n".join(text_parts)
+        if chunk_text.strip():
+            chunks.append(
+                {
+                    "text": chunk_text,
+                    "key": _slugify(current_heading) if current_heading else f"section-{len(chunks)}",
+                    "meta": {"heading": current_heading},
+                }
+            )
+
+    if not chunks and content.strip():
+        chunks.append({"text": content.strip(), "key": "full", "meta": {}})
+
+    return chunks
+
+
+def extract(
+    paths: Annotated[Optional[list[Path]], typer.Argument(help="Files or directories")] = None,
+):
+    """Extract markdown files, splitting by headings."""
+    for md_file in resolve_paths(paths or [], EXTENSIONS):
+        try:
+            chunks = _parse(md_file)
+        except Exception as e:
+            print(f"Error parsing {md_file}: {e}", file=sys.stderr)
+            continue
+
+        if not chunks:
+            continue
+
+        output(
+            {
+                "source_uri": f"file://{md_file.resolve()}",
+                "chunks": chunks,
+                "tags": ["markdown", "document"],
+                "title": md_file.stem,
+                "source_type": "markdown",
+                "context": json.dumps({"path": str(md_file)}),
+                "replace": True,
+                "created_at": get_file_mtime(md_file),
+            }
+        )

uridx-0.1.0/src/uridx/cli/extract/pdf.py

@@ -0,0 +1,62 @@
+"""Extract PDF files by page using pdfplumber."""
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from .base import output
+
+
+def extract(
+    path: Annotated[Optional[Path], typer.Argument(help="File or directory")] = None,
+):
+    """Extract PDF files by page (requires pdfplumber)."""
+    try:
+        import pdfplumber
+    except ImportError:
+        print("pdfplumber not installed. Install with: uv pip install 'uridx[pdf]'", file=sys.stderr)
+        raise typer.Exit(1)
+
+    root = path or Path.cwd()
+
+    if root.is_file():
+        files = [root]
+    else:
+        files = list(root.rglob("*.pdf"))
+
+    for pdf_file in files:
+        if not pdf_file.is_file():
+            continue
+
+        chunks = []
+        try:
+            with pdfplumber.open(pdf_file) as pdf:
+                for i, page in enumerate(pdf.pages):
+                    try:
+                        text = page.extract_text()
+                    except Exception as e:
+                        print(f"Error extracting page {i + 1} from {pdf_file}: {e}", file=sys.stderr)
+                        continue
+                    if text and text.strip():
+                        chunks.append({"text": text.strip(), "key": f"page-{i + 1}", "meta": {"page_number": i + 1}})
+        except Exception as e:
+            print(f"Error processing {pdf_file}: {e}", file=sys.stderr)
+            continue
+
+        if not chunks:
+            continue
+
+        output(
+            {
+                "source_uri": f"file://{pdf_file.resolve()}",
+                "chunks": chunks,
+                "tags": ["pdf", "document"],
+                "title": pdf_file.stem,
+                "source_type": "pdf",
+                "context": json.dumps({"path": str(pdf_file), "pages": len(chunks)}),
+                "replace": True,
+            }
+        )