okb 1.1.0__py3-none-any.whl → 1.1.0a0__py3-none-any.whl

okb/llm/extractors/entity.py

@@ -1,369 +0,0 @@
-"""Entity extraction from document content using LLM."""
-
-from __future__ import annotations
-
-import json
-import re
-
-from .base import ExtractedEntity
-
-ENTITY_SYSTEM_PROMPT = """\
-You are an expert at identifying named entities in text for a PERSONAL knowledge base.
-Extract only entities that are specific to the author's context - things an LLM wouldn't know about.
-
-Entity types to extract:
-- person: People the author knows, works with, or references (colleagues, contacts, clients)
-- project: Specific named projects/products/codebases (e.g., "Acme Dashboard", "customer-portal")
-          NOT git branches, environments, or workflow stages
-- technology: ONLY obscure/niche tools or internal systems - NOT well-known technologies
-- organization: Specific companies, teams, clients the author works with
-
-DO NOT extract:
-- Well-known technologies: JSON, HTTP, SQL, Python, JavaScript, Docker, AWS, PostgreSQL, React, etc.
-  (The LLM already knows these - they add no value to a personal knowledge base)
-- Code symbols: function names, method calls, variables, class names
-- Generic terms: "user", "data", "system", "database", "API", "server", "client"
-- Git branches/workflow terms: main, master, develop, release, staging, production, feature, hotfix
-- Generic process terms: deploy, build, test, migration, setup, config
-- Environment names: dev, prod, qa, uat, local
-- Issue or bug descriptions - those are documents, not entities
-- Famous people, major companies (Google, Microsoft, etc.) unless contextually relevant to author
-
-ONLY extract entities that would help answer "Who/what is X?" where X is specific to this person.
-
-For each entity found, extract:
-- name: The canonical name (proper noun)
-- entity_type: One of: person, project, technology, organization
-- aliases: Other names/abbreviations (optional)
-- description: Brief description based on context (optional)
-- mentions: Text snippets where entity appears (max 3)
-- confidence: How confident you are (0.0-1.0)
-
-Return JSON array. Return empty array [] if no context-specific entities found.
-"""
-
-ENTITY_USER_PROMPT = """\
-Document title: {title}
-Source type: {source_type}
-
-Content:
-{content}
-
-Extract named entities as JSON array.
-"""
-
-
-def extract_entities(
-    content: str,
-    title: str,
-    source_type: str | None = None,
-    min_confidence: float = 0.8,
-) -> list[ExtractedEntity]:
-    """Extract entities from document content using LLM.
-
-    Args:
-        content: Document content to analyze
-        title: Document title for context
-        source_type: Type of document (optional)
-        min_confidence: Minimum confidence threshold (0-1)
-
-    Returns:
-        List of extracted entities
-    """
-    from .. import complete
-
-    # Truncate content if too long
-    if len(content) > 20000:
-        content = content[:20000] + "\n\n[... content truncated ...]"
-
-    prompt = ENTITY_USER_PROMPT.format(
-        title=title,
-        source_type=source_type or "unknown",
-        content=content,
-    )
-
-    response = complete(
-        prompt=prompt,
-        system=ENTITY_SYSTEM_PROMPT,
-        max_tokens=2048,
-        use_cache=True,
-    )
-
-    if response is None:
-        return []
-
-    return _parse_entity_response(response.content, min_confidence, title)
-
-
-def _looks_like_code(name: str) -> bool:
-    """Check if entity name looks like code."""
-    # Contains parentheses (function calls)
-    if "(" in name or ")" in name:
-        return True
-    # Snake_case with underscores (likely variable/function)
-    if "_" in name and name.islower():
-        return True
-    # Starts with lowercase and contains dots (method chain)
-    if name and name[0].islower() and "." in name:
-        return True
-    # CamelCase starting with lowercase (variable/method name)
-    if name and name[0].islower() and any(c.isupper() for c in name):
-        return True
-    return False
-
-
-# Well-known technologies that add no value to a personal knowledge base
-COMMON_TECHNOLOGIES = frozenset(
-    s.lower()
-    for s in [
-        # Data formats
-        "JSON",
-        "XML",
-        "YAML",
-        "CSV",
-        "HTML",
-        "CSS",
-        "Markdown",
-        # Protocols
-        "HTTP",
-        "HTTPS",
-        "REST",
-        "GraphQL",
-        "WebSocket",
-        "TCP",
-        "UDP",
-        "SSH",
-        "FTP",
-        "SMTP",
-        # Languages
-        "Python",
-        "JavaScript",
-        "TypeScript",
-        "Java",
-        "Go",
-        "Rust",
-        "C",
-        "C++",
-        "Ruby",
-        "PHP",
-        "Swift",
-        "Kotlin",
-        "Scala",
-        "Bash",
-        "Shell",
-        "SQL",
-        "Lua",
-        # Major frameworks/tools
-        "React",
-        "Vue",
-        "Angular",
-        "Node.js",
-        "Django",
-        "Flask",
-        "FastAPI",
-        "Rails",
-        "Spring",
-        "Express",
-        "Next.js",
-        # Databases
-        "PostgreSQL",
-        "MySQL",
-        "MongoDB",
-        "Redis",
-        "SQLite",
-        "Elasticsearch",
-        "DynamoDB",
-        # Cloud/infra
-        "AWS",
-        "Azure",
-        "GCP",
-        "Docker",
-        "Kubernetes",
-        "Linux",
-        "Windows",
-        "macOS",
-        "Nginx",
-        "Apache",
-        # Tools
-        "Git",
-        "GitHub",
-        "GitLab",
-        "npm",
-        "pip",
-        "Webpack",
-        "VS Code",
-        "Vim",
-        "Emacs",
-    ]
-)
-
-# Generic git/workflow/environment terms that are not context-specific
-GENERIC_TERMS = frozenset(
-    s.lower()
-    for s in [
-        # Git branches
-        "main",
-        "master",
-        "develop",
-        "development",
-        "release",
-        "staging",
-        "production",
-        "feature",
-        "hotfix",
-        "bugfix",
-        # Environments
-        "dev",
-        "prod",
-        "test",
-        "qa",
-        "uat",
-        "local",
-        "sandbox",
-        # Workflow/process terms
-        "deploy",
-        "build",
-        "migration",
-        "setup",
-        "config",
-        "configuration",
-        "rollback",
-        "rollout",
-        # Generic architectural terms
-        "frontend",
-        "backend",
-        "api",
-        "service",
-        "server",
-        "client",
-        "app",
-        "application",
-        "module",
-        "component",
-        "library",
-        "package",
-        "plugin",
-        "extension",
-        # Generic data terms
-        "database",
-        "cache",
-        "queue",
-        "worker",
-        "scheduler",
-        "cron",
-    ]
-)
-
-
-def _parse_entity_response(
-    response_text: str, min_confidence: float, title: str = ""
-) -> list[ExtractedEntity]:
-    """Parse LLM response into ExtractedEntity objects."""
-    # Try to extract JSON from response
-    json_match = re.search(r"\[.*\]", response_text, re.DOTALL)
-    if not json_match:
-        return []
-
-    try:
-        entities_data = json.loads(json_match.group())
-    except json.JSONDecodeError:
-        return []
-
-    if not isinstance(entities_data, list):
-        return []
-
-    valid_types = {"person", "project", "technology", "organization"}
-    entities = []
-
-    for item in entities_data:
-        if not isinstance(item, dict):
-            continue
-
-        name = item.get("name", "").strip()
-        entity_type = item.get("entity_type")
-
-        if not name or not isinstance(name, str):
-            continue
-        if not entity_type or entity_type not in valid_types:
-            continue
-
-        # Filter: too short or too long
-        if len(name) < 3 or len(name) > 80:
-            continue
-
-        # Filter: looks like code
-        if _looks_like_code(name):
-            continue
-
-        # Filter: well-known technologies (LLM already knows these)
-        if name.lower() in COMMON_TECHNOLOGIES:
-            continue
-
-        # Filter: generic git/workflow/environment terms
-        if name.lower() in GENERIC_TERMS:
-            continue
-
-        # Filter: matches document title (source shouldn't be extracted as entity)
-        if title and name.lower() == title.lower():
-            continue
-
-        # Get confidence (default to 0.85 if not specified)
-        confidence = item.get("confidence", 0.85)
-        if not isinstance(confidence, int | float):
-            confidence = 0.85
-
-        if confidence < min_confidence:
-            continue
-
-        # Parse aliases
-        aliases = item.get("aliases", [])
-        if not isinstance(aliases, list):
-            aliases = []
-        aliases = [a for a in aliases if isinstance(a, str)]
-
-        # Parse mentions
-        mentions = item.get("mentions", [])
-        if not isinstance(mentions, list):
-            mentions = []
-        mentions = [m for m in mentions if isinstance(m, str)][:3]
-
-        entities.append(
-            ExtractedEntity(
-                name=name,
-                entity_type=entity_type,
-                aliases=aliases,
-                description=item.get("description"),
-                mentions=mentions,
-                confidence=float(confidence),
-            )
-        )
-
-    return entities
-
-
-def normalize_entity_name(name: str) -> str:
-    """Normalize entity name for deduplication and URL generation.
-
-    Examples:
-        "John Smith" -> "john-smith"
-        "AWS (Amazon Web Services)" -> "aws-amazon-web-services"
-        "React.js" -> "react-js"
-    """
-    # Lowercase
-    normalized = name.lower()
-    # Replace non-alphanumeric with spaces
-    normalized = re.sub(r"[^a-z0-9\s]", " ", normalized)
-    # Collapse whitespace and replace with hyphens
-    normalized = re.sub(r"\s+", "-", normalized.strip())
-    # Remove leading/trailing hyphens
-    normalized = normalized.strip("-")
-    return normalized
-
-
-def entity_source_path(entity_type: str, name: str) -> str:
-    """Generate source_path for an entity document.
-
-    Format: okb://entity/{type}/{normalized-name}
-    """
-    normalized = normalize_entity_name(name)
-    return f"okb://entity/{entity_type}/{normalized}"

okb/llm/extractors/todo.py

@@ -1,149 +0,0 @@
-"""TODO extraction from document content using LLM."""
-
-from __future__ import annotations
-
-import json
-import re
-from datetime import UTC, datetime
-
-from .base import ExtractedTodo
-
-TODO_SYSTEM_PROMPT = """\
-You are an expert at identifying action items and tasks in text.
-Extract TODO items from the given document content.
-
-Look for:
-- Explicit markers: TODO, FIXME, HACK, XXX, ACTION
-- Action phrases: "need to", "should", "must", "have to", "action item"
-- Deadlines and commitments: "by Friday", "before the meeting", "this week"
-- Questions implying needed work: "What about X?", "How do we handle Y?"
-- Incomplete items marked for follow-up
-
-For each TODO found, extract:
-- title: A concise description of the task (imperative form: "Fix the bug", not "The bug needs fixing")
-- content: Additional context or details (optional)
-- due_date: If a deadline is mentioned, in ISO format YYYY-MM-DD (optional)
-- priority: 1=urgent, 2=high, 3=normal, 4=low, 5=someday (optional)
-- assignee: Person responsible if mentioned (optional)
-- source_context: The exact text snippet where this TODO was found
-
-Return JSON array of extracted TODOs. Return empty array [] if none found.
-Be conservative - only extract clear action items, not vague mentions.
-"""
-
-TODO_USER_PROMPT = """\
-Document title: {title}
-Source type: {source_type}
-
-Content:
-{content}
-
-Extract all TODO items from this content as JSON array.
-"""
-
-
-def extract_todos(
-    content: str,
-    title: str,
-    source_type: str,
-    min_confidence: float = 0.7,
-) -> list[ExtractedTodo]:
-    """Extract TODO items from document content using LLM.
-
-    Args:
-        content: Document content to analyze
-        title: Document title for context
-        source_type: Type of document (markdown, code, org, etc.)
-        min_confidence: Minimum confidence threshold (0-1)
-
-    Returns:
-        List of extracted TODO items
-    """
-    from .. import complete
-
-    # Truncate content if too long (keep first ~20k chars for context)
-    if len(content) > 20000:
-        content = content[:20000] + "\n\n[... content truncated ...]"
-
-    prompt = TODO_USER_PROMPT.format(
-        title=title,
-        source_type=source_type,
-        content=content,
-    )
-
-    response = complete(
-        prompt=prompt,
-        system=TODO_SYSTEM_PROMPT,
-        max_tokens=2048,
-        use_cache=True,
-    )
-
-    if response is None:
-        return []
-
-    return _parse_todo_response(response.content, min_confidence)
-
-
-def _parse_todo_response(response_text: str, min_confidence: float) -> list[ExtractedTodo]:
-    """Parse LLM response into ExtractedTodo objects."""
-    # Try to extract JSON from response
-    json_match = re.search(r"\[.*\]", response_text, re.DOTALL)
-    if not json_match:
-        return []
-
-    try:
-        todos_data = json.loads(json_match.group())
-    except json.JSONDecodeError:
-        return []
-
-    if not isinstance(todos_data, list):
-        return []
-
-    todos = []
-    for item in todos_data:
-        if not isinstance(item, dict):
-            continue
-
-        title = item.get("title")
-        if not title or not isinstance(title, str):
-            continue
-
-        # Parse due_date if present
-        due_date = None
-        if due_str := item.get("due_date"):
-            try:
-                due_date = datetime.fromisoformat(due_str).replace(tzinfo=UTC)
-            except (ValueError, TypeError):
-                pass
-
-        # Parse priority
-        priority = None
-        if p := item.get("priority"):
-            try:
-                priority = int(p)
-                if priority < 1 or priority > 5:
-                    priority = None
-            except (ValueError, TypeError):
-                pass
-
-        # Get confidence (default to 0.8 if not specified)
-        confidence = item.get("confidence", 0.8)
-        if not isinstance(confidence, (int, float)):
-            confidence = 0.8
-
-        if confidence < min_confidence:
-            continue
-
-        todos.append(
-            ExtractedTodo(
-                title=title.strip(),
-                content=item.get("content"),
-                due_date=due_date,
-                priority=priority,
-                assignee=item.get("assignee"),
-                confidence=float(confidence),
-                source_context=item.get("source_context"),
-            )
-        )
-
-    return todos

okb/migrations/0008.enrichment.sql

@@ -1,46 +0,0 @@
--- LLM enrichment for document annotation (TODOs and entities)
--- depends: 0006.llm-cache
-
--- Track enrichment state on documents
-ALTER TABLE documents ADD COLUMN IF NOT EXISTS enriched_at TIMESTAMPTZ;
-ALTER TABLE documents ADD COLUMN IF NOT EXISTS enrichment_version INTEGER;
-
--- Index for "needs enrichment" queries
-CREATE INDEX IF NOT EXISTS idx_documents_needs_enrichment
-    ON documents(enriched_at) WHERE enriched_at IS NULL;
-
--- Pending entity suggestions (before approval)
-CREATE TABLE IF NOT EXISTS pending_entities (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    source_document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    entity_name TEXT NOT NULL,
-    entity_type TEXT NOT NULL,  -- person, project, technology, concept, organization
-    aliases JSONB DEFAULT '[]',
-    description TEXT,
-    mentions JSONB DEFAULT '[]',  -- Context snippets from source document
-    confidence REAL,
-    status TEXT DEFAULT 'pending',  -- pending, approved, rejected
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    reviewed_at TIMESTAMPTZ
-);
-
-CREATE INDEX IF NOT EXISTS idx_pending_entities_status ON pending_entities(status);
-CREATE INDEX IF NOT EXISTS idx_pending_entities_source ON pending_entities(source_document_id);
-CREATE INDEX IF NOT EXISTS idx_pending_entities_type ON pending_entities(entity_type);
-
--- Entity references (links entity documents to source documents)
--- When an entity is approved, it becomes a document with source_path like okb://entity/person/john-smith
--- This table tracks which documents mention each entity
-CREATE TABLE IF NOT EXISTS entity_refs (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    entity_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    mention_text TEXT NOT NULL,
-    context TEXT,  -- Surrounding text for context
-    confidence REAL,
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(entity_id, document_id, mention_text)
-);
-
-CREATE INDEX IF NOT EXISTS idx_entity_refs_entity ON entity_refs(entity_id);
-CREATE INDEX IF NOT EXISTS idx_entity_refs_document ON entity_refs(document_id);

okb/migrations/0009.entity-consolidation.sql

@@ -1,120 +0,0 @@
--- Entity consolidation: deduplication, cross-doc detection, clustering, relationships
--- depends: 0008.enrichment
-
--- Canonical mappings: alias text -> entity document
--- Used for deduplication and alias resolution
-CREATE TABLE IF NOT EXISTS entity_aliases (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    alias_text TEXT NOT NULL,
-    entity_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    confidence REAL,  -- How confident we are this alias belongs to entity
-    source TEXT DEFAULT 'manual',  -- 'manual', 'merge', 'extraction'
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(alias_text, entity_id)
-);
-
-CREATE INDEX IF NOT EXISTS idx_entity_aliases_text ON entity_aliases(LOWER(alias_text));
-CREATE INDEX IF NOT EXISTS idx_entity_aliases_entity ON entity_aliases(entity_id);
-
--- Proposed entity merges awaiting user confirmation
-CREATE TABLE IF NOT EXISTS pending_entity_merges (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    canonical_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    duplicate_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    confidence REAL NOT NULL,  -- How confident we are these are the same
-    reason TEXT,  -- Why we think they're the same ("embedding_similarity", "alias_match", "llm")
-    detected_at TIMESTAMPTZ DEFAULT NOW(),
-    status TEXT DEFAULT 'pending',  -- 'pending', 'approved', 'rejected'
-    reviewed_at TIMESTAMPTZ,
-    UNIQUE(canonical_id, duplicate_id)
-);
-
-CREATE INDEX IF NOT EXISTS idx_pending_merges_status ON pending_entity_merges(status);
-CREATE INDEX IF NOT EXISTS idx_pending_merges_confidence ON pending_entity_merges(confidence DESC);
-
--- Entity-to-entity relationships
-CREATE TABLE IF NOT EXISTS entity_relationships (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    source_entity_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    target_entity_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    relationship_type TEXT NOT NULL,  -- 'works_for', 'uses', 'belongs_to', 'related_to'
-    confidence REAL,
-    source TEXT DEFAULT 'extraction',  -- 'extraction', 'manual'
-    context TEXT,  -- Supporting context for the relationship
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(source_entity_id, target_entity_id, relationship_type)
-);
-
-CREATE INDEX IF NOT EXISTS idx_entity_rel_source ON entity_relationships(source_entity_id);
-CREATE INDEX IF NOT EXISTS idx_entity_rel_target ON entity_relationships(target_entity_id);
-CREATE INDEX IF NOT EXISTS idx_entity_rel_type ON entity_relationships(relationship_type);
-
--- Topic clusters group related entities and documents
-CREATE TABLE IF NOT EXISTS topic_clusters (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    name TEXT NOT NULL,
-    description TEXT,
-    centroid vector(768),  -- Cluster centroid embedding
-    member_count INTEGER DEFAULT 0,
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    updated_at TIMESTAMPTZ DEFAULT NOW()
-);
-
-CREATE INDEX IF NOT EXISTS idx_topic_clusters_centroid ON topic_clusters
-    USING hnsw (centroid vector_cosine_ops);
-
--- Cluster membership: entities and documents can belong to clusters
-CREATE TABLE IF NOT EXISTS topic_cluster_members (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    cluster_id UUID NOT NULL REFERENCES topic_clusters(id) ON DELETE CASCADE,
-    document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-    distance REAL,  -- Distance from cluster centroid
-    is_entity BOOLEAN DEFAULT FALSE,  -- True if document is an entity
-    added_at TIMESTAMPTZ DEFAULT NOW(),
-    UNIQUE(cluster_id, document_id)
-);
-
-CREATE INDEX IF NOT EXISTS idx_cluster_members_cluster ON topic_cluster_members(cluster_id);
-CREATE INDEX IF NOT EXISTS idx_cluster_members_document ON topic_cluster_members(document_id);
-
--- Proposed cluster merges awaiting confirmation
-CREATE TABLE IF NOT EXISTS pending_cluster_merges (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    primary_cluster_id UUID NOT NULL REFERENCES topic_clusters(id) ON DELETE CASCADE,
-    secondary_cluster_id UUID NOT NULL REFERENCES topic_clusters(id) ON DELETE CASCADE,
-    similarity REAL NOT NULL,  -- How similar the clusters are
-    status TEXT DEFAULT 'pending',  -- 'pending', 'approved', 'rejected'
-    detected_at TIMESTAMPTZ DEFAULT NOW(),
-    reviewed_at TIMESTAMPTZ,
-    UNIQUE(primary_cluster_id, secondary_cluster_id)
-);
-
-CREATE INDEX IF NOT EXISTS idx_pending_cluster_merges_status ON pending_cluster_merges(status);
-
--- Cross-document entity candidates: detected mentions not yet extracted as entities
-CREATE TABLE IF NOT EXISTS cross_doc_entity_candidates (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    text TEXT NOT NULL,  -- The mention text (normalized)
-    document_ids UUID[] NOT NULL,  -- Array of document IDs containing this mention
-    document_count INTEGER NOT NULL,  -- Number of documents (for quick filtering)
-    sample_contexts JSONB DEFAULT '[]',  -- Sample text contexts where it appears
-    suggested_type TEXT,  -- Suggested entity type
-    confidence REAL,
-    status TEXT DEFAULT 'pending',  -- 'pending', 'approved', 'rejected', 'exists'
-    created_at TIMESTAMPTZ DEFAULT NOW(),
-    reviewed_at TIMESTAMPTZ,
-    UNIQUE(text)
-);
-
-CREATE INDEX IF NOT EXISTS idx_cross_doc_status ON cross_doc_entity_candidates(status);
-CREATE INDEX IF NOT EXISTS idx_cross_doc_count ON cross_doc_entity_candidates(document_count DESC);
-
--- Track consolidation runs
-CREATE TABLE IF NOT EXISTS consolidation_runs (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    run_type TEXT NOT NULL,  -- 'dedup', 'cross_doc', 'cluster', 'relationship', 'full'
-    started_at TIMESTAMPTZ DEFAULT NOW(),
-    completed_at TIMESTAMPTZ,
-    stats JSONB DEFAULT '{}',  -- Run statistics
-    error TEXT  -- Error message if failed
-);

okb/migrations/0010.token-id.sql

@@ -1,7 +0,0 @@
--- Add ID column to tokens for easier revocation
--- depends: 0009.entity-consolidation
-
-ALTER TABLE tokens ADD COLUMN IF NOT EXISTS id SERIAL;
-
--- Create index for ID lookups
-CREATE INDEX IF NOT EXISTS tokens_id_idx ON tokens(id);