knowledge-fabric 0.1.1__py3-none-any.whl

knowledge_fabric/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Knowledge Fabric package."""

knowledge_fabric/chunking/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Chunking strategies."""
+
+from knowledge_fabric.chunking.service import ChunkingConfig, DocumentChunkingService
+
+__all__ = ["ChunkingConfig", "DocumentChunkingService"]
+

knowledge_fabric/chunking/service.py

@@ -0,0 +1,304 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Structure-aware chunking service for Markdown, HTML, PDF/pages, and fallback text."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from knowledge_fabric.ingestion.models import Chunk, Document, SourceFormat
+
+_HEADING_PATTERN = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
+_CODE_BLOCK_PATTERN = re.compile(r"(```[\s\S]*?```)")
+_TABLE_PATTERN = re.compile(r"((?:^\|.*?\|\s*$\n?)+)", re.MULTILINE)
+_PAGE_MARKER_PATTERN = re.compile(r"(?:\f|---\s*Page\s+(\d+)\s*---)", re.IGNORECASE)
+
+
+@dataclass(slots=True)
+class ChunkingConfig:
+    """Configuration parameters for document chunking."""
+
+    max_chars: int = 1200
+    overlap_chars: int = 150
+    preserve_code_blocks: bool = True
+    preserve_tables: bool = True
+    min_chunk_chars: int = 30
+
+
+class DocumentChunkingService:
+    """Split ingested documents into structure-preserving chunks."""
+
+    def __init__(
+        self,
+        max_chars: int = 1200,
+        overlap_chars: int = 150,
+        config: ChunkingConfig | None = None,
+    ) -> None:
+        if config is not None:
+            self._config = config
+        else:
+            if max_chars <= 0:
+                raise ValueError("max_chars must be greater than 0")
+            if overlap_chars < 0:
+                raise ValueError("overlap_chars must be >= 0")
+            if overlap_chars >= max_chars:
+                raise ValueError("overlap_chars must be smaller than max_chars")
+            self._config = ChunkingConfig(max_chars=max_chars, overlap_chars=overlap_chars)
+
+    @property
+    def max_chars(self) -> int:
+        return self._config.max_chars
+
+    @property
+    def overlap_chars(self) -> int:
+        return self._config.overlap_chars
+
+    def chunk_document(self, document: Document) -> list[Chunk]:
+        """Chunk document content with structure-aware logic according to format."""
+        if document.source_format is SourceFormat.MARKDOWN:
+            chunks = self._chunk_markdown_structure(document)
+            if chunks:
+                return chunks
+
+        # Page-aware chunking for PDFs and presentations
+        if document.source_format in (SourceFormat.PDF, SourceFormat.PPTX):
+            page_chunks = self._chunk_page_aware(document)
+            if page_chunks:
+                return page_chunks
+
+        return self._chunk_fallback(document)
+
+    def _chunk_markdown_structure(self, document: Document) -> list[Chunk]:
+        text = document.content_text.strip()
+        matches = list(_HEADING_PATTERN.finditer(text))
+        if not matches:
+            return []
+
+        chunks: list[Chunk] = []
+        chunk_index = 0
+        heading_stack: list[tuple[int, str]] = []
+
+        for pos, match in enumerate(matches):
+            level = len(match.group(1))
+            heading_title = match.group(2).strip()
+
+            # Update heading hierarchy stack
+            while heading_stack and heading_stack[-1][0] >= level:
+                heading_stack.pop()
+            heading_stack.append((level, heading_title))
+            heading_path = " > ".join(title for _, title in heading_stack)
+
+            start = match.start()
+            end = matches[pos + 1].start() if pos + 1 < len(matches) else len(text)
+            section_text = text[start:end].strip()
+
+            if len(section_text) <= self._config.max_chars:
+                offset_start = text.find(section_text, start)
+                offset_end = offset_start + len(section_text) if offset_start >= 0 else end
+                chunk = Chunk(
+                    document_uri=document.source_uri,
+                    chunk_index=chunk_index,
+                    chunk_text=section_text,
+                    heading_path=heading_path,
+                    heading_level=level,
+                    start_offset=max(0, offset_start),
+                    end_offset=offset_end,
+                    metadata={
+                        "heading": heading_title,
+                        "heading_level": level,
+                        "heading_path": heading_path,
+                        "start_offset": max(0, offset_start),
+                        "end_offset": offset_end,
+                    },
+                )
+                chunks.append(chunk)
+                chunk_index += 1
+                continue
+
+            # Split large section into atomic blocks (headings, code blocks, tables, paragraphs)
+            blocks = self._split_section_into_blocks(section_text)
+            accumulated_text = ""
+            accumulated_start = start
+
+            for block in blocks:
+                if not accumulated_text:
+                    accumulated_text = block
+                elif len(accumulated_text) + 1 + len(block) <= self._config.max_chars:
+                    accumulated_text = f"{accumulated_text}\n\n{block}"
+                else:
+                    # Emit current accumulated chunk
+                    offset_start = text.find(accumulated_text, accumulated_start)
+                    offset_end = offset_start + len(accumulated_text) if offset_start >= 0 else accumulated_start + len(accumulated_text)
+                    chunks.append(
+                        Chunk(
+                            document_uri=document.source_uri,
+                            chunk_index=chunk_index,
+                            chunk_text=accumulated_text,
+                            heading_path=heading_path,
+                            heading_level=level,
+                            start_offset=max(0, offset_start),
+                            end_offset=offset_end,
+                            metadata={
+                                "heading": heading_title,
+                                "heading_level": level,
+                                "heading_path": heading_path,
+                                "split_from_heading_section": True,
+                                "start_offset": max(0, offset_start),
+                                "end_offset": offset_end,
+                            },
+                        )
+                    )
+                    chunk_index += 1
+                    accumulated_start = max(0, offset_end - self._config.overlap_chars)
+                    accumulated_text = block
+
+            if accumulated_text:
+                offset_start = text.find(accumulated_text, accumulated_start)
+                offset_end = offset_start + len(accumulated_text) if offset_start >= 0 else accumulated_start + len(accumulated_text)
+                chunks.append(
+                    Chunk(
+                        document_uri=document.source_uri,
+                        chunk_index=chunk_index,
+                        chunk_text=accumulated_text,
+                        heading_path=heading_path,
+                        heading_level=level,
+                        start_offset=max(0, offset_start),
+                        end_offset=offset_end,
+                        metadata={
+                            "heading": heading_title,
+                            "heading_level": level,
+                            "heading_path": heading_path,
+                            "split_from_heading_section": True,
+                            "start_offset": max(0, offset_start),
+                            "end_offset": offset_end,
+                        },
+                    )
+                )
+                chunk_index += 1
+
+        return chunks
+
+    def _split_section_into_blocks(self, section_text: str) -> list[str]:
+        """Decompose a markdown section into discrete structural blocks (paragraphs, tables, code blocks)."""
+        # Split on double newline while keeping code blocks intact
+        raw_paragraphs = [p.strip() for p in re.split(r"\n\s*\n", section_text) if p.strip()]
+        blocks: list[str] = []
+        inside_code_fence = False
+        current_fence_block: list[str] = []
+
+        for p in raw_paragraphs:
+            fence_count = p.count("```")
+            if fence_count % 2 != 0:
+                inside_code_fence = not inside_code_fence
+
+            if inside_code_fence or fence_count > 0:
+                current_fence_block.append(p)
+                if not inside_code_fence:
+                    blocks.append("\n\n".join(current_fence_block))
+                    current_fence_block = []
+            else:
+                if current_fence_block:
+                    current_fence_block.append(p)
+                    blocks.append("\n\n".join(current_fence_block))
+                    current_fence_block = []
+                else:
+                    blocks.append(p)
+
+        if current_fence_block:
+            blocks.append("\n\n".join(current_fence_block))
+
+        # Further split any single block that exceeds max_chars
+        refined: list[str] = []
+        for b in blocks:
+            if len(b) <= self._config.max_chars:
+                refined.append(b)
+            else:
+                refined.extend(self._split_with_overlap(b))
+        return refined
+
+    def _chunk_page_aware(self, document: Document) -> list[Chunk]:
+        """Detect form-feed or slide delimiters and emit page-attributed chunks."""
+        text = document.content_text.strip()
+        pages = _PAGE_MARKER_PATTERN.split(text)
+        if len(pages) <= 1:
+            return []
+
+        chunks: list[Chunk] = []
+        chunk_index = 0
+        page_num = 1
+
+        for part in pages:
+            if not part:
+                continue
+            if part.isdigit():
+                page_num = int(part)
+                continue
+
+            page_text = part.strip()
+            if not page_text:
+                continue
+
+            splits = self._split_with_overlap(page_text)
+            for split in splits:
+                chunks.append(
+                    Chunk(
+                        document_uri=document.source_uri,
+                        chunk_index=chunk_index,
+                        chunk_text=split,
+                        metadata={
+                            "page_number": page_num,
+                            "strategy": "page_aware",
+                        },
+                    )
+                )
+                chunk_index += 1
+            page_num += 1
+
+        return chunks
+
+    def _chunk_fallback(self, document: Document) -> list[Chunk]:
+        """Fallback character window chunking for unstructured plain text."""
+        text = document.content_text.strip()
+        splits = self._split_with_overlap(text)
+        chunks: list[Chunk] = []
+        current_pos = 0
+
+        for index, chunk_text in enumerate(splits):
+            start = text.find(chunk_text, current_pos)
+            end = start + len(chunk_text) if start >= 0 else current_pos + len(chunk_text)
+            current_pos = max(0, end - self._config.overlap_chars)
+            chunks.append(
+                Chunk(
+                    document_uri=document.source_uri,
+                    chunk_index=index,
+                    chunk_text=chunk_text,
+                    start_offset=max(0, start),
+                    end_offset=end,
+                    metadata={
+                        "strategy": "fallback_window",
+                        "start_offset": max(0, start),
+                        "end_offset": end,
+                    },
+                )
+            )
+        return chunks
+
+    def _split_with_overlap(self, text: str) -> list[str]:
+        if not text:
+            return []
+        if len(text) <= self._config.max_chars:
+            return [text]
+
+        chunks: list[str] = []
+        start = 0
+        while start < len(text):
+            end = min(start + self._config.max_chars, len(text))
+            chunk = text[start:end].strip()
+            if chunk:
+                chunks.append(chunk)
+            if end == len(text):
+                break
+            start = max(0, end - self._config.overlap_chars)
+        return chunks

knowledge_fabric/config.py

@@ -0,0 +1,174 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Configuration loader for Knowledge Fabric runtime components."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import yaml
+
+
+
+@dataclass(slots=True)
+class DatabaseSettings:
+    host: str
+    port: int
+    name: str
+    user: str
+    password: str
+
+
+@dataclass(slots=True)
+class TikaSettings:
+    endpoint: str
+    request_timeout_seconds: int
+
+
+@dataclass(slots=True)
+class EmbeddingSettings:
+    provider: str
+    dimension: int
+    batch_size: int
+
+
+@dataclass(slots=True)
+class RetrievalSettings:
+    default_top_k: int
+    lexical_weight: float
+    vector_weight: float
+    rrf_k: int
+    backend: str = "postgres"
+
+
+@dataclass(slots=True)
+class RerankingSettings:
+    provider: str = "none"
+    top_n: int = 5
+
+
+@dataclass(slots=True)
+class AppSettings:
+    name: str
+    environment: str
+    log_level: str
+
+
+@dataclass(slots=True)
+class Settings:
+    app: AppSettings
+    database: DatabaseSettings
+    tika: TikaSettings
+    embeddings: EmbeddingSettings
+    retrieval: RetrievalSettings
+    reranking: RerankingSettings
+
+
+def load_settings(path: str | Path = "config/settings.yaml") -> Settings:
+    config_file = Path(path)
+    if config_file.exists():
+        raw = yaml.safe_load(config_file.read_text(encoding="utf-8")) or {}
+    else:
+        raw = {}
+
+    app_raw = raw.get("app") or {}
+    db_raw = raw.get("database") or {}
+    tika_raw = raw.get("tika") or {}
+    emb_raw = raw.get("embeddings") or {}
+    retrieval_raw = raw.get("retrieval") or {}
+    reranking_raw = raw.get("reranking") or {}
+
+    # Database env overrides
+    db_url = os.environ.get("DATABASE_URL")
+    if db_url:
+        parsed = urlparse(db_url)
+        if parsed.hostname:
+            db_raw["host"] = parsed.hostname
+        if parsed.port:
+            db_raw["port"] = parsed.port
+        if parsed.path and parsed.path.lstrip("/"):
+            db_raw["name"] = parsed.path.lstrip("/")
+        if parsed.username:
+            db_raw["user"] = parsed.username
+        if parsed.password is not None:
+            db_raw["password"] = parsed.password
+
+    if os.environ.get("KF_DB_HOST") or os.environ.get("POSTGRES_HOST"):
+        db_raw["host"] = os.environ.get("KF_DB_HOST") or os.environ.get("POSTGRES_HOST")
+    if os.environ.get("KF_DB_PORT") or os.environ.get("POSTGRES_PORT"):
+        db_raw["port"] = int(os.environ.get("KF_DB_PORT") or os.environ.get("POSTGRES_PORT") or 5432)
+    if os.environ.get("KF_DB_NAME") or os.environ.get("POSTGRES_DB"):
+        db_raw["name"] = os.environ.get("KF_DB_NAME") or os.environ.get("POSTGRES_DB")
+    if os.environ.get("KF_DB_USER"):
+        db_raw["user"] = os.environ.get("KF_DB_USER")
+    elif os.environ.get("POSTGRES_USER"):
+        db_raw["user"] = os.environ.get("POSTGRES_USER")
+    if "KF_DB_PASSWORD" in os.environ:
+        db_raw["password"] = os.environ.get("KF_DB_PASSWORD")
+    elif "POSTGRES_PASSWORD" in os.environ:
+        db_raw["password"] = os.environ.get("POSTGRES_PASSWORD", "")
+
+    # Tika env overrides
+    if os.environ.get("TIKA_ENDPOINT"):
+        tika_raw["endpoint"] = os.environ["TIKA_ENDPOINT"]
+
+    # Embedding env overrides
+    if os.environ.get("EMBEDDING_PROVIDER"):
+        emb_raw["provider"] = os.environ["EMBEDDING_PROVIDER"]
+    if os.environ.get("EMBEDDING_DIMENSION"):
+        emb_raw["dimension"] = int(os.environ["EMBEDDING_DIMENSION"])
+
+    # Retrieval env overrides
+    if os.environ.get("RETRIEVAL_STORE_BACKEND") or os.environ.get("RETRIEVAL_BACKEND"):
+        retrieval_raw["backend"] = os.environ.get("RETRIEVAL_STORE_BACKEND") or os.environ.get("RETRIEVAL_BACKEND")
+
+    # Reranking env overrides
+    if os.environ.get("RERANKER") or os.environ.get("RERANKING_PROVIDER"):
+        reranking_raw["provider"] = os.environ.get("RERANKER") or os.environ.get("RERANKING_PROVIDER")
+
+    return Settings(
+        app=AppSettings(
+            name=str(app_raw.get("name", "knowledge-fabric")),
+            environment=str(app_raw.get("environment", "local")),
+            log_level=str(app_raw.get("log_level", "INFO")),
+        ),
+        database=DatabaseSettings(
+            host=str(db_raw.get("host", "localhost")),
+            port=int(db_raw.get("port", 5432)),
+            name=str(db_raw.get("name", "knowledge_fabric")),
+            user=str(db_raw.get("user", "kf_app")),
+            password=str(db_raw.get("password", "kf_app_password")),
+        ),
+        tika=TikaSettings(
+            endpoint=str(tika_raw.get("endpoint", "http://localhost:9998")),
+            request_timeout_seconds=int(tika_raw.get("request_timeout_seconds", 30)),
+        ),
+        embeddings=EmbeddingSettings(
+            provider=str(emb_raw.get("provider", "mock")),
+            dimension=int(emb_raw.get("dimension", 1536)),
+            batch_size=int(emb_raw.get("batch_size", 64)),
+        ),
+        retrieval=RetrievalSettings(
+            default_top_k=int(retrieval_raw.get("default_top_k", 10)),
+            lexical_weight=float(retrieval_raw.get("lexical_weight", 0.5)),
+            vector_weight=float(retrieval_raw.get("vector_weight", 0.5)),
+            rrf_k=int(retrieval_raw.get("rrf_k", 60)),
+            backend=str(retrieval_raw.get("backend", "postgres")),
+        ),
+        reranking=RerankingSettings(
+            provider=str(reranking_raw.get("provider", "none")),
+            top_n=int(reranking_raw.get("top_n", 5)),
+        ),
+    )
+
+
+def _as_dict(raw: dict[str, Any], key: str) -> dict[str, Any]:
+    value = raw.get(key)
+    if not isinstance(value, dict):
+        raise ValueError(f"Expected mapping for '{key}'")
+    return value

knowledge_fabric/db/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Database access and schema bootstrap."""
+
+from knowledge_fabric.db.audit import AuditLogger
+from knowledge_fabric.db.connection import create_postgres_connection_factory
+from knowledge_fabric.db.dimension_guard import DimensionMismatchError, check_embedding_dimension
+from knowledge_fabric.db.repository import KnowledgeRepository
+
+__all__ = [
+    "AuditLogger",
+    "DimensionMismatchError",
+    "KnowledgeRepository",
+    "check_embedding_dimension",
+    "create_postgres_connection_factory",
+]

knowledge_fabric/db/audit.py

@@ -0,0 +1,59 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Audit and retrieval run logging."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+ConnectionFactory = Callable[[], Any]
+
+
+class AuditLogger:
+    """Writes retrieval telemetry into `retrieval_runs` and `audit_events`."""
+
+    def __init__(self, connection_factory: ConnectionFactory) -> None:
+        self._connection_factory = connection_factory
+
+    def log_retrieval(
+        self,
+        *,
+        query_text: str,
+        top_k: int,
+        retrieval_mode: str,
+        result_count: int,
+        latency_ms: int,
+        trace_id: str | None = None,
+        tenant_id: str | None = None,
+        details: dict[str, object] | None = None,
+    ) -> None:
+        effective_tenant = tenant_id or "default"
+        details_payload = details or {}
+        connection = self._connection_factory()
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT set_config('app.tenant_id', %s, true)", [effective_tenant])
+            cursor.execute(
+                """
+                INSERT INTO retrieval_runs (tenant_id, query_text, top_k, retrieval_mode, filters, latency_ms, result_count)
+                VALUES (%s, %s, %s, %s, %s::jsonb, %s, %s)
+                """,
+                [effective_tenant, query_text, top_k, retrieval_mode, "{}", latency_ms, result_count],
+            )
+            cursor.execute(
+                """
+                INSERT INTO audit_events (tenant_id, event_type, actor, trace_id, event_payload)
+                VALUES (%s, %s, %s, %s, %s::jsonb)
+                """,
+                [effective_tenant, "retrieval.executed", "system", trace_id, _json_dump(details_payload)],
+            )
+            if hasattr(connection, "commit"):
+                connection.commit()
+
+
+def _json_dump(payload: dict[str, object]) -> str:
+    # Late import keeps baseline runtime dependency minimal unless logging is used.
+    import json
+
+    return json.dumps(payload, separators=(",", ":"), sort_keys=True)

knowledge_fabric/db/connection.py

@@ -0,0 +1,85 @@
+# Copyright 2026 Invarcore Organization
+# SPDX-License-Identifier: Apache-2.0
+
+"""Database connection helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+import os
+from typing import Any
+
+from knowledge_fabric.config import DatabaseSettings
+
+ConnectionFactory = Callable[[], Any]
+
+# Prohibited superuser usernames that bypass PostgreSQL Row-Level Security (RLS)
+PROHIBITED_SUPERUSER_ROLES = frozenset({"postgres", "root", "superuser", "admin"})
+
+
+def create_postgres_connection_factory(
+    settings: DatabaseSettings,
+    *,
+    enforce_non_superuser: bool | None = None,
+) -> ConnectionFactory:
+    """Create a lazy psycopg connection factory from database settings.
+
+    Restricts PostgreSQL connection roles to least-privilege tenant roles rather
+    than superuser to prevent Row-Level Security (RLS) bypass (KF-01).
+    """
+
+    def _connect() -> Any:
+        import psycopg
+
+        # Determine non-superuser enforcement
+        should_enforce = enforce_non_superuser
+        if should_enforce is None:
+            should_enforce = os.environ.get("KNOWLEDGE_ENFORCE_NON_SUPERUSER", "true").lower() not in (
+                "false",
+                "0",
+                "off",
+                "no",
+            )
+
+        if should_enforce and settings.user.lower() in PROHIBITED_SUPERUSER_ROLES:
+            raise PermissionError(
+                f"PostgreSQL connection role '{settings.user}' is a prohibited superuser role. "
+                "Knowledge Fabric requires a dedicated least-privilege tenant role (e.g., 'kf_app') "
+                "to ensure Row-Level Security (RLS) policies are enforced (KF-01). "
+                "Set KNOWLEDGE_ENFORCE_NON_SUPERUSER=false to override for migrations/admin tooling."
+            )
+
+        connect_timeout = int(os.environ.get("KNOWLEDGE_DB_CONNECT_TIMEOUT", "2"))
+        conn = psycopg.connect(
+            host=settings.host,
+            port=settings.port,
+            dbname=settings.name,
+            user=settings.user,
+            password=settings.password,
+            connect_timeout=connect_timeout,
+            autocommit=False,
+        )
+
+        if should_enforce:
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SHOW is_superuser")
+                    row = cur.fetchone()
+                    if row and str(row[0]).lower() in ("on", "true", "1", "yes"):
+                        conn.close()
+                        raise PermissionError(
+                            f"PostgreSQL connection role '{settings.user}' has superuser privileges (is_superuser=on). "
+                            "Knowledge Fabric requires a dedicated least-privilege tenant role (e.g., 'kf_app') "
+                            "to ensure Row-Level Security (RLS) policies are enforced (KF-01). "
+                            "Set KNOWLEDGE_ENFORCE_NON_SUPERUSER=false to override for migrations/admin tooling."
+                        )
+            except PermissionError:
+                raise
+            except Exception:
+                # If SHOW is_superuser is unsupported or fails for an unexpected non-permission reason, do not block
+                pass
+
+        return conn
+
+    return _connect
+