docs-kit 0.1.1__py3-none-any.whl

docs_kit/connectors/embeddings/fastembed.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from fastembed import SparseTextEmbedding, TextEmbedding
+from qdrant_client.models import SparseVector
+
+
+class FastEmbedDenseEmbedding:
+    """Local dense embeddings via fastembed — no API key required."""
+
+    def __init__(self, model: str = "BAAI/bge-small-en-v1.5"):
+        self._embedder = TextEmbedding(model_name=model)
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return [embedding.tolist() for embedding in self._embedder.embed(texts)]
+
+
+class FastEmbedSparseEmbedding:
+    """Sparse BM25 embeddings via fastembed."""
+
+    def __init__(self, model: str = "Qdrant/bm25"):
+        self._embedder = SparseTextEmbedding(model_name=model)
+
+    def embed(self, texts: list[str]) -> list[SparseVector]:
+        results = list(self._embedder.embed(texts))
+        sparse_vectors: list[SparseVector] = []
+        for result in results:
+            indices = result.indices.tolist() if hasattr(result.indices, "tolist") else list(result.indices)
+            values = result.values.tolist() if hasattr(result.values, "tolist") else list(result.values)
+            sparse_vectors.append(SparseVector(indices=indices, values=values))
+        return sparse_vectors

docs_kit/connectors/fetchers/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from typing import Protocol, runtime_checkable
+from docs_kit.core.models import Document
+
+
+@runtime_checkable
+class Fetcher(Protocol):
+    def fetch(self, url: str) -> list[Document]: ...

docs_kit/connectors/fetchers/gitbook.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from docs_kit.connectors.fetchers.llms_txt import LlmsTxtFetcher
+
+
+class GitBookFetcher(LlmsTxtFetcher):
+    """Fetches documentation from a GitBook site using its AI-native endpoints."""

docs_kit/connectors/fetchers/llms_txt.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import logging
+import re
+import httpx
+
+from docs_kit.core.models import Document
+
+logger = logging.getLogger(__name__)
+
+
+class LlmsTxtFetcher:
+    """Fetches documentation from any site that supports the llms.txt standard.
+
+    Tries /llms-full.txt first (entire docs in one file), then /llms.txt
+    (index of individual pages). Raises ValueError if neither is available.
+
+    This is a shared base class for GitBookFetcher and MintlifyFetcher.
+    """
+
+    def __init__(self, timeout: float = 30.0):
+        self._timeout = timeout
+
+    def fetch(self, base_url: str) -> list[Document]:
+        base_url = base_url.rstrip("/")
+        try:
+            with httpx.Client(timeout=self._timeout, follow_redirects=True) as client:
+                docs = self._try_llms_full_txt(base_url, client)
+                if docs is not None:
+                    return docs
+                docs = self._try_llms_txt(base_url, client)
+                if docs is not None:
+                    return docs
+                raise ValueError(
+                    f"Could not fetch docs from {base_url!r}: "
+                    "no /llms-full.txt or /llms.txt found, or all discovered pages failed."
+                )
+        except httpx.RequestError as exc:
+            raise ValueError(f"Network error fetching {base_url!r}: {exc}") from exc
+
+    def _try_llms_full_txt(self, base_url: str, client: httpx.Client) -> list[Document] | None:
+        resp = client.get(f"{base_url}/llms-full.txt")
+        if resp.status_code == 200 and resp.text.strip():
+            return [Document(
+                source=f"{base_url}/llms-full.txt",
+                content=resp.text,
+                metadata={"format": "markdown", "fetch_method": "llms-full.txt"},
+            )]
+        return None
+
+    def _try_llms_txt(self, base_url: str, client: httpx.Client) -> list[Document] | None:
+        resp = client.get(f"{base_url}/llms.txt")
+        if resp.status_code != 200:
+            return None
+        urls = self._parse_llms_txt(resp.text)
+        documents = []
+        for url in urls:
+            page_resp = client.get(url)
+            if page_resp.status_code == 200 and page_resp.text.strip():
+                documents.append(Document(
+                    source=url,
+                    content=page_resp.text,
+                    metadata={"format": "markdown", "fetch_method": "llms.txt"},
+                ))
+            else:
+                logger.warning("Skipped %s (status %d)", url, page_resp.status_code)
+        return documents if documents else None
+
+    @staticmethod
+    def _parse_llms_txt(content: str) -> list[str]:
+        """Extract URLs from llms.txt index format."""
+        urls = []
+        for line in content.splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            # Markdown link: [Title](url)
+            match = re.search(r'\(https?://[^\)]+\)', line)
+            if match:
+                urls.append(match.group(0)[1:-1])  # strip parens
+                continue
+            # Bare URL
+            if line.startswith("http://") or line.startswith("https://"):
+                urls.append(line.split()[0])  # take first token
+        return urls

docs_kit/connectors/fetchers/mintlify.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import logging
+import re
+import xml.etree.ElementTree as ET
+import httpx
+
+from docs_kit.connectors.fetchers.llms_txt import LlmsTxtFetcher
+from docs_kit.core.html_utils import extract_main_content
+from docs_kit.core.models import Document
+
+logger = logging.getLogger(__name__)
+
+
+class MintlifyFetcher(LlmsTxtFetcher):
+    """Fetches documentation from a Mintlify site.
+
+    Strategy order:
+    1. /llms-full.txt  — full docs in one file (if the site owner enabled it)
+    2. /llms.txt       — index of individual pages (if the site owner enabled it)
+    3. /sitemap.xml    — fallback: parse the sitemap and scrape each page's HTML
+
+    Most Mintlify sites support the llms.txt standard so strategies 1 and 2
+    will succeed. Strategy 3 is a fallback for sites where llms.txt is disabled.
+    """
+
+    def fetch(self, base_url: str) -> list[Document]:
+        base_url = base_url.rstrip("/")
+        try:
+            with httpx.Client(timeout=self._timeout, follow_redirects=True) as client:
+                docs = self._try_llms_full_txt(base_url, client)
+                if docs is not None:
+                    return docs
+
+                docs = self._try_llms_txt(base_url, client)
+                if docs is not None:
+                    return docs
+
+                docs = self._try_sitemap(base_url, client)
+                if docs is not None:
+                    return docs
+
+                raise ValueError(
+                    f"Could not fetch docs from {base_url!r}: "
+                    "no /llms-full.txt, /llms.txt, or /sitemap.xml found, "
+                    "or all discovered pages failed. Is this a public Mintlify site?"
+                )
+        except httpx.RequestError as exc:
+            raise ValueError(f"Network error fetching {base_url!r}: {exc}") from exc
+
+    def _try_sitemap(self, base_url: str, client: httpx.Client) -> list[Document] | None:
+        resp = client.get(f"{base_url}/sitemap.xml")
+        if resp.status_code != 200 or not resp.text.strip():
+            return None
+
+        urls = self._parse_sitemap(resp.text)
+        if not urls:
+            return None
+
+        documents = []
+        for url in urls:
+            page_resp = client.get(url)
+            if page_resp.status_code == 200 and page_resp.text.strip():
+                content = extract_main_content(page_resp.text)
+                if content:
+                    documents.append(Document(
+                        source=url,
+                        content=content,
+                        metadata={"format": "markdown", "fetch_method": "sitemap.xml"},
+                    ))
+            else:
+                logger.warning("Skipped %s (status %d)", url, page_resp.status_code)
+
+        return documents if documents else None
+
+    @staticmethod
+    def _parse_sitemap(xml_content: str) -> list[str]:
+        """Extract page URLs from a standard XML sitemap."""
+        urls = []
+        try:
+            root = ET.fromstring(xml_content)
+            # Sitemap namespace
+            ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+            for loc in root.findall(".//sm:loc", ns):
+                if loc.text:
+                    urls.append(loc.text.strip())
+            # Fallback: no namespace
+            if not urls:
+                for loc in root.findall(".//loc"):
+                    if loc.text:
+                        urls.append(loc.text.strip())
+        except ET.ParseError as exc:
+            logger.warning("Failed to parse sitemap.xml: %s", exc)
+        return urls

docs_kit/connectors/parsers/__init__.py

@@ -0,0 +1,4 @@
+from docs_kit.connectors.parsers.text import TextLoader
+from docs_kit.connectors.parsers.markdown import MarkdownLoader
+
+__all__ = ["TextLoader", "MarkdownLoader"]

docs_kit/connectors/parsers/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from pathlib import Path
+from typing import Protocol
+from docs_kit.core.models import Document
+
+class DocumentLoader(Protocol):
+    supported_extensions: list[str]
+    def load(self, path: Path) -> Document: ...

docs_kit/connectors/parsers/markdown.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from pathlib import Path
+from docs_kit.core.models import Document
+
+class MarkdownLoader:
+    supported_extensions = [".md"]
+    def load(self, path: Path) -> Document:
+        return Document(source=str(path), content=path.read_text(encoding="utf-8"))

docs_kit/connectors/parsers/text.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from pathlib import Path
+from docs_kit.core.models import Document
+
+class TextLoader:
+    supported_extensions = [".txt"]
+    def load(self, path: Path) -> Document:
+        return Document(source=str(path), content=path.read_text(encoding="utf-8"))

docs_kit/connectors/vector_stores/__init__.py

@@ -0,0 +1,3 @@
+from docs_kit.connectors.vector_stores.qdrant import QdrantStore
+
+__all__ = ["QdrantStore"]

docs_kit/connectors/vector_stores/base.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+from typing import Protocol
+from qdrant_client.models import SparseVector
+from docs_kit.core.models import Chunk, RetrievedChunk
+
+class VectorStore(Protocol):
+    def ensure_collection(self, vector_size: int) -> None: ...
+    def upsert(self, chunks: list[Chunk], dense_vectors: list[list[float]], sparse_vectors: list[SparseVector] | None = None, recreate: bool = False) -> int: ...
+    def query(self, dense_vector: list[float], sparse_vector: SparseVector | None = None, limit: int = 5, score_threshold: float = 0.0) -> list[RetrievedChunk]: ...
+    def collection_stats(self) -> dict[str, bool | int | None]: ...
+    def list_sources(self) -> list[str]: ...
+    def get_by_source(self, source: str) -> list[RetrievedChunk]: ...
+    def upsert_document(self, source: str, content: str, recreate: bool = False) -> None: ...
+    def get_document_content(self, source: str) -> str | None: ...
+    def close(self) -> None: ...

docs_kit/connectors/vector_stores/qdrant.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+import uuid
+from datetime import datetime, timezone
+from qdrant_client import QdrantClient
+from qdrant_client.http import models as rest
+from qdrant_client.http.exceptions import UnexpectedResponse
+from qdrant_client.models import FieldCondition, Filter, Fusion, FusionQuery, MatchValue, Prefetch, SparseVector
+from docs_kit.core.models import Chunk, RetrievedChunk
+
+class QdrantStore:
+    def __init__(self, client: QdrantClient | None = None, collection_name: str = "knowledge_base",
+                 url: str = "", local_path: str = ".qdrant",
+                 dense_prefetch_limit: int = 20, sparse_prefetch_limit: int = 20):
+        if client is not None:
+            self._client = client
+        elif url:
+            self._client = QdrantClient(url=url)
+        else:
+            self._client = QdrantClient(path=local_path)
+        self._collection_name = collection_name
+        self._documents_collection_name = f"{collection_name}__documents"
+        self._dense_prefetch_limit = dense_prefetch_limit
+        self._sparse_prefetch_limit = sparse_prefetch_limit
+
+    def ensure_collection(self, vector_size: int) -> None:
+        collections = self._client.get_collections().collections
+        if any(c.name == self._collection_name for c in collections):
+            info = self._client.get_collection(self._collection_name)
+            params = getattr(getattr(info, "config", None), "params", None)
+            vectors_config = getattr(params, "vectors", None)
+            sparse_config = getattr(params, "sparse_vectors", None)
+
+            # Must have named dense+bm25 schema (not legacy single-vector).
+            has_named_dense = isinstance(vectors_config, dict) and "dense" in vectors_config
+            if not has_named_dense:
+                raise ValueError(
+                    f"Collection '{self._collection_name}' exists but uses the old single-vector schema. "
+                    "Re-ingest with recreate=True to upgrade to the hybrid dense+BM25 schema."
+                )
+
+            # Dense dimension must match the current embedding model.
+            existing_size = getattr(vectors_config.get("dense"), "size", None)
+            if existing_size is not None and existing_size != vector_size:
+                raise ValueError(
+                    f"Collection '{self._collection_name}' has dense vectors of size {existing_size} "
+                    f"but current embedding model produces size {vector_size}. "
+                    "Re-ingest with recreate=True to rebuild the collection."
+                )
+
+            # Must have the bm25 sparse index for hybrid retrieval.
+            has_bm25 = isinstance(sparse_config, dict) and "bm25" in sparse_config
+            if not has_bm25:
+                raise ValueError(
+                    f"Collection '{self._collection_name}' is missing the 'bm25' sparse vector index "
+                    "required for hybrid retrieval. Re-ingest with recreate=True."
+                )
+            return
+        self._client.create_collection(
+            collection_name=self._collection_name,
+            vectors_config={"dense": rest.VectorParams(size=vector_size, distance=rest.Distance.COSINE)},
+            sparse_vectors_config={"bm25": rest.SparseVectorParams(modifier=rest.Modifier.IDF)},
+        )
+
+    def ensure_documents_collection(self) -> None:
+        collections = self._client.get_collections().collections
+        if any(c.name == self._documents_collection_name for c in collections):
+            return
+        self._client.create_collection(
+            collection_name=self._documents_collection_name,
+            vectors_config={"doc": rest.VectorParams(size=1, distance=rest.Distance.COSINE)},
+        )
+
+    def upsert(self, chunks: list[Chunk], dense_vectors: list[list[float]],
+               sparse_vectors: list[SparseVector] | None = None, recreate: bool = False) -> int:
+        if not chunks:
+            return 0
+        if recreate:
+            if self._client.collection_exists(self._collection_name):
+                self._client.delete_collection(self._collection_name)
+            if self._client.collection_exists(self._documents_collection_name):
+                self._client.delete_collection(self._documents_collection_name)
+            self.ensure_collection(len(dense_vectors[0]))
+        else:
+            self.ensure_collection(len(dense_vectors[0]))
+        ingested_at = datetime.now(timezone.utc).isoformat()
+        points = []
+        for i, chunk in enumerate(chunks):
+            vector: dict = {"dense": dense_vectors[i]}
+            if sparse_vectors:
+                vector["bm25"] = sparse_vectors[i]
+            points.append(rest.PointStruct(
+                id=str(uuid.uuid4()), vector=vector,
+                payload={"source": chunk.source, "chunk_index": chunk.chunk_index, "text": chunk.text, "ingested_at": ingested_at},
+            ))
+        self._client.upsert(collection_name=self._collection_name, points=points)
+        return len(points)
+
+    def upsert_document(self, source: str, content: str, recreate: bool = False) -> None:
+        if recreate and self._client.collection_exists(self._documents_collection_name):
+            self._client.delete_collection(self._documents_collection_name)
+        self.ensure_documents_collection()
+        self._client.delete(
+            collection_name=self._documents_collection_name,
+            points_selector=Filter(
+                must=[FieldCondition(key="source", match=MatchValue(value=source))]
+            ),
+        )
+        point = rest.PointStruct(
+            id=str(uuid.uuid4()),
+            vector={"doc": [1.0]},
+            payload={"source": source, "content": content, "ingested_at": datetime.now(timezone.utc).isoformat()},
+        )
+        self._client.upsert(collection_name=self._documents_collection_name, points=[point])
+
+    def query(self, dense_vector: list[float], sparse_vector: SparseVector | None = None,
+              limit: int = 5, score_threshold: float = 0.0) -> list[RetrievedChunk]:
+        if sparse_vector is not None:
+            search_result = self._client.query_points(
+                collection_name=self._collection_name,
+                prefetch=[
+                    Prefetch(query=sparse_vector, using="bm25", limit=self._sparse_prefetch_limit),
+                    Prefetch(query=dense_vector, using="dense", limit=self._dense_prefetch_limit),
+                ],
+                query=FusionQuery(fusion=Fusion.RRF), limit=limit,
+                score_threshold=score_threshold, with_payload=True,
+            )
+        else:
+            search_result = self._client.query_points(
+                collection_name=self._collection_name, query=dense_vector, using="dense",
+                limit=limit, score_threshold=score_threshold, with_payload=True,
+            )
+        points = getattr(search_result, "points", search_result)
+        results: list[RetrievedChunk] = []
+        for point in points:
+            payload = point.payload or {}
+            text = str(payload.get("text", ""))
+            if text:
+                results.append(RetrievedChunk(
+                    source=str(payload.get("source", "unknown")),
+                    chunk_index=int(payload.get("chunk_index", 0)),
+                    text=text, score=float(getattr(point, "score", 0.0)),
+                ))
+        return results
+
+    def collection_stats(self) -> dict[str, bool | int | None]:
+        try:
+            info = self._client.get_collection(self._collection_name)
+        except UnexpectedResponse as exc:
+            if getattr(exc, "status_code", None) == 404:
+                return {"collection_exists": False, "points_count": 0}
+            raise
+        points_count = getattr(info, "points_count", None)
+        if points_count is None:
+            result = getattr(info, "result", None)
+            points_count = getattr(result, "points_count", None)
+        return {"collection_exists": True, "points_count": int(points_count) if points_count is not None else None}
+
+    def list_sources(self) -> list[str]:
+        if not self._client.collection_exists(self._collection_name):
+            return []
+        sources: set[str] = set()
+        offset = None
+        while True:
+            points, next_offset = self._client.scroll(
+                collection_name=self._collection_name,
+                limit=100,
+                offset=offset,
+                with_payload=["source"],
+                with_vectors=False,
+            )
+            for point in points:
+                src = (point.payload or {}).get("source")
+                if src is not None:
+                    sources.add(str(src))
+            if next_offset is None:
+                break
+            offset = next_offset
+        return sorted(sources)
+
+    def get_by_source(self, source: str) -> list[RetrievedChunk]:
+        if not self._client.collection_exists(self._collection_name):
+            return []
+        results = []
+        offset = None
+        while True:
+            points, next_offset = self._client.scroll(
+                collection_name=self._collection_name,
+                scroll_filter=Filter(must=[
+                    FieldCondition(key="source", match=MatchValue(value=source))
+                ]),
+                limit=100,
+                offset=offset,
+                with_payload=True,
+                with_vectors=False,
+            )
+            for point in points:
+                payload = point.payload or {}
+                text = str(payload.get("text", ""))
+                if text:
+                    results.append(RetrievedChunk(
+                        source=str(payload.get("source", source)),
+                        chunk_index=int(payload.get("chunk_index", 0)),
+                        text=text,
+                        score=1.0,
+                    ))
+            if next_offset is None:
+                break
+            offset = next_offset
+        return sorted(results, key=lambda c: c.chunk_index)
+
+    def get_document_content(self, source: str) -> str | None:
+        if not self._client.collection_exists(self._documents_collection_name):
+            return None
+        points, _next_offset = self._client.scroll(
+            collection_name=self._documents_collection_name,
+            scroll_filter=Filter(
+                must=[FieldCondition(key="source", match=MatchValue(value=source))]
+            ),
+            limit=1,
+            with_payload=True,
+            with_vectors=False,
+        )
+        if not points:
+            return None
+        payload = points[0].payload or {}
+        content = payload.get("content")
+        if content is None:
+            return None
+        return str(content)
+
+    def delete_source(self, source: str) -> bool:
+        """Delete all chunks and the document for a given source. Returns True if anything was deleted."""
+        deleted_any = False
+        source_filter = Filter(must=[FieldCondition(key="source", match=MatchValue(value=source))])
+        if self._client.collection_exists(self._collection_name):
+            result = self._client.delete(
+                collection_name=self._collection_name,
+                points_selector=source_filter,
+            )
+            if getattr(result, "status", None) is not None:
+                deleted_any = True
+        if self._client.collection_exists(self._documents_collection_name):
+            result = self._client.delete(
+                collection_name=self._documents_collection_name,
+                points_selector=source_filter,
+            )
+            if getattr(result, "status", None) is not None:
+                deleted_any = True
+        return deleted_any
+
+    def list_sources_with_dates(self) -> list[dict]:
+        """List all ingested document sources with their ingestion timestamps."""
+        if not self._client.collection_exists(self._documents_collection_name):
+            return []
+        entries = []
+        offset = None
+        while True:
+            points, next_offset = self._client.scroll(
+                collection_name=self._documents_collection_name,
+                limit=100,
+                offset=offset,
+                with_payload=["source", "ingested_at"],
+                with_vectors=False,
+            )
+            for point in points:
+                payload = point.payload or {}
+                src = payload.get("source")
+                if src is not None:
+                    entries.append({
+                        "source": str(src),
+                        "ingested_at": str(payload.get("ingested_at", "unknown")),
+                    })
+            if next_offset is None:
+                break
+            offset = next_offset
+        return sorted(entries, key=lambda e: e["source"])
+
+    def close(self) -> None:
+        self._client.close()