deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/embedding/validation.py

@@ -0,0 +1,128 @@
+"""Validation helpers for embedding vectors."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import math
+from numbers import Real
+from typing import Any
+
+
+def _context(
+    *,
+    binding: str | None,
+    model: str | None,
+    batch_index: int | None,
+    total_batches: int | None,
+) -> str:
+    parts: list[str] = []
+    if binding:
+        parts.append(f"binding={binding}")
+    if model:
+        parts.append(f"model={model}")
+    if batch_index is not None and total_batches is not None:
+        parts.append(f"batch={batch_index}/{total_batches}")
+    return f" ({', '.join(parts)})" if parts else ""
+
+
+def _raise_invalid_vector(message: str, *, item_index: int, context: str) -> None:
+    raise ValueError(
+        "Embedding provider returned invalid vector "
+        f"at item {item_index}{context}: {message}. "
+        "RAG requires dense numeric embeddings; check the embedding provider/model "
+        "and re-index the knowledge base after fixing it."
+    )
+
+
+def validate_embedding_batch(
+    embeddings: Any,
+    *,
+    expected_count: int,
+    binding: str | None = None,
+    model: str | None = None,
+    batch_index: int | None = None,
+    total_batches: int | None = None,
+    start_index: int = 0,
+) -> list[list[float]]:
+    """Return normalized float vectors or raise a clear provider error.
+
+    Provider smoke tests and RAG indexing both ultimately need a list of dense
+    numeric vectors. A single ``None`` coordinate otherwise reaches LlamaIndex's
+    similarity code and fails later as ``NoneType * float``.
+    """
+
+    context = _context(
+        binding=binding,
+        model=model,
+        batch_index=batch_index,
+        total_batches=total_batches,
+    )
+
+    if (
+        embeddings is None
+        or isinstance(embeddings, (str, bytes))
+        or not isinstance(embeddings, Sequence)
+    ):
+        raise ValueError(
+            "Embedding provider returned invalid embeddings payload"
+            f"{context}: expected a list of {expected_count} vector(s), "
+            f"got {type(embeddings).__name__}."
+        )
+
+    actual_count = len(embeddings)
+    if actual_count != expected_count:
+        raise ValueError(
+            "Embedding provider returned an unexpected number of vectors"
+            f"{context}: expected {expected_count}, got {actual_count}. "
+            "This usually means the provider dropped one or more inputs; "
+            "RAG indexing/search cannot safely continue."
+        )
+
+    normalized: list[list[float]] = []
+    for local_index, vector in enumerate(embeddings):
+        item_index = start_index + local_index
+        if vector is None:
+            _raise_invalid_vector("vector is null", item_index=item_index, context=context)
+        if isinstance(vector, (str, bytes)) or not isinstance(vector, Sequence):
+            _raise_invalid_vector(
+                f"expected a numeric sequence, got {type(vector).__name__}",
+                item_index=item_index,
+                context=context,
+            )
+        if len(vector) == 0:
+            _raise_invalid_vector("vector is empty", item_index=item_index, context=context)
+
+        normalized_vector: list[float] = []
+        for dim_index, value in enumerate(vector):
+            if value is None:
+                _raise_invalid_vector(
+                    f"dimension {dim_index} is null",
+                    item_index=item_index,
+                    context=context,
+                )
+            if isinstance(value, bool) or not isinstance(value, Real):
+                _raise_invalid_vector(
+                    f"dimension {dim_index} is {type(value).__name__}, not a number",
+                    item_index=item_index,
+                    context=context,
+                )
+            numeric = float(value)
+            if not math.isfinite(numeric):
+                _raise_invalid_vector(
+                    f"dimension {dim_index} is not finite",
+                    item_index=item_index,
+                    context=context,
+                )
+            normalized_vector.append(numeric)
+
+        normalized.append(normalized_vector)
+
+    dims = {len(vector) for vector in normalized}
+    if len(dims) > 1:
+        raise ValueError(
+            "Embedding provider returned inconsistent vector dimensions"
+            f"{context}: dimensions={sorted(dims)}. "
+            "Use a single embedding model/dimension and re-index the knowledge base."
+        )
+
+    return normalized

deeptutor/services/github_source/__init__.py

@@ -0,0 +1,5 @@
+"""GitHub-source sync subsystem: pull Markdown from GitHub repos into KBs."""
+
+from deeptutor.services.github_source.sync_service import get_sync_service
+
+__all__ = ["get_sync_service"]

deeptutor/services/github_source/client.py

@@ -0,0 +1,164 @@
+"""Minimal async GitHub REST API client used by the KB source-sync engine.
+
+Self-contained, depends only on ``httpx`` (already a project dependency).
+No ``gh`` or ``git`` CLI required.  A ``GITHUB_TOKEN`` environment variable
+is optional but recommended — it lifts the unauthenticated rate limit from
+60 to 5 000 requests/hour and enables private-repo access.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+import os
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+GITHUB_API_BASE = "https://api.github.com"
+DEFAULT_TIMEOUT_S = 30.0
+MAX_FILE_BYTES = 2 * 1024 * 1024  # 2 MB cap per individual file download
+
+
+@dataclass(frozen=True)
+class TreeEntry:
+    """One entry from a GitHub tree listing."""
+
+    path: str
+    sha: str
+    type: str  # "blob" | "tree"
+
+
+@dataclass(frozen=True)
+class FileChange:
+    """One file changed between two commits."""
+
+    path: str
+    status: str  # "added" | "removed" | "modified" | "renamed" | ...
+    sha: str
+
+
+@dataclass(frozen=True)
+class GitHubAPIError(Exception):
+    """Raised when the GitHub API returns a non-2xx response."""
+
+    status_code: int
+    message: str
+
+    def __str__(self) -> str:
+        return f"GitHub API {self.status_code}: {self.message}"
+
+
+def _token() -> str | None:
+    return os.environ.get("GITHUB_TOKEN") or None
+
+
+def _headers() -> dict[str, str]:
+    h: dict[str, str] = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+        "User-Agent": "DeepTutor/1.0 (+https://hkuds.dev/deeptutor)",
+    }
+    token = _token()
+    if token:
+        h["Authorization"] = f"Bearer {token}"
+    return h
+
+
+class GitHubClient:
+    """Thin async wrapper around the handful of REST endpoints we need."""
+
+    def __init__(
+        self,
+        *,
+        timeout_s: float = DEFAULT_TIMEOUT_S,
+        client_factory: Any = None,
+    ) -> None:
+        self._timeout_s = timeout_s
+        self._client_factory = client_factory
+
+    async def _request_json(self, method: str, url: str, **kw: Any) -> Any:
+        factory = self._client_factory or (lambda: httpx.AsyncClient(timeout=self._timeout_s))
+        async with factory() as client:
+            resp = await client.request(method, url, headers=_headers(), **kw)
+        if resp.status_code >= 400:
+            try:
+                body = resp.json()
+                msg = body.get("message", resp.text[:500])
+            except Exception:
+                msg = resp.text[:500]
+            raise GitHubAPIError(resp.status_code, str(msg))
+        return resp.json()
+
+    async def _request_bytes(self, url: str, **kw: Any) -> bytes:
+        factory = self._client_factory or (lambda: httpx.AsyncClient(timeout=self._timeout_s))
+        async with factory() as client:
+            resp = await client.get(url, headers=_headers(), **kw)
+        if resp.status_code >= 400:
+            raise GitHubAPIError(resp.status_code, resp.text[:500])
+        return resp.content
+
+    async def get_default_branch(self, repo: str) -> str:
+        data = await self._request_json("GET", f"{GITHUB_API_BASE}/repos/{repo}")
+        return data.get("default_branch", "main")
+
+    async def get_latest_commit_sha(self, repo: str, branch: str) -> str:
+        data = await self._request_json("GET", f"{GITHUB_API_BASE}/repos/{repo}/commits/{branch}")
+        return data["sha"]
+
+    async def get_tree(
+        self,
+        repo: str,
+        branch: str,
+        *,
+        path_prefix: str = "",
+        glob: str = "*.md",
+    ) -> list[TreeEntry]:
+        data = await self._request_json(
+            "GET",
+            f"{GITHUB_API_BASE}/repos/{repo}/git/trees/{branch}",
+            params={"recursive": "1"},
+        )
+        entries: list[TreeEntry] = []
+        prefix = path_prefix.strip("/")
+        for item in data.get("tree", []):
+            if item.get("type") != "blob":
+                continue
+            p = item.get("path", "")
+            if prefix and not p.startswith(prefix + "/") and p != prefix:
+                continue
+            if not _match_glob(p, glob):
+                continue
+            entries.append(TreeEntry(path=p, sha=item.get("sha", ""), type="blob"))
+        return entries
+
+    async def compare_commits(self, repo: str, base: str, head: str) -> list[FileChange]:
+        data = await self._request_json(
+            "GET",
+            f"{GITHUB_API_BASE}/repos/{repo}/compare/{base}...{head}",
+        )
+        changes: list[FileChange] = []
+        for item in data.get("files", []):
+            changes.append(
+                FileChange(
+                    path=item.get("filename", ""),
+                    status=item.get("status", "modified"),
+                    sha=item.get("sha", ""),
+                )
+            )
+        return changes
+
+    async def download_file(self, repo: str, path: str, ref: str) -> bytes:
+        url = f"https://raw.githubusercontent.com/{repo}/{ref}/{path}"
+        content = await self._request_bytes(url)
+        if len(content) > MAX_FILE_BYTES:
+            content = content[:MAX_FILE_BYTES]
+        return content
+
+
+def _match_glob(path: str, glob: str) -> bool:
+    import fnmatch
+
+    return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path.rsplit("/", 1)[-1], glob)

deeptutor/services/github_source/sync.py

@@ -0,0 +1,384 @@
+"""Core sync logic: pull Markdown from a GitHub repo into a KB's raw/ dir."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import logging
+from pathlib import Path
+import re
+from urllib.parse import urlsplit, urlunsplit
+
+from deeptutor.knowledge.add_documents import DEFAULT_BASE_DIR
+from deeptutor.services.github_source.client import (
+    GitHubAPIError,
+    GitHubClient,
+)
+
+logger = logging.getLogger(__name__)
+
+SYNC_INTERVAL_HOURS = 24
+MARKDOWN_EXTENSIONS = (".md", ".markdown")
+
+
+@dataclass
+class SyncResult:
+    ok: bool
+    skipped: bool = False
+    files_added: int = 0
+    files_updated: int = 0
+    files_removed: int = 0
+    error: str = ""
+
+
+def _utcnow_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def redact_sync_error(exc: Exception) -> str:
+    """Return a bounded sync error without credential-shaped values."""
+    message = str(exc).strip() or type(exc).__name__
+
+    def redact_url(match: re.Match[str]) -> str:
+        try:
+            parsed = urlsplit(match.group(0))
+            host = parsed.hostname or ""
+            if parsed.port is not None:
+                host = f"{host}:{parsed.port}"
+            return urlunsplit((parsed.scheme, host, parsed.path, "", ""))
+        except ValueError:
+            # Error strings are untrusted input too. A malformed bracket or port
+            # must not make credential redaction mask the original sync failure.
+            return "[redacted-url]"
+
+    message = re.sub(r"https?://[^\s,;]+", redact_url, message)
+    message = re.sub(r"\bsk-[A-Za-z0-9_-]{4,}\b", "[redacted]", message)
+    message = re.sub(
+        r"(?i)((?:api[_-]?key|authorization|token|password)\s*[:=]\s*)[^\s,;]+",
+        r"\1[redacted]",
+        message,
+    )
+    return message[:1000]
+
+
+def _record_sync_failure(kb_name: str, source: dict, base_dir: str, error: str) -> None:
+    try:
+        from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+        KnowledgeBaseManager(base_dir=base_dir).update_github_source_state(
+            kb_name=kb_name,
+            source_id=source["id"],
+            last_sync_status="error",
+            last_sync_error=error,
+        )
+    except Exception:
+        logger.warning("Could not persist GitHub sync failure status for '%s'", kb_name)
+
+
+def _record_sync_success(
+    kb_name: str,
+    source: dict,
+    base_dir: str,
+    *,
+    sha: str,
+    files_synced: int | None = None,
+) -> None:
+    """Persist "GitHub was checked and the source is current".
+
+    A same-SHA check is a successful sync that had nothing to transfer, so it
+    owes the same freshness stamp as one that did: without it the hourly
+    service read the source as stale forever and re-queried GitHub every
+    cycle, and an error left by an earlier transient failure stayed on the
+    source after a later check had confirmed it current (#1489). ``files_synced``
+    is omitted on that path so the last real transfer's count survives.
+    """
+    from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+    fields: dict = {} if files_synced is None else {"files_synced": files_synced}
+    KnowledgeBaseManager(base_dir=base_dir).update_github_source_state(
+        kb_name=kb_name,
+        source_id=source["id"],
+        last_synced_sha=sha,
+        last_synced_at=_utcnow_iso(),
+        last_sync_status="success",
+        last_sync_error=None,
+        **fields,
+    )
+
+
+def _is_markdown(path: str) -> bool:
+    return path.lower().endswith(MARKDOWN_EXTENSIONS)
+
+
+def _raw_rel_path(github_path: str, path_prefix: str) -> str:
+    prefix = path_prefix.strip("/")
+    if prefix and github_path.startswith(prefix + "/"):
+        return github_path[len(prefix) + 1 :]
+    return github_path
+
+
+def _contained_dest(raw_dir: Path, rel: str) -> Path | None:
+    """Resolve *rel* under *raw_dir*, or ``None`` if it escapes.
+
+    ``rel`` comes from the GitHub API's tree/compare response, so it is remote
+    input. Two shapes would otherwise write outside the KB: a ``..`` segment,
+    and — more quietly — an absolute path, because ``Path("/kb") / "/etc/x"``
+    is ``/etc/x``, silently discarding the base. git rejects both in tree
+    entries today, but a downloader must not depend on the remote to enforce
+    where it writes.
+    """
+    candidate = (raw_dir / rel).resolve()
+    root = raw_dir.resolve()
+    if candidate != root and root not in candidate.parents:
+        logger.warning("GitHub sync: refusing path outside the KB raw dir: %s", rel)
+        return None
+    return candidate
+
+
+def _filter_markdown_changes(changes, path_prefix, glob):
+    from fnmatch import fnmatch
+
+    prefix = path_prefix.strip("/")
+    result = []
+    for ch in changes:
+        p = ch.path
+        if prefix and not p.startswith(prefix + "/") and p != prefix:
+            continue
+        if not (_is_markdown(p) and (fnmatch(p, glob) or fnmatch(p.rsplit("/", 1)[-1], glob))):
+            continue
+        result.append(ch)
+    return result
+
+
+def _filter_markdown_entries(entries, path_prefix, glob):
+    from fnmatch import fnmatch
+
+    prefix = path_prefix.strip("/")
+    result = []
+    for e in entries:
+        p = e.path
+        if prefix and not p.startswith(prefix + "/") and p != prefix:
+            continue
+        if not (_is_markdown(p) and (fnmatch(p, glob) or fnmatch(p.rsplit("/", 1)[-1], glob))):
+            continue
+        result.append(e)
+    return result
+
+
+async def sync_source(kb_name, source, *, base_dir=DEFAULT_BASE_DIR, client=None):
+    from deeptutor.services.rag.provider_binding import resolve_bound_provider
+
+    accepted_indexing_snapshot = None
+    if resolve_bound_provider(base_dir, kb_name) == "lightrag":
+        from deeptutor.services.rag.pipelines.lightrag.indexing_policy import (
+            bind_target,
+            resolve_write_snapshot,
+        )
+
+        kb_dir = Path(base_dir) / kb_name
+        try:
+            accepted_indexing_snapshot = bind_target(
+                resolve_write_snapshot(kb_dir, base_dir=str(base_dir), kb_name=kb_name), kb_dir
+            )
+        except Exception as exc:
+            error = redact_sync_error(exc)
+            _record_sync_failure(kb_name, source, base_dir, error)
+            return SyncResult(ok=False, error=error)
+    indexing_options = (
+        {"accepted_indexing_snapshot": accepted_indexing_snapshot}
+        if accepted_indexing_snapshot is not None
+        else {}
+    )
+    client = client or GitHubClient()
+    kb_dir = Path(base_dir) / kb_name
+    raw_dir = kb_dir / "raw"
+    raw_dir.mkdir(parents=True, exist_ok=True)
+
+    repo = source["repo"]
+    branch = source.get("branch") or "main"
+    path_prefix = source.get("path") or ""
+    glob = source.get("glob") or "*.md"
+    old_sha = source.get("last_synced_sha") or ""
+
+    try:
+        latest_sha = await client.get_latest_commit_sha(repo, branch)
+    except GitHubAPIError as exc:
+        error = redact_sync_error(exc)
+        _record_sync_failure(kb_name, source, base_dir, error)
+        return SyncResult(ok=False, error=error)
+    except Exception as exc:
+        error = f"Failed to fetch latest SHA: {redact_sync_error(exc)}"
+        _record_sync_failure(kb_name, source, base_dir, error)
+        return SyncResult(ok=False, error=error)
+
+    if old_sha and old_sha == latest_sha:
+        _record_sync_success(kb_name, source, base_dir, sha=latest_sha)
+        return SyncResult(ok=True, skipped=True)
+
+    try:
+        if not old_sha:
+            result = await _full_sync(
+                client,
+                kb_name,
+                raw_dir,
+                repo,
+                branch,
+                path_prefix,
+                glob,
+                latest_sha,
+                base_dir,
+                **indexing_options,
+            )
+        else:
+            result = await _incremental_sync(
+                client,
+                kb_name,
+                raw_dir,
+                repo,
+                branch,
+                path_prefix,
+                glob,
+                old_sha,
+                latest_sha,
+                base_dir,
+                **indexing_options,
+            )
+    except GitHubAPIError as exc:
+        error = redact_sync_error(exc)
+        _record_sync_failure(kb_name, source, base_dir, error)
+        return SyncResult(ok=False, error=error)
+    except Exception as exc:
+        error = redact_sync_error(exc)
+        _record_sync_failure(kb_name, source, base_dir, error)
+        return SyncResult(ok=False, error=error)
+
+    if not result.ok:
+        _record_sync_failure(kb_name, source, base_dir, result.error)
+        return result
+
+    _record_sync_success(
+        kb_name,
+        source,
+        base_dir,
+        sha=latest_sha,
+        files_synced=result.files_added + result.files_updated,
+    )
+    return result
+
+
+async def _full_sync(
+    client,
+    kb_name,
+    raw_dir,
+    repo,
+    branch,
+    path_prefix,
+    glob,
+    latest_sha,
+    base_dir,
+    *,
+    accepted_indexing_snapshot=None,
+):
+    tree = await client.get_tree(repo, branch, path_prefix=path_prefix, glob=glob)
+    entries = _filter_markdown_entries(tree, path_prefix, glob)
+    downloaded = []
+    for entry in entries:
+        rel = _raw_rel_path(entry.path, path_prefix)
+        dest = _contained_dest(raw_dir, rel)
+        if dest is None:
+            continue
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        content = await client.download_file(repo, entry.path, latest_sha)
+        dest.write_bytes(content)
+        downloaded.append(str(dest))
+    if downloaded:
+        await _index_files(
+            kb_name,
+            downloaded,
+            base_dir,
+            **(
+                {"accepted_indexing_snapshot": accepted_indexing_snapshot}
+                if accepted_indexing_snapshot is not None
+                else {}
+            ),
+        )
+    return SyncResult(ok=True, files_added=len(downloaded))
+
+
+async def _incremental_sync(
+    client,
+    kb_name,
+    raw_dir,
+    repo,
+    branch,
+    path_prefix,
+    glob,
+    old_sha,
+    new_sha,
+    base_dir,
+    *,
+    accepted_indexing_snapshot=None,
+):
+    all_changes = await client.compare_commits(repo, old_sha, new_sha)
+    changes = _filter_markdown_changes(all_changes, path_prefix, glob)
+    added_or_modified = []
+    removed = []
+    for ch in changes:
+        rel = _raw_rel_path(ch.path, path_prefix)
+        if ch.status == "removed":
+            removed.append(rel)
+        else:
+            added_or_modified.append(ch.path)
+    downloaded = []
+    for gh_path in added_or_modified:
+        rel = _raw_rel_path(gh_path, path_prefix)
+        dest = _contained_dest(raw_dir, rel)
+        if dest is None:
+            continue
+        content = await client.download_file(repo, gh_path, new_sha)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(content)
+        downloaded.append(str(dest))
+    if downloaded:
+        await _index_files(
+            kb_name,
+            downloaded,
+            base_dir,
+            **(
+                {"accepted_indexing_snapshot": accepted_indexing_snapshot}
+                if accepted_indexing_snapshot is not None
+                else {}
+            ),
+        )
+    removed_count = 0
+    for rel in removed:
+        target = _contained_dest(raw_dir, rel)
+        if target is not None and target.exists():
+            try:
+                from deeptutor.knowledge.add_documents import remove_raw_document
+
+                kb_dir = Path(base_dir) / kb_name
+                remove_raw_document(kb_dir, target)
+                removed_count += 1
+            except Exception as exc:
+                logger.warning("Failed to remove %s: %s", rel, exc)
+    return SyncResult(ok=True, files_added=len(downloaded), files_removed=removed_count)
+
+
+async def _index_files(kb_name, file_paths, base_dir, *, accepted_indexing_snapshot=None):
+    if not file_paths:
+        return 0
+    from deeptutor.knowledge.add_documents import add_documents
+
+    count = await add_documents(
+        kb_name=kb_name,
+        source_files=file_paths,
+        base_dir=base_dir,
+        allow_duplicates=False,
+        **(
+            {"accepted_indexing_snapshot": accepted_indexing_snapshot}
+            if accepted_indexing_snapshot is not None
+            else {}
+        ),
+    )
+    return count or 0