fusion-rag 0.8.32__py3-none-any.whl

fusion_rag/__init__.py

@@ -0,0 +1,21 @@
+"""Fusion-RAG — Apple Silicon native offline vector knowledge base backend.
+
+All model inference (embeddings, chat) goes through fusion-mlx HTTP API.
+Never imports MLX, mlx-lm, or any engine code directly.
+"""
+
+from .embed.client import EmbeddingClient
+from .engine.chunker import Chunker
+from .engine.document import DocumentParser, DocumentType
+from .engine.knowledge_base import KnowledgeBase, KnowledgeBaseConfig
+from .store.metadata_store import MetadataStore
+
+__all__ = [
+    "Chunker",
+    "DocumentParser",
+    "DocumentType",
+    "EmbeddingClient",
+    "KnowledgeBase",
+    "KnowledgeBaseConfig",
+    "MetadataStore",
+]

fusion_rag/_data_dir.py

@@ -0,0 +1,22 @@
+# Issue #96: single-point data-dir resolution via fusion_core.config.
+# fusion-core's convention passes the SHORT app name ("rag" — see
+# fusion-core tests/test_config_conventions.py), so the env var read is
+# FUSION_RAG_DATA_DIR; passing the full "fusion-rag" name would double the
+# FUSION_ prefix on the env lookup. The bare-app default (~/.rag) would
+# orphan every existing store, so the pre-#96 location ~/.fusion-rag is
+# passed as the explicit default. Call sites stay one-line: data_dir().
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from fusion_core.config import service_data_dir
+
+logger = logging.getLogger(__name__)
+
+
+def data_dir() -> Path:
+    resolved = service_data_dir("rag", default=Path.home() / ".fusion-rag")
+    logger.debug("fusion-rag data dir resolved: %s", resolved)
+    return resolved

fusion_rag/_validators.py

@@ -0,0 +1,183 @@
+"""Input validation helpers — single trust-boundary layer.
+
+Centralizes all external-input confinement (硬伤3):
+  - kb_id / table_name / identifier -> regex whitelist
+  - filesystem path -> configured root + is_relative_to + symlink reject
+  - URL -> scheme whitelist + private-network reject (SSRF guard)
+  - git repo_url -> scheme whitelist + ext:: reject (RCE guard)
+
+No business logic here. Pure validation + raise on violation. Every
+caller that takes external input and feeds it to fs/SQL/network/git
+must route through these helpers.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import re
+import socket
+from pathlib import Path
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+# identifier whitelist: letters/digits/_-. , 1-64 chars, no path separators
+_IDENT_RE = re.compile(r"^[A-Za-z0-9_\-]{1,64}$")
+# SQL identifier: letters/digits/_., must start with letter/_
+_SQL_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*$")
+
+_ALLOWED_URL_SCHEMES = frozenset({"http", "https"})
+_ALLOWED_GIT_SCHEMES = frozenset({"http", "https", "git", "ssh"})
+
+# RFC1918 + loopback + link-local + unique-local v6 ranges
+_PRIVATE_NETWORKS = [
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("0.0.0.0/8"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+class ValidationError(ValueError):
+    """Raised when external input violates trust-boundary rules."""
+
+
+def validate_identifier(name: str, *, field: str = "identifier") -> str:
+    """Validate a KB id / rule id / generic identifier.
+
+    Rejects path separators, dots, and any non-whitelisted char.
+    """
+    # P4-8: the regex ^[A-Za-z0-9_\\-]{1,64}$ already rejects "."/".." (dots
+    # not in the charset), so the prior `if name in (".", "..")` branch was
+    # unreachable dead code — a sign the validation layers weren't reasoning
+    # about each other. Removed; the regex is the sole authority.
+    if not name or not _IDENT_RE.match(name):
+        logger.warning("validate_identifier reject: field=%s value=%r", field, name)
+        raise ValidationError(f"invalid {field}: must match [A-Za-z0-9_-]{{1,64}}")
+    return name
+
+
+def validate_sql_identifier(name: str, *, field: str = "table_name") -> str:
+    """Validate a SQL table/column identifier against injection.
+
+    Identifiers cannot be bound as params, so we whitelist the charset
+    and reject anything containing quotes / semicolons / path separators.
+    Caller should still wrap the result in double quotes for SQLite PRAGMA.
+    """
+    if not name or not _SQL_IDENT_RE.match(name):
+        logger.warning("validate_sql_identifier reject: field=%s value=%r", field, name)
+        raise ValidationError(f"invalid {field}: must match [A-Za-z_][A-Za-z0-9_.]*")
+    if '"' in name or ";" in name or "--" in name:
+        raise ValidationError(f"invalid {field}: forbidden chars")
+    return name
+
+
+def validate_path_under_root(
+    path: str,
+    *,
+    root: str | Path,
+    field: str = "file_path",
+    allow_root: bool = False,
+) -> Path:
+    """Confine a filesystem path to a configured root (LFI / path-traversal guard).
+
+    Resolves symlinks, then asserts the resolved path is relative to root.
+    Rejects symlink escapes and any path that leaves root.
+
+    NOTE (P1-11, TOCTOU): this validates the path AT CALL TIME. resolve() follows
+    symlinks, so a symlink that points inside root at check-time passes; if an
+    attacker swaps it to point outside root between this check and the caller's
+    open(), the guard is defeated (classic TOCTOU). A pure-path validator cannot
+    close this — callers that read untrusted paths MUST open with O_NOFOLLOW (or
+    os.open + lstat-vs-fstat) to make the check-then-use atomic. This function is
+    the first line, not the only line.
+    """
+    if not path:
+        raise ValidationError(f"invalid {field}: empty")
+    if not root:
+        raise ValidationError(f"invalid root for {field}: empty")
+    root_resolved = Path(root).expanduser().resolve()
+    target = Path(path).expanduser().resolve()
+    if not allow_root and target == root_resolved:
+        # P1-11: this branch was a dead `pass` — the docstring claimed it
+        # disallowed operating on root itself, but it never raised, so the
+        # promise was a lie. All current callers validate a file UNDER root
+        # (never the root dir), so rejecting root-equal matches intent. If a
+        # future caller legitimately needs the root, pass allow_root=True.
+        logger.warning("validate_path_under_root reject: field=%s path equals root=%s", field, root_resolved)
+        raise ValidationError(f"invalid {field}: path equals ingest root (not a file under it)")
+    if not target.is_relative_to(root_resolved):
+        logger.warning(
+            "validate_path_under_root reject: field=%s path=%s resolved=%s root=%s",
+            field,
+            path,
+            target,
+            root_resolved,
+        )
+        raise ValidationError(f"invalid {field}: escapes ingest root")
+    # Symlink check: if the raw path is a symlink, ensure it doesn't escape.
+    raw = Path(path)
+    if raw.is_symlink() and not Path(raw.readlink()).resolve().is_relative_to(root_resolved):
+        raise ValidationError(f"invalid {field}: symlink escapes root")
+    return target
+
+
+def _host_is_private(host: str) -> bool:
+    try:
+        addr = ipaddress.ip_address(socket.gethostbyname(host))
+    except (OSError, ValueError):
+        return False
+    return any(addr in net for net in _PRIVATE_NETWORKS)
+
+
+def validate_url(url: str, *, field: str = "url", allow_private: bool = False) -> str:
+    """Validate an HTTP(S) URL — scheme whitelist + SSRF guard.
+
+    Rejects non-http(s) schemes, and (unless allow_private) hosts resolving
+    to private/loopback/link-local ranges.
+    """
+    if not url:
+        raise ValidationError(f"invalid {field}: empty")
+    parsed = urlparse(url)
+    scheme = (parsed.scheme or "").lower()
+    if scheme not in _ALLOWED_URL_SCHEMES:
+        logger.warning("validate_url reject: field=%s scheme=%s", field, scheme)
+        raise ValidationError(f"invalid {field}: scheme must be http/https")
+    host = parsed.hostname or ""
+    if not host:
+        raise ValidationError(f"invalid {field}: no host")
+    if not allow_private and _host_is_private(host):
+        logger.warning("validate_url reject: field=%s host=%s is private", field, host)
+        raise ValidationError(f"invalid {field}: private/loopback host denied")
+    return url
+
+
+def validate_git_url(url: str, *, field: str = "repo_url") -> str:
+    """Validate a git clone URL — scheme whitelist + ext:: RCE guard.
+
+    Rejects ext:: (and any non-whitelisted scheme) which git treats as an
+    executable transport protocol (RCE via post-checkout hook chain).
+    """
+    if not url:
+        raise ValidationError(f"invalid {field}: empty")
+    # Reject ext:: outright — it is an executable transport.
+    if url.startswith("ext::") or "\n" in url or "\0" in url:
+        logger.warning("validate_git_url reject: field=%s ext/nul/newline", field)
+        raise ValidationError(f"invalid {field}: ext:: transport denied")
+    parsed = urlparse(url)
+    scheme = (parsed.scheme or "").lower()
+    if scheme and scheme not in _ALLOWED_GIT_SCHEMES:
+        logger.warning("validate_git_url reject: field=%s scheme=%s", field, scheme)
+        raise ValidationError(f"invalid {field}: scheme must be http/https/git/ssh")
+    # For http/https also apply SSRF guard.
+    if scheme in ("http", "https"):
+        host = parsed.hostname or ""
+        if host and _host_is_private(host):
+            raise ValidationError(f"invalid {field}: private/loopback host denied")
+    return url

fusion_rag/api/__init__.py

@@ -0,0 +1 @@
+"""Fusion-RAG package."""

fusion_rag/api/access.py

@@ -0,0 +1,180 @@
+"""P0-5/P0-6: KB-level access control dependency.
+
+Wires the existing permission module (ACL + PermissionManager) into the REST
+surface — before this, permission rules were CRUD-able but never enforced,
+which is worse than no permissions: admins believed access was restricted when
+it was not.
+
+Semantics:
+- NoAuth (FUSION_RAG_AUTH_BACKEND=none or no admin key): allow. Local-first
+  default — a single-user offline box must not start 401-ing.
+- Authenticated subject == "admin": allow any action (ACL bypass).
+- KB has NO permission rules: allow. An open KB keeps working; enforcement
+  only engages once an admin has actually written a rule for that KB. This is
+  the backward-compat path — existing KBs are not silently locked out.
+- KB HAS rules for the subject: fail-closed. PermissionManager.check_permission
+  decides; a denied check is 403, a backend error is 403 (never allow-on-error).
+
+This dependency runs the same kb_id validation + KB lookup as _get_base, so
+endpoints can keep their own _get_base call (it is a cheap dict get) OR rely
+on the dependency's resolution. To keep the diff surgical, endpoints keep their
+existing _get_base; the dependency resolves independently and its 404/400
+fires before the handler body runs.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import Depends, HTTPException
+
+from .._validators import validate_identifier
+from .app_state import get_kb_manager
+from .auth import verify_api_key
+
+logger = logging.getLogger(__name__)
+
+
+def _resolve_base(kb_id: str):
+    # Mirror of each router's _get_base: confine kb_id then fetch. Duplicated
+    # deliberately (Rule 3) so access.py has no import edge into any router
+    # module — routers import access.py, never the reverse.
+    try:
+        validate_identifier(kb_id, field="kb_id")
+    except ValueError:
+        raise HTTPException(400, f"Invalid kb_id: {kb_id}")
+    # Issue #61: scope by the request's authoritative tenant when isolation is
+    # on. The ACL dependency (require_kb_action) runs BEFORE the handler's
+    # _get_base on most routes — without tenant scoping here a tenant-A caller
+    # would pass the ACL gate on tenant-B's KB. The 404 (not 403) on a
+    # cross-tenant mismatch avoids leaking the KB's existence.
+    from .tenant import tenant_scope
+
+    tenant, require_match = tenant_scope()
+    try:
+        return get_kb_manager().get(kb_id, tenant=tenant, require_tenant_match=require_match)
+    except KeyError:
+        raise HTTPException(404, f"Knowledge base '{kb_id}' not found")
+
+
+def _check_kb_access(kb_id: str, subject: str | None, action: str, resource_path: str = "/") -> None:
+    # NoAuth -> subject is None -> open (local-first default).
+    if subject is None:
+        return
+    # Admin key authenticates as "admin" (auth.py verify) -> ACL bypass.
+    if subject == "admin":
+        return
+    kb = _resolve_base(kb_id)
+    storage_path = kb.vector_path.rsplit("/vectors", 1)[0] if "/vectors" in kb.vector_path else kb.vector_path
+    # A-P1-2: reuse the pooled PermissionManager (app_state admin pool) instead
+    # of constructing one per request — each prior PermissionManager(...) opened
+    # a fresh sqlite conn + re-initialized on every ACL check. One per storage
+    # path, reused; closed from the lifespan shutdown.
+    from .app_state import get_permission_manager
+
+    pm = get_permission_manager(storage_path)
+    # Open-KB path: if no rules exist for this KB at all, treat it as open.
+    # Enforcement only engages once a rule is written (backward compat).
+    try:
+        all_rules = pm.list_rules(kb_id)
+    except Exception as e:
+        # Fail-closed on a permission-store read error — never allow-on-error.
+        logger.error("permission store read failed for kb=%s, denying (fail-closed): %s", kb_id, e)
+        raise HTTPException(403, "Access denied (permission store unavailable)")
+    if not all_rules:
+        logger.debug("access: no permission rules for kb=%s -> open KB, allow", kb_id)
+        return
+    # H4 fix: pass the REAL resource_path, not a hardcoded "/". The prior code
+    # always checked "/" — and every path-prefixed rule matches "/" (rule root
+    # is a prefix of "/"), so path-level ACL was never enforced: a subject
+    # restricted to "/docs/restricted/*" could read "/docs/public/*" too.
+    # KB-scoped endpoints pass "/" (the KB is the resource); doc-specific
+    # endpoints resolve doc_id -> file_path so a rule scoped to a subtree only
+    # permits that subtree.
+    try:
+        allowed = pm.check_permission(kb_id, subject, action, resource_path)
+    except Exception as e:
+        logger.error("permission check failed for kb=%s subject=%s, denying: %s", kb_id, subject, e)
+        raise HTTPException(403, "Access denied")
+    if not allowed:
+        logger.warning("access denied: kb=%s subject=%s action=%s path=%s", kb_id, subject, action, resource_path)
+        raise HTTPException(403, "Access denied")
+    logger.debug("access granted: kb=%s subject=%s action=%s path=%s", kb_id, subject, action, resource_path)
+
+
+def require_admin():
+    """S-P1-3/S-P1-1/S-P1-2: admin-only FastAPI dependency for destructive
+    mutations that must not be reachable by a non-admin subject — API key
+    management (create/delete/list keys) and permission-rule mutation
+    (add/delete rules).
+
+    Semantics match the ACL bypass path: NoAuth (subject is None) -> allow
+    (local-first default); admin subject -> allow; any other authenticated
+    subject -> 403. A stored key can never authenticate as "admin" (auth.py
+    rejects reserved names), so this gate cannot be passed by a non-admin key.
+    """
+    def _dep(subject: str | None = Depends(verify_api_key)) -> str | None:
+        if subject is None:
+            return subject
+        if subject == "admin":
+            return subject
+        logger.warning("admin-only endpoint denied to subject=%s", subject)
+        raise HTTPException(403, "Admin privileges required")
+    return _dep
+
+
+def require_kb_action(action: str, resolve_doc_path: bool = False):
+    """Build a FastAPI dependency enforcing `action` on the path's kb_id.
+
+    Usage: `dependencies=[Depends(require_kb_action("read"))]` on a route that
+    has a {kb_id} path param, or `subject = Depends(require_kb_action("read"))`
+    as a handler param to also receive the subject string.
+
+    resolve_doc_path: for doc-specific routes (/bases/{kb_id}/documents/{doc_id})
+    set True — the dependency resolves doc_id -> the doc's stored file_path and
+    passes it as the ACL resource_path, so a rule scoped to a path subtree only
+    permits that subtree. KB-scoped routes leave it False (resource = "/" = KB).
+    """
+
+    def _dep(
+        kb_id: str,
+        doc_id: str | None = None,
+        subject: str | None = Depends(verify_api_key),
+    ) -> str | None:
+        resource_path = "/"
+        if resolve_doc_path and doc_id:
+            resource_path = _resolve_doc_resource_path(kb_id, doc_id)
+        _check_kb_access(kb_id, subject, action, resource_path)
+        return subject
+
+    return _dep
+
+
+def _resolve_doc_resource_path(kb_id: str, doc_id: str) -> str:
+    # H4: map a path-param doc_id to the stored file_path so ACL rules scoped to
+    # a path subtree can actually enforce at the doc level. If the doc is absent
+    # (404) or the store read errors, fall back to "/" — the handler will 404
+    # on the missing doc anyway, and failing open to "/" here would let a
+    # subtree-restricted subject delete a doc outside its subtree. Instead, on
+    # a store error we deny (fail-closed) by returning a path unlikely to match
+    # any rule, so ACL.check's no-match -> deny fires. A missing doc returns "/"
+    # because the subsequent handler 404s regardless.
+    try:
+        kb = _resolve_base(kb_id)
+    except HTTPException:
+        return "/"
+    storage_path = kb.vector_path.rsplit("/vectors", 1)[0] if "/vectors" in kb.vector_path else kb.vector_path
+    try:
+        # A-P1-1: reuse the pooled MetadataStore (app_state meta pool) instead
+        # of constructing one per doc-level ACL resolution — each prior
+        # MetadataStore(...) opened a fresh sqlite conn on every gated request.
+        from .app_state import get_meta_store
+
+        ms = get_meta_store(f"{storage_path}/metadata.db")
+        doc = ms.get_document(doc_id)
+        if not doc:
+            return "/"
+        return doc.get("file_path", "/") or "/"
+    except Exception as e:
+        logger.warning("access: doc resource_path resolve failed for kb=%s doc=%s: %s", kb_id, doc_id, e)
+        return "/__acl_unresolved__"