ragfabric 0.3.1__py3-none-any.whl

ragfabric-0.3.1.dist-info/METADATA

@@ -0,0 +1,37 @@
+Metadata-Version: 2.5
+Name: ragfabric
+Version: 0.3.1
+Summary: RagFabric: self hosted, measurement first RAG platform. Installs the engine, the API and the ragfabric command.
+Project-URL: Homepage, https://github.com/ranjan-del/ragfabric
+Project-URL: Repository, https://github.com/ranjan-del/ragfabric
+Project-URL: Documentation, https://github.com/ranjan-del/ragfabric/tree/main/docs
+Project-URL: Changelog, https://github.com/ranjan-del/ragfabric/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/ranjan-del/ragfabric/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <3.14,>=3.13
+Requires-Dist: ragfabric-core==0.3.1
+Requires-Dist: ragfabric-sdk==0.3.1
+Requires-Dist: ragfabric-server==0.3.1
+Requires-Dist: typer>=0.27
+Provides-Extra: anthropic
+Requires-Dist: ragfabric-core[anthropic]==0.3.1; extra == 'anthropic'
+Provides-Extra: bm25
+Requires-Dist: ragfabric-core[bm25]==0.3.1; extra == 'bm25'
+Provides-Extra: chroma
+Requires-Dist: ragfabric-core[chroma]==0.3.1; extra == 'chroma'
+Provides-Extra: openai
+Requires-Dist: ragfabric-core[openai]==0.3.1; extra == 'openai'
+Provides-Extra: rerank
+Requires-Dist: ragfabric-core[rerank]==0.3.1; extra == 'rerank'
+Description-Content-Type: text/markdown
+
+# ragfabric
+
+`pip install ragfabric` installs the engine, the HTTP API and the `ragfabric` command. See the repository README.

ragfabric-0.3.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+ragfabric_cli/__init__.py,sha256=8B-VD41-WFEgo-LVSQI5LcLE5UtZ6v38cbFdnwZFjDw,30
+ragfabric_cli/main.py,sha256=_JNdgukCMZY9VKinmC_kwLCwu6sT0C1HGsXdlP-opiU,8071
+ragfabric_cli/commands/__init__.py,sha256=3qHg7CY7ESOstohCc0rYKfSESzpAuiMHEMLm8QPoiHg,41
+ragfabric_cli/commands/access.py,sha256=NvqyyAZBi2DRAV9O-pnCoE-KCJ6E7dCJUhfOMKOLGbg,4563
+ragfabric_cli/commands/ask.py,sha256=ctKRNzBSssmeqMeRjbIIYnAGSultK4DU0ix-KImoZ5k,8494
+ragfabric_cli/commands/common.py,sha256=zR958BOF4RKqCc1k3xmJUq42FjTNfkmjkSczyjueqHg,1074
+ragfabric_cli/commands/graph.py,sha256=vIUbVF9TV2Ki-08R6CZSjv6q789DpfRpG-35bmuUsvo,5152
+ragfabric_cli/commands/ingest.py,sha256=35MXrSJJhj-vSbsD_Rpn2qpSBUCG4_KMptI-BzmBc0c,1845
+ragfabric_cli/commands/reconcile.py,sha256=tXvtoEdORD55DpO97vr_Gy8yhJL6eobv5ISFIQv6je4,2737
+ragfabric_cli/commands/reindex.py,sha256=v7EDk5A-lzDepTuLvfU204ut9dtuUXx2_-7PSLInjWg,2636
+ragfabric_cli/commands/users.py,sha256=J-a4xQ8u0nIGcLKpEL560G6iXXr0X1pNV7sipH1suIk,1735
+ragfabric_cli/commands/worker.py,sha256=IJ2DQvSvaZJlesld4AF6DvVECC0UFMUZVDqK0Lm_flg,1932
+ragfabric-0.3.1.dist-info/METADATA,sha256=-qLEqDqtoD8b9wpbA1TP7Y6YjYSEQ5LvW0DnhubFPu8,1686
+ragfabric-0.3.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+ragfabric-0.3.1.dist-info/entry_points.txt,sha256=Gw-r4m1ceIpGfI4sDCoDpwto3BHIyhYsBpzePF7FGdk,53
+ragfabric-0.3.1.dist-info/RECORD,,

ragfabric-0.3.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

ragfabric-0.3.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ragfabric = ragfabric_cli.main:app

ragfabric_cli/__init__.py

@@ -0,0 +1 @@
+"""RagFabric command line."""

ragfabric_cli/commands/__init__.py

@@ -0,0 +1 @@
+"""Sub commands of the ragfabric CLI."""

ragfabric_cli/commands/access.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import typer
+
+from ragfabric_cli.commands.common import collection_by_name, session, user_by_email
+from ragfabric_core.auth import service
+from ragfabric_core.auth.api_keys import create_api_key
+from ragfabric_core.models.access import ApiKey, CollectionGrant, Group
+from ragfabric_core.models.document import Collection
+
+groups_app = typer.Typer(help="Groups and membership.")
+grants_app = typer.Typer(help="Collection grants.")
+keys_app = typer.Typer(help="API keys.")
+
+
+def _group(db, name: str) -> Group:
+    group = db.query(Group).filter(Group.name == name).first()
+    if group is None:
+        typer.echo(f"group not found: {name}")
+        raise typer.Exit(code=1)
+    return group
+
+
+@groups_app.command("create")
+def create_group(name: str, description: str = typer.Option("")) -> None:
+    with session() as db:
+        if db.query(Group).filter(Group.name == name).first() is not None:
+            typer.echo(f"group already exists: {name}")
+            raise typer.Exit(code=1)
+        service.create_group(db, name, description)
+        db.commit()
+    typer.echo(f"created group {name}")
+
+
+@groups_app.command("add-member")
+def add_member(group_name: str, email: str) -> None:
+    with session() as db:
+        group = _group(db, group_name)
+        user = user_by_email(db, email)
+        service.add_member(db, group.id, user.id)
+        db.commit()
+    typer.echo(f"added {email} to {group_name}")
+
+
+@groups_app.command("list")
+def list_groups() -> None:
+    with session() as db:
+        for g in db.query(Group).order_by(Group.name).all():
+            members = service_members(db, g.id)
+            typer.echo(f"{g.id}\t{g.name}\t{members} member(s)")
+
+
+def service_members(db, group_id: int) -> int:
+    from ragfabric_core.models.access import GroupMember
+
+    return db.query(GroupMember).filter(GroupMember.group_id == group_id).count()
+
+
+@grants_app.command("add")
+def add_grant(
+    group: str = typer.Option(...),
+    collection: str = typer.Option(...),
+    permission: str = typer.Option("read"),
+) -> None:
+    with session() as db:
+        g = _group(db, group)
+        c = collection_by_name(db, collection)
+        try:
+            service.grant_collection(db, g.id, c.id, permission)
+        except ValueError as exc:
+            typer.echo(str(exc))
+            raise typer.Exit(code=1) from None
+        db.commit()
+    typer.echo(f"granted {permission} on {collection} to {group}")
+
+
+@grants_app.command("list")
+def list_grants() -> None:
+    with session() as db:
+        rows = (
+            db.query(CollectionGrant, Group.name, Collection.name)
+            .join(Group, Group.id == CollectionGrant.group_id)
+            .join(Collection, Collection.id == CollectionGrant.collection_id)
+            .all()
+        )
+        for grant, group_name, collection_name in rows:
+            typer.echo(f"{grant.id}\t{group_name}\t{collection_name}\t{grant.permission}")
+
+
+@keys_app.command("create")
+def create_key(
+    name: str = typer.Option(...),
+    user: str = typer.Option(...),
+    collection: list[str] = typer.Option([], "--collection"),
+    strategy: list[str] = typer.Option([], "--strategy"),
+    rate_limit: int | None = typer.Option(
+        None,
+        "--rate-limit",
+        help="Requests/minute; defaults to limits.rate_limit_per_minute in ragfabric.yaml.",
+    ),
+) -> None:
+    with session() as db:
+        owner = user_by_email(db, user)
+        collection_ids = [collection_by_name(db, c).id for c in collection]
+        key, plaintext = create_api_key(
+            db,
+            name=name,
+            user_id=owner.id,
+            collection_ids=collection_ids,
+            strategies=strategy,
+            rate_limit_per_minute=rate_limit,
+        )
+        db.commit()
+        typer.echo(f"created key {key.id} ({name}) for {user}")
+    typer.echo("store this now, it is not shown again:")
+    typer.echo(plaintext)
+
+
+@keys_app.command("list")
+def list_keys() -> None:
+    with session() as db:
+        for k in db.query(ApiKey).order_by(ApiKey.id).all():
+            typer.echo(
+                f"{k.id}\t{k.name}\t{k.key_prefix}...\t{'active' if k.is_active else 'revoked'}\tlimit {k.rate_limit_per_minute}/min"
+            )
+
+
+@keys_app.command("revoke")
+def revoke_key(key_id: int) -> None:
+    with session() as db:
+        key = db.get(ApiKey, key_id)
+        if key is None:
+            typer.echo(f"key not found: {key_id}")
+            raise typer.Exit(code=1)
+        key.is_active = False
+        db.commit()
+    typer.echo(f"revoked key {key_id}")

ragfabric_cli/commands/ask.py

@@ -0,0 +1,197 @@
+"""ragfabric ask: ask a running RagFabric server a question.
+
+Goes through the SDK over HTTP rather than calling core in process: the
+normal case is a remote server, and a second in process code path would
+drift from what API users experience. The cost is that this command needs a
+running server to talk to; the help text says so.
+"""
+
+from __future__ import annotations
+
+import os
+from enum import StrEnum
+
+import typer
+
+from ragfabric_sdk import Client
+from ragfabric_sdk.errors import RagFabricError
+
+DEFAULT_URL = "http://localhost:8000"
+
+
+class Strategy(StrEnum):
+    """Retrieval strategies this command can ask the server for.
+
+    An Enum rather than a free string so Typer refuses an unknown name while
+    parsing, before the command body runs. A typo then costs nothing: no
+    request leaves the machine, no embedding call is spent, and the error
+    names the valid choices instead of arriving as a 422 from the server.
+    The members mirror the server's own Literal in schemas/search.py; the
+    server stays the authority and still validates what it is sent.
+    """
+
+    traditional = "traditional"
+    vectorless = "vectorless"
+    agentic = "agentic"
+    graph = "graph"
+
+
+def ask(
+    question: str = typer.Argument(..., help="The question to ask."),
+    url: str = typer.Option(
+        None, "--url", help=f"Server URL (env RAGFABRIC_URL, default {DEFAULT_URL})."
+    ),
+    token: str = typer.Option(None, "--token", help="JWT bearer token (env RAGFABRIC_TOKEN)."),
+    api_key: str = typer.Option(None, "--api-key", help="rf_ API key (env RAGFABRIC_API_KEY)."),
+    top_k: int = typer.Option(8, "--top-k", min=1, max=50, help="Chunks to retrieve."),
+    threshold: float = typer.Option(
+        0.0, "--threshold", min=0.0, max=1.0, help="Similarity threshold."
+    ),
+    collection: int = typer.Option(None, "--collection", help="Collection id to search."),
+    strategy: Strategy = typer.Option(
+        Strategy.traditional,
+        "--strategy",
+        help=(
+            "Retrieval strategy. traditional embeds the question and searches the "
+            "vector index; vectorless ranks with BM25 fused with ts_rank_cd and "
+            "never calls an embedding model; agentic splits the question into "
+            "parts, retrieves per part, and repairs or abandons the parts it "
+            "cannot answer, reporting which those were; graph walks the knowledge "
+            "graph from the entities the question names and prints the relationships "
+            "it walked and any relationship claims the citation contract dropped."
+        ),
+    ),
+    no_stream: bool = typer.Option(
+        False, "--no-stream", help="Wait for the whole answer instead of streaming tokens."
+    ),
+    as_json: bool = typer.Option(
+        False, "--json", help="Print the whole answer payload as JSON (implies --no-stream)."
+    ),
+) -> None:
+    """Ask a question and print the cited answer.
+
+    Needs a running RagFabric server: this command talks to it over HTTP
+    through the RagFabric SDK, it does not answer the question locally.
+    Streamed stdout can contain a stale, superseded draft ahead of the
+    corrected answer (bytes already printed cannot be recalled); a machine
+    consumer should use --json or --no-stream instead of parsing the stream.
+    """
+    url = url or os.environ.get("RAGFABRIC_URL") or DEFAULT_URL
+    token = token or os.environ.get("RAGFABRIC_TOKEN")
+    api_key = api_key or os.environ.get("RAGFABRIC_API_KEY")
+    if not token and not api_key:
+        typer.echo(
+            "no credentials: pass --token or --api-key, or set RAGFABRIC_TOKEN "
+            "or RAGFABRIC_API_KEY",
+            err=True,
+        )
+        raise typer.Exit(2)
+
+    params: dict[str, object] = {
+        "top_k": top_k,
+        "similarity_threshold": threshold,
+        "strategy": strategy.value,
+    }
+    if collection is not None:
+        params["collection_id"] = collection
+
+    client = Client(url, token=token, api_key=api_key)
+    try:
+        if as_json or no_stream:
+            answer = client.ask(question, **params)
+            if as_json:
+                typer.echo(answer.model_dump_json(indent=2))
+            else:
+                typer.echo(answer.answer)
+                typer.echo("")
+                _print_sources([citation.model_dump() for citation in answer.citations])
+                _print_graph(
+                    answer.subgraph.model_dump() if answer.subgraph is not None else None,
+                    [claim.model_dump() for claim in answer.dropped_relationship_claims],
+                )
+            return
+
+        citations: list[dict] = []
+        subgraph: dict | None = None
+        dropped_relationship_claims: list[dict] = []
+        run_id = None
+        latency_ms = None
+        for event in client.ask_stream(question, **params):
+            if event.event == "retrieval":
+                subgraph = event.data.get("subgraph")
+            elif event.event == "token":
+                typer.echo(event.data.get("text", ""), nl=False)
+            elif event.event == "superseded":
+                # The caller already saw the rejected tokens printed above,
+                # and there is no way to un-print a terminal: say so on
+                # stderr (never mixed into a piped stdout file) and then put
+                # the full corrected answer on stdout so a script reading
+                # stdout still ends up with the complete, correct text, even
+                # though it is preceded by the stale draft.
+                typer.echo(
+                    "\nnotice: the streamed answer above failed the citation "
+                    "contract and was corrected; the corrected answer follows.",
+                    err=True,
+                )
+                typer.echo("\n")
+                typer.echo(event.data.get("text", ""), nl=False)
+                dropped_relationship_claims = event.data.get("dropped_relationship_claims", [])
+            elif event.event == "citations":
+                citations = event.data.get("citations", [])
+            elif event.event == "done":
+                run_id = event.data.get("run_id")
+                latency_ms = event.data.get("latency_ms")
+        typer.echo("")
+        _print_sources(citations)
+        _print_graph(subgraph, dropped_relationship_claims)
+        if run_id is not None:
+            typer.echo(f"run {run_id} in {latency_ms}ms")
+    except RagFabricError as exc:
+        typer.echo(f"error: {exc}", err=True)
+        raise typer.Exit(1) from exc
+    except Exception as exc:  # connection refused, DNS failure, timeout
+        typer.echo(f"could not reach {url}: {exc}", err=True)
+        raise typer.Exit(1) from exc
+    finally:
+        client.close()
+
+
+def _print_sources(citations: list[dict]) -> None:
+    used = [citation for citation in citations if citation.get("used")]
+    if not used:
+        return
+    typer.echo("sources:")
+    for citation in used:
+        name = citation.get("filename") or f"document {citation.get('document_id')}"
+        page = f" p{citation['page']}" if citation.get("page") else ""
+        typer.echo(f"  {citation['marker']} {name}{page}")
+
+
+def _print_graph(subgraph: dict | None, dropped: list[dict]) -> None:
+    """The relationships the graph strategy walked, and the claims about them it dropped.
+
+    Printed as ``name RELATION name`` in the direction walked, the same reading
+    the answer's ``[E k]`` markers were numbered against. Nothing is printed
+    for a strategy that walks no graph.
+    """
+    if subgraph is not None:
+        names = {node["id"]: node["name"] for node in subgraph.get("nodes", [])}
+        edges = subgraph.get("edges", [])
+        if edges:
+            typer.echo("graph:")
+            for number, edge in enumerate(edges, start=1):
+                start, end = edge["source_id"], edge["target_id"]
+                if edge.get("reversed"):
+                    start, end = end, start
+                typer.echo(
+                    f"  [E {number}] {names.get(start, start)} {edge['walked_as']} "
+                    f"{names.get(end, end)}"
+                )
+        elif subgraph.get("empty_reason"):
+            typer.echo(f"graph: nothing walked ({subgraph['empty_reason']})")
+        if subgraph.get("truncated"):
+            typer.echo("graph: the walk was cut by the node budget")
+    if dropped:
+        typer.echo("dropped relationship claims:")
+        for claim in dropped:
+            typer.echo(f"  {claim['reason']}: {claim['text']}")

ragfabric_cli/commands/common.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+
+import typer
+from sqlalchemy.orm import Session
+
+from ragfabric_core.models.document import Collection
+from ragfabric_core.models.user import User
+
+
+@contextmanager
+def session():
+    from ragfabric_core.runtime import get_session_factory
+
+    db: Session = get_session_factory()()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+def user_by_email(db: Session, email: str) -> User:
+    user = db.query(User).filter(User.email == email.lower()).first()
+    if user is None:
+        typer.echo(f"user not found: {email}")
+        raise typer.Exit(code=1)
+    return user
+
+
+def collection_by_name(db: Session, name: str, create: bool = False) -> Collection:
+    collection = db.query(Collection).filter(Collection.name == name).first()
+    if collection is None:
+        if not create:
+            typer.echo(f"collection not found: {name}")
+            raise typer.Exit(code=1)
+        collection = Collection(name=name)
+        db.add(collection)
+        db.flush()
+    return collection

ragfabric_cli/commands/graph.py

@@ -0,0 +1,125 @@
+"""ragfabric graph merges: inspect and undo recorded entity merges (ruling R37).
+
+An operator command, like ``users``, ``groups``, ``grants`` and ``keys``: it
+opens the configured database directly and acts on it, so running it at all
+requires the database credentials only an administrator holds. There is no
+HTTP endpoint for merges in this phase; the Phase 9 console owns that.
+
+Every merge entity resolution makes is recorded as an ``EntityMerge`` row with
+the evidence that justified it (Task 5). Without a way to read and reverse
+those rows the record would be write-only, so ``list`` and ``show`` read them
+and ``undo`` calls ``graph.resolve.unmerge`` and commits only if it succeeds.
+Unmerge is globally last-in, first-out (ruling R30): an older merge cannot be
+undone while a newer one is in place, and ``undo`` says which ones block it
+and exits non-zero rather than attempting a partial repair.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import typer
+
+from ragfabric_cli.commands.common import session
+from ragfabric_core.graph.resolve import UnmergeBlocked, UnmergeResult, unmerge
+from ragfabric_core.models.graph import Entity, EntityMerge
+
+graph_app = typer.Typer(help="Knowledge graph administration.")
+merges_app = typer.Typer(help="Recorded entity merges: list, show and undo.")
+graph_app.add_typer(merges_app, name="merges")
+
+_RESULT_FIELDS = (
+    "restored_entity_id",
+    "survivor_entity_id",
+    "moved_relationship_ids",
+    "shared_relationship_ids",
+    "restored_relationship_ids",
+    "unrestored_relationship_ids",
+    "changed_chunk_ids",
+    "restored_without_sources",
+)
+
+
+def _survivor_name(db, entity_id: int) -> str:
+    entity = db.get(Entity, entity_id)
+    return entity.name if entity is not None else "?"
+
+
+@merges_app.command("list")
+def list_merges() -> None:
+    """Every recorded merge, oldest first. Undo them newest first."""
+    with session() as db:
+        records = db.query(EntityMerge).order_by(EntityMerge.id).all()
+        if not records:
+            typer.echo("no recorded merges")
+            return
+        for record in records:
+            typer.echo(
+                f"{record.id}\t{record.method}\t{record.merged_name} ({record.merged_entity_type})"
+                f" -> {record.surviving_entity_id} {_survivor_name(db, record.surviving_entity_id)}"
+                f"\t{len(record.merged_source_chunk_ids)} chunk(s)\t{record.created_at:%Y-%m-%d}"
+            )
+
+
+def _record(db, merge_id: int) -> EntityMerge:
+    record = db.get(EntityMerge, merge_id)
+    if record is None:
+        typer.echo(f"merge not found: {merge_id}", err=True)
+        raise typer.Exit(code=1)
+    return record
+
+
+@merges_app.command("show")
+def show_merge(merge_id: int) -> None:
+    """One merge: what was merged into what, by which method, and the evidence for it."""
+    with session() as db:
+        record = _record(db, merge_id)
+        evidence: dict[str, Any] = dict(record.evidence)
+        restore: dict[str, Any] = evidence.pop("restore", {}) or {}
+        typer.echo(f"merge {record.id} ({record.method}, {record.created_at:%Y-%m-%d %H:%M})")
+        typer.echo(f"merged: {record.merged_name} ({record.merged_entity_type})")
+        typer.echo(
+            f"into: {record.surviving_entity_id} {_survivor_name(db, record.surviving_entity_id)}"
+        )
+        typer.echo(f"model: {record.model or 'none'}")
+        typer.echo(f"merged aliases: {', '.join(record.merged_aliases) or 'none'}")
+        typer.echo(f"source chunks: {sorted(record.merged_source_chunk_ids)}")
+        typer.echo("evidence:")
+        typer.echo(json.dumps(evidence, indent=2, sort_keys=True, default=str))
+        # The restore payload is what makes the unmerge exact, not justification;
+        # summarised here so an operator can see what an undo would touch.
+        typer.echo(
+            f"undo would restore: {len(restore.get('relationships', []))} recorded "
+            f"relationship(s), {len(restore.get('dropped_self_loops', []))} dropped self "
+            f"loop(s), aliases appended {restore.get('aliases_appended', [])}"
+        )
+
+
+@merges_app.command("undo")
+def undo_merge(merge_id: int) -> None:
+    """Reverse one merge. Refused while any newer merge is still in place."""
+    with session() as db:
+        try:
+            result = unmerge(db, merge_id)
+        except UnmergeBlocked as exc:
+            db.rollback()
+            blocking = " ".join(str(i) for i in exc.blocking_merge_ids)
+            typer.echo(
+                f"cannot undo merge {merge_id}: blocked by later merges: {blocking} "
+                "(undo those first, newest first)",
+                err=True,
+            )
+            raise typer.Exit(code=1) from None
+        except (LookupError, ValueError) as exc:
+            db.rollback()
+            typer.echo(f"cannot undo merge {merge_id}: {exc}", err=True)
+            raise typer.Exit(code=1) from None
+        db.commit()
+    typer.echo(f"undid merge {merge_id}")
+    _print_result(result)
+
+
+def _print_result(result: UnmergeResult) -> None:
+    for field in _RESULT_FIELDS:
+        typer.echo(f"{field}: {getattr(result, field)}")

ragfabric_cli/commands/ingest.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from ragfabric_cli.commands.common import collection_by_name, session, user_by_email
+from ragfabric_core.ingest.parser import SUPPORTED_FORMATS
+from ragfabric_core.ingest.pipeline import ingest_document
+
+
+def _files(path: Path, recursive: bool) -> list[Path]:
+    if path.is_file():
+        return [path]
+    pattern = "**/*" if recursive else "*"
+    return sorted(
+        p
+        for p in path.glob(pattern)
+        if p.is_file() and p.suffix.lower().lstrip(".") in SUPPORTED_FORMATS
+    )
+
+
+def ingest(
+    path: Path = typer.Argument(..., exists=True),
+    collection: str | None = typer.Option(None, "--collection"),
+    recursive: bool = typer.Option(False, "--recursive"),
+    owner: str | None = typer.Option(None, "--owner", help="Email of the owning user."),
+) -> None:
+    """Ingest one file or every supported file in a directory."""
+    files = _files(path, recursive)
+    if not files:
+        typer.echo("no supported files found")
+        raise typer.Exit(code=1)
+    failed = 0
+    with session() as db:
+        collection_id = collection_by_name(db, collection, create=True).id if collection else None
+        owner_id = user_by_email(db, owner).id if owner else None
+        db.commit()
+        for file in files:
+            doc = ingest_document(
+                db,
+                filename=file.name,
+                data=file.read_bytes(),
+                collection_id=collection_id,
+                owner_id=owner_id,
+            )
+            typer.echo(
+                f"{file.name}: {doc.status} ({doc.num_chunks} chunks){' ' + doc.error if doc.error else ''}"
+            )
+            failed += doc.status == "failed"
+    typer.echo(f"{len(files) - failed} ingested, {failed} failed")
+    if failed:
+        raise typer.Exit(code=1)

ragfabric_cli/commands/reconcile.py

@@ -0,0 +1,60 @@
+"""ragfabric reconcile: retry documents stuck mid fan-out.
+
+index_document writes the vector store and the lexical store as two separate
+commits (workers/handlers.py), so a hard crash between them leaves a document
+stuck in "indexing" forever with no exception for anything to have caught.
+This command is the operator-triggered retry: it finds every document stuck
+in "indexing" past the grace period and re-runs indexing for it, which is
+safe because index_document is idempotent (see workers/handlers.py).
+
+Known limitation: "stuck" is judged purely by age, with no lock or worker
+heartbeat behind it. A document that is merely slow (a large file, or a
+rate-limited embedding provider) rather than crashed can be re-run by this
+command WHILE a live worker is still processing it. The result still
+converges, since the underlying writes are idempotent, but the embedding
+provider gets called a second time, which costs money and can itself trigger
+the same rate limiting that made the document slow. The safe procedure is to
+stop the worker(s) before running this command.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from ragfabric_cli.commands.common import session
+from ragfabric_core.providers.registry import build_embedding_provider
+from ragfabric_core.runtime import get_config, get_session_factory
+from ragfabric_core.stores.registry import build_lexical_store, build_vector_store
+from ragfabric_core.workers.handlers import reconcile_stuck_indexing
+
+
+def reconcile(
+    older_than_seconds: int = typer.Option(
+        300,
+        "--older-than",
+        min=0,
+        help="Only retry documents that have been stuck in 'indexing' for at least this long.",
+    ),
+) -> None:
+    """Retry any document left stuck in 'indexing' by a crashed fan out.
+
+    Limitation: a document merely running slowly, not crashed, can be re-run
+    here while a worker is still indexing it. Writes converge (idempotent),
+    but the embedding provider is called twice, which costs money and can
+    trigger the same rate limit that slowed the document down. For that
+    reason, stop the worker(s) before running this command.
+    """
+    cfg = get_config()
+    sf = get_session_factory()
+    provider = build_embedding_provider(cfg.embeddings)
+    vector_store = build_vector_store(cfg.vector_store, sf, embedding_model=provider.model)
+    lexical_store = build_lexical_store(cfg.lexical_store, sf)
+    with session() as db:
+        count = reconcile_stuck_indexing(
+            db,
+            embedding_provider=provider,
+            vector_store=vector_store,
+            lexical_store=lexical_store,
+            older_than_seconds=older_than_seconds,
+        )
+    typer.echo(f"reconciled {count} document(s) stuck in indexing")

ragfabric_cli/commands/reindex.py

@@ -0,0 +1,69 @@
+"""ragfabric reindex: re-embed the corpus under the active embedding model."""
+
+from __future__ import annotations
+
+import typer
+
+from ragfabric_cli.commands.common import session
+from ragfabric_core.ingest.reindex import reindex_all
+from ragfabric_core.providers.registry import build_embedding_provider
+from ragfabric_core.runtime import get_config, get_session_factory
+from ragfabric_core.stores.registry import build_lexical_store, build_vector_store
+
+
+def reindex(
+    batch_size: int = typer.Option(64, "--batch-size", min=1, help="Chunks per embedding call."),
+    document_id: int | None = typer.Option(None, "--document-id", help="Limit to one document."),
+    lexical_only: bool = typer.Option(
+        False,
+        "--lexical-only",
+        help="Rebuild only the lexical index. No embedding calls, no vector writes.",
+    ),
+    yes: bool = typer.Option(False, "--yes", help="Do not ask for confirmation."),
+) -> None:
+    """Re-embed every chunk with the configured embedding model.
+
+    With --lexical-only, rebuild the lexical index instead. That is what a
+    corpus indexed before BM25 needs: those chunk_search rows carry doc_len = 0
+    and a tsv with no term frequency, so BM25 excludes them, and re-embedding
+    the whole corpus to fix a lexical column would cost provider money for
+    nothing.
+    """
+    cfg = get_config()
+    sf = get_session_factory()
+    provider = None if lexical_only else build_embedding_provider(cfg.embeddings)
+    if lexical_only:
+        typer.echo("This rebuilds the lexical index only. No vectors are touched.")
+    else:
+        typer.echo(
+            f"This replaces every stored vector using {provider.model} ({provider.dim} dims)."
+        )
+    if not yes and not typer.confirm("Continue?"):
+        typer.echo("aborted")
+        raise typer.Exit(1)
+
+    vector_store = (
+        None
+        if lexical_only
+        else build_vector_store(cfg.vector_store, sf, embedding_model=provider.model)
+    )
+    lexical_store = build_lexical_store(cfg.lexical_store, sf)
+
+    def progress(done: int, total: int) -> None:
+        typer.echo(f"  {done}/{total}")
+
+    with session() as db:
+        count = reindex_all(
+            db,
+            embedding_provider=provider,
+            vector_store=vector_store,
+            lexical_store=lexical_store,
+            lexical_only=lexical_only,
+            batch_size=batch_size,
+            document_id=document_id,
+            on_progress=progress,
+        )
+    if lexical_only:
+        typer.echo(f"rebuilt the lexical index for {count} chunks")
+    else:
+        typer.echo(f"re-embedded {count} chunks with {provider.model}")

ragfabric_cli/commands/users.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import typer
+
+from ragfabric_cli.commands.common import session, user_by_email
+from ragfabric_core.models.user import Role, User
+from ragfabric_core.security import hash_password
+
+app = typer.Typer(help="Users.")
+
+
+@app.command("create")
+def create(
+    email: str = typer.Option(...),
+    password: str = typer.Option(..., prompt=False),
+    role: str = typer.Option("user"),
+) -> None:
+    if role not in {Role.ADMIN.value, Role.USER.value}:
+        typer.echo("role must be user or admin")
+        raise typer.Exit(code=1)
+    with session() as db:
+        if db.query(User).filter(User.email == email.lower()).first() is not None:
+            typer.echo(f"user already exists: {email}")
+            raise typer.Exit(code=1)
+        db.add(User(email=email.lower(), hashed_password=hash_password(password), role=role))
+        db.commit()
+    typer.echo(f"created {email} ({role})")
+
+
+@app.command("list")
+def list_users() -> None:
+    with session() as db:
+        for u in db.query(User).order_by(User.id).all():
+            typer.echo(f"{u.id}\t{u.email}\t{u.role}\t{'active' if u.is_active else 'inactive'}")
+
+
+@app.command("set-role")
+def set_role(email: str, role: str) -> None:
+    if role not in {Role.ADMIN.value, Role.USER.value}:
+        typer.echo("role must be user or admin")
+        raise typer.Exit(code=1)
+    with session() as db:
+        user = user_by_email(db, email)
+        user.role = role
+        db.commit()
+    typer.echo(f"{email} is now {role}")
+
+
+@app.command("deactivate")
+def deactivate(email: str) -> None:
+    with session() as db:
+        user = user_by_email(db, email)
+        user.is_active = False
+        db.commit()
+    typer.echo(f"{email} deactivated")

ragfabric_cli/commands/worker.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import logging
+import threading
+
+import typer
+
+from ragfabric_core.providers.registry import build_embedding_provider, build_llm_provider
+from ragfabric_core.queue.registry import build_queue
+from ragfabric_core.runtime import get_config, get_session_factory
+from ragfabric_core.stores.registry import build_lexical_store, build_vector_store
+from ragfabric_core.workers.runner import Worker, default_handlers, install_sigterm_handler
+
+
+def _stores():
+    cfg = get_config()
+    sf = get_session_factory()
+    provider = build_embedding_provider(cfg.embeddings)
+    return (
+        build_vector_store(cfg.vector_store, sf, embedding_model=provider.model),
+        build_lexical_store(cfg.lexical_store, sf),
+    )
+
+
+def worker(
+    once: bool = typer.Option(False, "--once", help="Process at most one job and exit."),
+) -> None:
+    """Run the indexing worker against the configured queue."""
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    cfg = get_config()
+    queue = build_queue(cfg)
+    if queue is None:
+        typer.echo(
+            "ingestion.indexing is inline; set `indexing: queue` in ragfabric.yaml to run a worker"
+        )
+        raise typer.Exit(code=1)
+    sf = get_session_factory()
+    vector_store, lexical_store = _stores()
+    handlers = default_handlers(
+        embedding_provider=build_embedding_provider(cfg.embeddings),
+        vector_store=vector_store,
+        lexical_store=lexical_store,
+        graph_settings=cfg.graph_store,
+        llm=build_llm_provider(cfg.llm) if cfg.graph_store.enabled else None,
+    )
+    w = Worker(queue, sf, handlers)
+    if once:
+        typer.echo("processed 1 job" if w.run_once(timeout_seconds=1.0) else "queue empty")
+        return
+    stop = threading.Event()
+    install_sigterm_handler(stop)
+    typer.echo("worker started; waiting for jobs")
+    w.run_forever(stop)
+    typer.echo("worker stopped")

ragfabric_cli/main.py

@@ -0,0 +1,206 @@
+"""The ragfabric command.
+
+Thin by design: every command calls a function in ragfabric_core or starts
+ragfabric_server. No logic lives here that the API or the tests cannot reach
+another way.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import typer
+from pydantic import ValidationError
+
+from ragfabric_cli.commands import access as access_commands
+from ragfabric_cli.commands import graph as graph_commands
+from ragfabric_cli.commands import ingest as ingest_commands
+from ragfabric_cli.commands import users as users_commands
+from ragfabric_cli.commands.ask import ask as ask_command
+from ragfabric_cli.commands.reconcile import reconcile as reconcile_command
+from ragfabric_cli.commands.reindex import reindex as reindex_command
+from ragfabric_cli.commands.worker import worker as worker_command
+from ragfabric_core import __version__
+from ragfabric_core.config_file import load_config, resolve_config_path
+from ragfabric_core.db import migrate
+from ragfabric_core.providers.base import ProviderError
+from ragfabric_core.providers.registry import build_embedding_provider, build_llm_provider
+
+app = typer.Typer(
+    help="RagFabric: self hosted, measurement first RAG platform.", no_args_is_help=True
+)
+db_app = typer.Typer(help="Database migrations.")
+config_app = typer.Typer(help="Configuration.")
+app.add_typer(db_app, name="db")
+app.add_typer(config_app, name="config")
+app.add_typer(users_commands.app, name="users")
+app.add_typer(access_commands.groups_app, name="groups")
+app.add_typer(access_commands.grants_app, name="grants")
+app.add_typer(access_commands.keys_app, name="keys")
+app.add_typer(graph_commands.graph_app, name="graph")
+app.command("ingest")(ingest_commands.ingest)
+app.command("worker")(worker_command)
+app.command("reindex")(reindex_command)
+app.command("reconcile")(reconcile_command)
+app.command("ask")(ask_command)
+
+
+def _database_url() -> str:
+    from ragfabric_core.config import get_settings
+
+    return os.environ.get("DATABASE_URL") or get_settings().database_url
+
+
+@app.command()
+def version() -> None:
+    """Print the installed version."""
+    typer.echo(f"ragfabric {__version__}")
+
+
+@db_app.command()
+def upgrade(revision: str = typer.Option("head", help="Target revision.")) -> None:
+    """Apply migrations up to REVISION (default head)."""
+    migrate.upgrade(_database_url(), revision)
+    typer.echo(f"database upgraded to {revision}")
+
+
+@db_app.command()
+def downgrade(revision: str = typer.Option("base", help="Target revision.")) -> None:
+    """Roll migrations back to REVISION (default base, which removes every table)."""
+    migrate.downgrade(_database_url(), revision)
+    typer.echo(f"database downgraded to {revision}")
+
+
+@config_app.command("validate")
+def config_validate(
+    path: Path | None = typer.Option(None, "--path", help="ragfabric.yaml to validate."),
+    check_providers: bool = typer.Option(
+        False, "--check-providers", help="Also make one live call per provider."
+    ),
+) -> None:
+    """Load the configuration and report the active implementation for each interface."""
+    resolved = resolve_config_path(path)
+    typer.echo(f"config file: {resolved if resolved else 'none (defaults)'}")
+    try:
+        cfg = load_config(path)
+    except ValidationError as exc:
+        typer.echo("configuration is invalid:")
+        for err in exc.errors():
+            typer.echo(f"  {'.'.join(str(p) for p in err['loc'])}: {err['msg']}")
+        raise typer.Exit(code=1) from None
+
+    problems: list[str] = []
+    try:
+        llm = build_llm_provider(cfg.llm)
+        typer.echo(f"llm: {cfg.llm.provider} ({llm.default_model})")
+    except ProviderError as exc:
+        problems.append(str(exc))
+    try:
+        emb = build_embedding_provider(cfg.embeddings)
+        typer.echo(f"embeddings: {cfg.embeddings.provider} ({emb.model})")
+    except ProviderError as exc:
+        problems.append(str(exc))
+    typer.echo(f"reranker: {cfg.reranker.kind}")
+    typer.echo(f"vector_store: {cfg.vector_store.kind}")
+    typer.echo(f"lexical_store: {cfg.lexical_store.kind}")
+    vectorless = cfg.strategies.vectorless
+    typer.echo(
+        f"vectorless: top_k {vectorless.top_k}, k1 {vectorless.k1}, b {vectorless.b}, "
+        f"fusion_k {vectorless.fusion_k}, "
+        f"boosts phrase {vectorless.phrase_boost} identifier {vectorless.identifier_boost}"
+    )
+    agentic = cfg.strategies.agentic
+    typer.echo(
+        f"agentic: max_iterations {agentic.max_iterations}, "
+        f"max_llm_calls {agentic.max_llm_calls}, "
+        f"max_cost_usd {agentic.max_cost_usd}, max_latency_ms {agentic.max_latency_ms}, "
+        f"assess {agentic.assess_strictness}, tools {' '.join(agentic.tools)}"
+    )
+    graph_store = cfg.graph_store
+    typer.echo(
+        f"graph_store: {graph_store.kind} ({'enabled' if graph_store.enabled else 'disabled'}), "
+        f"extraction_model {graph_store.extraction_model or 'provider default'}, "
+        f"confidence_floor {graph_store.confidence_floor}, "
+        f"similarity_threshold {graph_store.similarity_threshold}, "
+        f"entity_types {' '.join(graph_store.entity_types)}, "
+        f"relation_types {' '.join(graph_store.relation_types)}"
+    )
+    graph = cfg.strategies.graph
+    typer.echo(f"graph: max_hops {graph.max_hops}, node_budget {graph.node_budget}")
+    typer.echo(f"cache: {cfg.cache.kind}")
+    typer.echo(f"router: {cfg.router.mode} (min_confidence {cfg.router.min_confidence})")
+
+    if check_providers and not problems:
+        from ragfabric_core.providers.base import Message
+
+        try:
+            out = llm.complete(
+                [Message(role="user", content="Reply with the single word: pong")], max_tokens=8
+            )
+            typer.echo(
+                f"llm live check: ok ({out.input_tokens} in, {out.output_tokens} out, {out.latency_ms} ms)"
+            )
+            vec = emb.embed(["ping"])
+            typer.echo(f"embeddings live check: ok (dim {len(vec.vectors[0])})")
+        except ProviderError as exc:
+            problems.append(str(exc))
+
+    if problems:
+        typer.echo("problems:")
+        for p in problems:
+            typer.echo(f"  {p}")
+        raise typer.Exit(code=1)
+    typer.echo("configuration ok")
+
+
+@app.command()
+def init(
+    force: bool = typer.Option(
+        False, "--force", help="Overwrite existing .env and ragfabric.yaml."
+    ),
+) -> None:
+    """Create .env and ragfabric.yaml from the examples, then validate."""
+    import shutil
+    from pathlib import Path
+
+    for example, target in ((".env.example", ".env"), ("ragfabric.example.yaml", "ragfabric.yaml")):
+        src, dst = Path(example), Path(target)
+        if not src.exists():
+            typer.echo(f"{example} not found in the current directory")
+            raise typer.Exit(code=1)
+        if dst.exists() and not force:
+            typer.echo(f"{target} already exists (use --force to overwrite)")
+            continue
+        shutil.copyfile(src, dst)
+        typer.echo(f"wrote {target}")
+    from ragfabric_core.runtime import reset_config
+
+    reset_config()
+    config_validate(path=None, check_providers=False)
+
+
+@app.command()
+def serve(
+    host: str = typer.Option(
+        "127.0.0.1",
+        help=(
+            "Bind address. Defaults to localhost only; pass --host 0.0.0.0 "
+            "explicitly to expose this on every network interface."
+        ),
+    ),
+    port: int = typer.Option(8000, help="Port."),
+    reload: bool = typer.Option(False, help="Auto reload (development only)."),
+) -> None:
+    """Run the HTTP API with uvicorn.
+
+    Behaviour change: this used to default to 0.0.0.0, which exposed a
+    development server on every network interface unless a caller happened
+    to override it. It now defaults to 127.0.0.1 (localhost only); naming
+    --host 0.0.0.0 explicitly still works exactly as before for a deployment
+    that means to bind every interface (e.g. inside a container behind its
+    own network boundary).
+    """
+    import uvicorn
+
+    uvicorn.run("ragfabric_server.main:app", host=host, port=port, reload=reload)