ragfabric-core 0.3.1__tar.gz

ragfabric_core-0.3.1/.gitignore

@@ -0,0 +1,49 @@
+# Environment
+.env
+.env.*
+!.env.example
+
+# local configuration (copy of ragfabric.example.yaml)
+ragfabric.yaml
+
+# OS
+.DS_Store
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+venv/
+.pytest_cache/
+*.egg-info/
+
+# Local data / databases
+*.db
+backend/data/uploads/
+
+# Node / Angular
+node_modules/
+dist/
+.angular/
+npm-debug.log*
+
+# Editors
+.idea/
+.vscode/
+
+# Frontend test artifacts
+coverage/
+out-tsc/
+
+# Local planning notes and scratch (never committed)
+MEMORY*.md
+*.local.md
+.notes/
+.scratch/
+
+# Secrets and keys (belt and braces; .env already excluded above)
+*.pem
+*.key
+*.p12
+secrets/

ragfabric_core-0.3.1/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.5
+Name: ragfabric-core
+Version: 0.3.1
+Summary: RagFabric engine: retrieval strategies, providers, stores, ingestion, evaluation, access policy.
+Project-URL: Homepage, https://github.com/ranjan-del/ragfabric
+Project-URL: Repository, https://github.com/ranjan-del/ragfabric
+Project-URL: Documentation, https://github.com/ranjan-del/ragfabric/tree/main/docs
+Project-URL: Changelog, https://github.com/ranjan-del/ragfabric/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/ranjan-del/ragfabric/issues
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <3.14,>=3.13
+Requires-Dist: alembic>=1.20
+Requires-Dist: bcrypt>=5.0
+Requires-Dist: email-validator>=2.3
+Requires-Dist: numpy>=2.3
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.44
+Requires-Dist: opentelemetry-sdk>=1.44
+Requires-Dist: pgvector>=0.5
+Requires-Dist: psycopg[binary]>=3.3
+Requires-Dist: pydantic-settings>=2.15
+Requires-Dist: pydantic>=2.13
+Requires-Dist: pypdf>=6.18
+Requires-Dist: python-docx>=1.2
+Requires-Dist: python-jose[cryptography]>=3.5
+Requires-Dist: python-pptx>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: redis>=8.1
+Requires-Dist: sqlalchemy>=2.0.52
+Requires-Dist: tiktoken>=0.12
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=1.5; extra == 'anthropic'
+Provides-Extra: bm25
+Requires-Dist: rank-bm25>=0.2.2; extra == 'bm25'
+Provides-Extra: chroma
+Requires-Dist: chromadb>=1.5; extra == 'chroma'
+Provides-Extra: openai
+Requires-Dist: openai>=3.13; extra == 'openai'
+Provides-Extra: rerank
+Requires-Dist: sentence-transformers>=5.1; extra == 'rerank'
+Description-Content-Type: text/markdown
+
+# ragfabric-core
+
+The RagFabric engine. See the repository README.

ragfabric_core-0.3.1/README.md

@@ -0,0 +1,3 @@
+# ragfabric-core
+
+The RagFabric engine. See the repository README.

ragfabric_core-0.3.1/pyproject.toml

@@ -0,0 +1,64 @@
+[project]
+name = "ragfabric-core"
+version = "0.3.1"
+description = "RagFabric engine: retrieval strategies, providers, stores, ingestion, evaluation, access policy."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.13,<3.14"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+  "pydantic>=2.13",
+  "pydantic-settings>=2.15",
+  "sqlalchemy>=2.0.52",
+  "alembic>=1.20",
+  "psycopg[binary]>=3.3",
+  "numpy>=2.3",
+  "pypdf>=6.18",
+  "python-docx>=1.2",
+  "python-pptx>=1.0",
+  "python-jose[cryptography]>=3.5",
+  "bcrypt>=5.0",
+  "email-validator>=2.3",
+  "pyyaml>=6.0",
+  "pgvector>=0.5",
+  "redis>=8.1",
+  "opentelemetry-sdk>=1.44",
+  "opentelemetry-exporter-otlp-proto-http>=1.44",
+  "tiktoken>=0.12",
+]
+
+[project.optional-dependencies]
+chroma = ["chromadb>=1.5"]
+# In-process BM25 (stores/bm25_memory.py). Optional because it is not the
+# production path: the SQL store needs no extra at all, and a deployment
+# that never uses the in-process store should not carry the dependency.
+bm25 = ["rank_bm25>=0.2.2"]
+rerank = ["sentence-transformers>=5.1"]
+# Ollama also goes through OpenAICompatibleLLM/OpenAICompatibleEmbeddings
+# (providers/openai_compat.py), since Ollama serves the OpenAI chat and
+# embeddings API at /v1, so an Ollama-only deployment still needs this
+# extra installed; only a deployment that uses neither OpenAI, Azure, vLLM,
+# LM Studio nor Ollama can skip it.
+openai = ["openai>=3.13"]
+anthropic = ["anthropic>=1.5"]
+
+[project.urls]
+Homepage = "https://github.com/ranjan-del/ragfabric"
+Repository = "https://github.com/ranjan-del/ragfabric"
+Documentation = "https://github.com/ranjan-del/ragfabric/tree/main/docs"
+Changelog = "https://github.com/ranjan-del/ragfabric/blob/main/CHANGELOG.md"
+Issues = "https://github.com/ranjan-del/ragfabric/issues"
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragfabric_core"]

ragfabric_core-0.3.1/src/ragfabric_core/__init__.py

@@ -0,0 +1,3 @@
+"""RagFabric engine."""
+
+__version__ = "0.3.1"

ragfabric_core-0.3.1/src/ragfabric_core/agent/__init__.py

@@ -0,0 +1 @@
+"""The bounded retrieval agent: state, contracts, tools, nodes and the loop."""

ragfabric_core-0.3.1/src/ragfabric_core/agent/contracts.py

@@ -0,0 +1,70 @@
+"""What the agent asks a model for, and what it will accept back.
+
+Every decision the agent delegates to a model comes back as JSON validated
+against a schema here. There is no native tool calling involved, deliberately:
+``LLMProvider`` exposes only ``complete`` and ``stream``, and building a
+tool-calling surface would be a second subsystem that could not be exercised
+without paid API credit. JSON contracts work on every provider, including a
+3B model running locally, and they let the scripted test double drive every
+branch of the loop offline.
+
+The parsers are deliberately forgiving about *packaging* and strict about
+*content*. Small models wrap JSON in a fenced block, preface it with a sentence
+of agreement, or explain themselves afterwards. None of that is a real failure,
+so it is tolerated. A missing field, an empty plan or an invented repair move
+is a real failure, and it returns a ``ContractViolation`` rather than raising,
+so the caller decides whether to retry with a corrective instruction or fall
+back to a safe default. A malformed response must never crash a request and
+must never be silently counted as success.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from ragfabric_core.agent.state import RepairMove
+from ragfabric_core.json_contract import ContractViolation, parse_contract
+
+
+class PlannedSubQuestion(BaseModel):
+    text: str = Field(min_length=1)
+    tool: str = Field(min_length=1)
+    why: str = ""
+
+
+class PlanResponse(BaseModel):
+    sub_questions: list[PlannedSubQuestion] = Field(min_length=1)
+
+
+class Verdict(BaseModel):
+    sub_question: str = Field(min_length=1)
+    answered: bool
+    missing: str | None = None
+
+
+class AssessResponse(BaseModel):
+    verdicts: list[Verdict]
+
+
+class RepairResponse(BaseModel):
+    move: RepairMove
+    rewritten_query: str | None = None
+    why: str = ""
+
+
+def parse_plan(text: str) -> PlanResponse | ContractViolation:
+    """A plan with no sub-questions is refused.
+
+    An empty plan leaves the loop nothing to retrieve for, and it would then
+    report no progress on the next pass, which hides the real problem.
+    """
+    return parse_contract(text, "plan", PlanResponse)
+
+
+def parse_assess(text: str) -> AssessResponse | ContractViolation:
+    return parse_contract(text, "assess", AssessResponse)
+
+
+def parse_repair(text: str) -> RepairResponse | ContractViolation:
+    """An invented move is refused by the schema before it reaches the policy."""
+    return parse_contract(text, "repair", RepairResponse)

ragfabric_core-0.3.1/src/ragfabric_core/agent/cost.py

@@ -0,0 +1,91 @@
+"""What a run has spent, and the honest answer when nobody knows (ADR 0004).
+
+The agent is the one strategy whose spend is not knowable before it runs, so a
+cost cap has to be checked against tokens that have actually been returned.
+This ledger holds those tokens, per model, and prices them through the same
+``pricing.yaml`` table the rest of the product reads.
+
+**The unpriced model is the whole design.** ``estimate_cost`` reports
+``usd=None, known=False`` for a model the table has no price for, and there are
+only two ways to carry that into a cap, one of which is a lie:
+
+- treat unknown as zero, and the cap silently never binds while the trace still
+  implies it was watching;
+- treat unknown as infinite, and every run on a self hosted model stops on its
+  first call.
+
+So unknown means the cap cannot bind, and the ledger says which models it could
+not price. The caller records that alongside the run rather than reporting a
+number nobody measured. A price of zero is a different thing entirely: local
+inference is priced at zero in the table, with a source saying why, and that is
+a known cost of nothing rather than an unknown cost.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from ragfabric_core.pricing import PricingTable, estimate_cost
+
+
+@lru_cache(maxsize=1)
+def _shared_table() -> PricingTable:
+    """The packaged pricing table, read once per process.
+
+    Cached because the loop asks for a price several times per run and the file
+    cannot change under a running process.
+    """
+    return PricingTable.load()
+
+
+class CostLedger:
+    """Real token counts per model, priced only where a price exists."""
+
+    def __init__(self, table: PricingTable | None = None) -> None:
+        self._table = table if table is not None else _shared_table()
+        self._tokens: dict[tuple[str, str], list[int]] = {}
+
+    def record(self, provider: str, model: str, input_tokens: int, output_tokens: int) -> None:
+        """Add one completion's reported tokens.
+
+        A node that made no model call passes an empty provider and is ignored,
+        because an entry for a call that never happened would put a model in
+        the unpriced list on the strength of nothing.
+        """
+        if not provider:
+            return
+        spent = self._tokens.setdefault((provider, model), [0, 0])
+        spent[0] += input_tokens
+        spent[1] += output_tokens
+
+    def unpriced_models(self) -> list[str]:
+        """The ``provider/model`` keys this run used that the table cannot price."""
+        return sorted(
+            f"{provider}/{model}"
+            for (provider, model), (input_tokens, output_tokens) in self._tokens.items()
+            if not estimate_cost(self._table, provider, model, input_tokens, output_tokens).known
+        )
+
+    @property
+    def known(self) -> bool:
+        """True when every model used so far has a price.
+
+        A run that has made no call at all is known: nothing has been spent,
+        and that is a measurement rather than a guess.
+        """
+        return not self.unpriced_models()
+
+    def usd(self) -> float | None:
+        """The estimated spend, or ``None`` when any model used has no price.
+
+        Deliberately all or nothing. Summing the priced part of a run and
+        presenting it as the total would understate the bill by exactly the
+        amount nobody can compute.
+        """
+        if not self.known:
+            return None
+        total = 0.0
+        for (provider, model), (input_tokens, output_tokens) in self._tokens.items():
+            estimate = estimate_cost(self._table, provider, model, input_tokens, output_tokens)
+            total += estimate.usd or 0.0
+        return round(total, 8)