rag-failcase-kit 0.1.0__tar.gz

rag_failcase_kit-0.1.0/.gitignore

@@ -0,0 +1,28 @@
+# Python caches
+__pycache__/
+*.py[cod]
+*.pyo
+
+# Test and tooling caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Build artifacts
+build/
+dist/
+*.egg-info/
+
+# Local logs generated by examples
+logs/
+
+# Environment files
+.env
+.env.*

rag_failcase_kit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rag-failcase-kit contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rag_failcase_kit-0.1.0/PKG-INFO

@@ -0,0 +1,221 @@
+Metadata-Version: 2.4
+Name: rag-failcase-kit
+Version: 0.1.0
+Summary: Lightweight JSONL tracing for debugging failed RAG answers.
+Project-URL: Homepage, https://github.com/slayer011019/rag-failcase-kit
+Project-URL: Repository, https://github.com/slayer011019/rag-failcase-kit
+Project-URL: Issues, https://github.com/slayer011019/rag-failcase-kit/issues
+Author: rag-failcase-kit contributors
+License: MIT
+License-File: LICENSE
+Keywords: debugging,jsonl,logging,rag,retrieval
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Debuggers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: twine>=5.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# rag-failcase-kit
+
+`rag-failcase-kit` is a lightweight Python package for recording one RAG pipeline run as one JSONL trace.
+
+When a RAG application returns an empty, inaccurate, or weakly relevant answer, it is often hard to tell where the failure happened. The issue may be retrieval quality, prompt construction, fallback logic, answer generation, or a runtime error. In small projects this often becomes a pile of `print` statements, which is difficult to search, compare, or replay later.
+
+This package keeps the first version intentionally small: it records query, retrieved documents, scores, fallback usage, answer preview, latency, status, and errors into a JSONL file.
+
+## Installation
+
+From PyPI:
+
+```bash
+pip install rag-failcase-kit
+```
+
+From a local checkout:
+
+```bash
+pip install -e .
+```
+
+For development and release checks:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## 30-Second Usage
+
+```python
+from rag_failcase_kit import RagLogger
+
+logger = RagLogger(project="demo-rag", log_path="logs/rag_traces.jsonl")
+
+with logger.trace("What is RAG?") as trace:
+    trace.log_retrieval(
+        [
+            {
+                "content": "Retrieval-augmented generation combines retrieval with text generation.",
+                "source": "docs/rag.md",
+                "score": 0.91,
+            }
+        ],
+        top_k=3,
+        min_score=0.7,
+    )
+    trace.log_answer("RAG retrieves relevant context and uses it to generate an answer.")
+```
+
+The log directory is created automatically. Each trace is appended as one JSON object per line.
+
+## CLI Usage
+
+The package installs a `rag-failcase` command:
+
+```bash
+rag-failcase summary logs/rag_traces.jsonl
+rag-failcase failures logs/rag_traces.jsonl
+rag-failcase inspect logs/rag_traces.jsonl --trace-id <trace_id>
+```
+
+All CLI commands print JSON output.
+
+Example summary:
+
+```json
+{
+  "total_traces": 12,
+  "successes": 9,
+  "failures": 3,
+  "fallback_used": 4,
+  "avg_latency_ms": 128.42
+}
+```
+
+## Log Schema
+
+Example JSONL record:
+
+```json
+{
+  "trace_id": "9fcd7b91-6b12-46fb-8ff1-5f6f1ef49b15",
+  "project": "demo-rag",
+  "query": "What is RAG?",
+  "retrieval": {
+    "documents": [
+      {
+        "content_preview": "Retrieval-augmented generation combines retrieval with text generation.",
+        "metadata": {
+          "chunk_id": "rag-001"
+        },
+        "source": "docs/rag.md",
+        "score": 0.91
+      }
+    ],
+    "top_k": 3,
+    "min_score": 0.7
+  },
+  "retrieved_count": 1,
+  "fallback": {
+    "used": false,
+    "strategy": null,
+    "reason": null
+  },
+  "answer": "RAG retrieves relevant context and uses it to generate an answer.",
+  "answer_preview": "RAG retrieves relevant context and uses it to generate an answer.",
+  "latency_ms": 3.241,
+  "status": "success",
+  "error": null,
+  "created_at": "2026-07-09T00:00:00+00:00"
+}
+```
+
+Document content is stored as a bounded preview to avoid unexpectedly large logs.
+
+## Real RAG Integration
+
+Wrap the part of your RAG pipeline that handles one user query:
+
+```python
+from rag_failcase_kit import RagLogger
+
+logger = RagLogger(project="support-bot", log_path="logs/support_rag.jsonl")
+
+
+def answer_question(query, retriever, llm):
+    with logger.trace(query) as trace:
+        documents = retriever.search(query)
+        trace.log_retrieval(documents, top_k=5, min_score=0.75)
+
+        if not documents:
+            trace.log_fallback("no_documents", reason="Retriever returned no documents.")
+            answer = "I do not have enough context to answer."
+            trace.log_answer(answer)
+            return answer
+
+        answer = llm.generate(query=query, documents=documents)
+        trace.log_answer(answer)
+        return answer
+```
+
+`documents` can be dictionaries or document-like objects. For objects, the logger safely checks common fields such as `page_content`, `metadata`, `source`, and `score`.
+
+## Optional Integrations
+
+The package does not depend on FastAPI, LangChain, or LangGraph.
+
+FastAPI example:
+
+```bash
+pip install fastapi uvicorn
+uvicorn examples.fastapi_example:app --reload
+```
+
+LangGraph-style example:
+
+```bash
+python examples/langgraph_example.py
+```
+
+See:
+
+- [examples/basic_usage.py](examples/basic_usage.py)
+- [examples/fastapi_example.py](examples/fastapi_example.py)
+- [examples/langgraph_example.py](examples/langgraph_example.py)
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+python -m build
+twine check dist/*
+```
+
+To test the console script in a clean environment:
+
+```bash
+python -m venv .venv-test
+. .venv-test/bin/activate
+pip install dist/rag_failcase_kit-0.1.0-py3-none-any.whl
+rag-failcase --help
+```
+
+## Roadmap
+
+- Add prompt logging with explicit redaction controls.
+- Add richer failure categorization helpers.
+- Add Markdown or HTML reports.
+- Add log rotation and sampling controls.
+- Add optional adapters for popular RAG frameworks without making them required dependencies.

rag_failcase_kit-0.1.0/README.md

@@ -0,0 +1,192 @@
+# rag-failcase-kit
+
+`rag-failcase-kit` is a lightweight Python package for recording one RAG pipeline run as one JSONL trace.
+
+When a RAG application returns an empty, inaccurate, or weakly relevant answer, it is often hard to tell where the failure happened. The issue may be retrieval quality, prompt construction, fallback logic, answer generation, or a runtime error. In small projects this often becomes a pile of `print` statements, which is difficult to search, compare, or replay later.
+
+This package keeps the first version intentionally small: it records query, retrieved documents, scores, fallback usage, answer preview, latency, status, and errors into a JSONL file.
+
+## Installation
+
+From PyPI:
+
+```bash
+pip install rag-failcase-kit
+```
+
+From a local checkout:
+
+```bash
+pip install -e .
+```
+
+For development and release checks:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## 30-Second Usage
+
+```python
+from rag_failcase_kit import RagLogger
+
+logger = RagLogger(project="demo-rag", log_path="logs/rag_traces.jsonl")
+
+with logger.trace("What is RAG?") as trace:
+    trace.log_retrieval(
+        [
+            {
+                "content": "Retrieval-augmented generation combines retrieval with text generation.",
+                "source": "docs/rag.md",
+                "score": 0.91,
+            }
+        ],
+        top_k=3,
+        min_score=0.7,
+    )
+    trace.log_answer("RAG retrieves relevant context and uses it to generate an answer.")
+```
+
+The log directory is created automatically. Each trace is appended as one JSON object per line.
+
+## CLI Usage
+
+The package installs a `rag-failcase` command:
+
+```bash
+rag-failcase summary logs/rag_traces.jsonl
+rag-failcase failures logs/rag_traces.jsonl
+rag-failcase inspect logs/rag_traces.jsonl --trace-id <trace_id>
+```
+
+All CLI commands print JSON output.
+
+Example summary:
+
+```json
+{
+  "total_traces": 12,
+  "successes": 9,
+  "failures": 3,
+  "fallback_used": 4,
+  "avg_latency_ms": 128.42
+}
+```
+
+## Log Schema
+
+Example JSONL record:
+
+```json
+{
+  "trace_id": "9fcd7b91-6b12-46fb-8ff1-5f6f1ef49b15",
+  "project": "demo-rag",
+  "query": "What is RAG?",
+  "retrieval": {
+    "documents": [
+      {
+        "content_preview": "Retrieval-augmented generation combines retrieval with text generation.",
+        "metadata": {
+          "chunk_id": "rag-001"
+        },
+        "source": "docs/rag.md",
+        "score": 0.91
+      }
+    ],
+    "top_k": 3,
+    "min_score": 0.7
+  },
+  "retrieved_count": 1,
+  "fallback": {
+    "used": false,
+    "strategy": null,
+    "reason": null
+  },
+  "answer": "RAG retrieves relevant context and uses it to generate an answer.",
+  "answer_preview": "RAG retrieves relevant context and uses it to generate an answer.",
+  "latency_ms": 3.241,
+  "status": "success",
+  "error": null,
+  "created_at": "2026-07-09T00:00:00+00:00"
+}
+```
+
+Document content is stored as a bounded preview to avoid unexpectedly large logs.
+
+## Real RAG Integration
+
+Wrap the part of your RAG pipeline that handles one user query:
+
+```python
+from rag_failcase_kit import RagLogger
+
+logger = RagLogger(project="support-bot", log_path="logs/support_rag.jsonl")
+
+
+def answer_question(query, retriever, llm):
+    with logger.trace(query) as trace:
+        documents = retriever.search(query)
+        trace.log_retrieval(documents, top_k=5, min_score=0.75)
+
+        if not documents:
+            trace.log_fallback("no_documents", reason="Retriever returned no documents.")
+            answer = "I do not have enough context to answer."
+            trace.log_answer(answer)
+            return answer
+
+        answer = llm.generate(query=query, documents=documents)
+        trace.log_answer(answer)
+        return answer
+```
+
+`documents` can be dictionaries or document-like objects. For objects, the logger safely checks common fields such as `page_content`, `metadata`, `source`, and `score`.
+
+## Optional Integrations
+
+The package does not depend on FastAPI, LangChain, or LangGraph.
+
+FastAPI example:
+
+```bash
+pip install fastapi uvicorn
+uvicorn examples.fastapi_example:app --reload
+```
+
+LangGraph-style example:
+
+```bash
+python examples/langgraph_example.py
+```
+
+See:
+
+- [examples/basic_usage.py](examples/basic_usage.py)
+- [examples/fastapi_example.py](examples/fastapi_example.py)
+- [examples/langgraph_example.py](examples/langgraph_example.py)
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+python -m build
+twine check dist/*
+```
+
+To test the console script in a clean environment:
+
+```bash
+python -m venv .venv-test
+. .venv-test/bin/activate
+pip install dist/rag_failcase_kit-0.1.0-py3-none-any.whl
+rag-failcase --help
+```
+
+## Roadmap
+
+- Add prompt logging with explicit redaction controls.
+- Add richer failure categorization helpers.
+- Add Markdown or HTML reports.
+- Add log rotation and sampling controls.
+- Add optional adapters for popular RAG frameworks without making them required dependencies.

rag_failcase_kit-0.1.0/examples/basic_usage.py

@@ -0,0 +1,20 @@
+"""Basic usage example for rag-failcase-kit."""
+
+from rag_failcase_kit import RagLogger
+
+
+logger = RagLogger(project="demo-rag", log_path="logs/rag_traces.jsonl")
+
+with logger.trace("What is RAG?") as trace:
+    trace.log_retrieval(
+        [
+            {
+                "content": "Retrieval-augmented generation combines retrieval with text generation.",
+                "source": "docs/rag.md",
+                "score": 0.91,
+            }
+        ],
+        top_k=3,
+        min_score=0.7,
+    )
+    trace.log_answer("RAG retrieves relevant context and uses it to generate an answer.")

rag_failcase_kit-0.1.0/examples/fastapi_example.py

@@ -0,0 +1,33 @@
+"""FastAPI integration example.
+
+Install FastAPI separately to run this file:
+
+    pip install fastapi uvicorn
+    uvicorn examples.fastapi_example:app --reload
+"""
+
+from fastapi import FastAPI
+
+from rag_failcase_kit import RagLogger
+
+
+app = FastAPI()
+logger = RagLogger(project="fastapi-rag", log_path="logs/fastapi_rag.jsonl")
+
+
+@app.get("/ask")
+def ask(query: str) -> dict:
+    """Record one trace around a minimal RAG-like request."""
+
+    with logger.trace(query) as trace:
+        documents = [{"content": "FastAPI is a Python web framework.", "score": 0.88}]
+        trace.log_retrieval(documents, top_k=1)
+
+        if not documents:
+            trace.log_fallback("no_context", reason="Retriever returned no documents.")
+            answer = "I do not have enough context to answer."
+        else:
+            answer = "FastAPI is commonly used to build Python APIs."
+
+        trace.log_answer(answer)
+        return {"answer": answer, "trace_id": trace.trace_id}

rag_failcase_kit-0.1.0/examples/langgraph_example.py

@@ -0,0 +1,38 @@
+"""LangGraph-style integration example without importing LangGraph.
+
+The package does not depend on LangGraph. This example shows where tracing can
+be placed inside graph node functions.
+"""
+
+from typing import Dict, List, TypedDict
+
+from rag_failcase_kit import RagLogger
+
+
+class State(TypedDict, total=False):
+    query: str
+    documents: List[Dict[str, object]]
+    answer: str
+    trace_id: str
+
+
+logger = RagLogger(project="langgraph-rag", log_path="logs/langgraph_rag.jsonl")
+
+
+def rag_node(state: State) -> State:
+    """Example node that records retrieval and answer generation."""
+
+    query = state["query"]
+    with logger.trace(query) as trace:
+        documents = [{"content": "Graph nodes can share state.", "score": 0.83}]
+        trace.log_retrieval(documents, top_k=2)
+
+        answer = "A graph node can log retrieval and answer events inside one trace."
+        trace.log_answer(answer)
+
+        return {
+            **state,
+            "documents": documents,
+            "answer": answer,
+            "trace_id": trace.trace_id,
+        }

rag_failcase_kit-0.1.0/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "rag-failcase-kit"
+version = "0.1.0"
+description = "Lightweight JSONL tracing for debugging failed RAG answers."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [
+  { name = "rag-failcase-kit contributors" },
+]
+keywords = ["rag", "logging", "debugging", "jsonl", "retrieval"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Debuggers",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/slayer011019/rag-failcase-kit"
+Repository = "https://github.com/slayer011019/rag-failcase-kit"
+Issues = "https://github.com/slayer011019/rag-failcase-kit/issues"
+
+[project.optional-dependencies]
+dev = [
+  "build>=1.2",
+  "pytest>=7.0",
+  "twine>=5.0",
+]
+
+[project.scripts]
+rag-failcase = "rag_failcase_kit.cli:main"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rag_failcase_kit"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "README.md",
+  "LICENSE",
+  "pyproject.toml",
+  "src",
+  "examples",
+  "tests",
+]

rag_failcase_kit-0.1.0/src/rag_failcase_kit/__init__.py

@@ -0,0 +1,8 @@
+"""Small JSONL tracing toolkit for RAG failure analysis."""
+
+from rag_failcase_kit.logger import RagLogger
+from rag_failcase_kit.trace import RagTrace
+
+__all__ = ["RagLogger", "RagTrace"]
+
+__version__ = "0.1.0"

rag_failcase_kit-0.1.0/src/rag_failcase_kit/cli.py

@@ -0,0 +1,74 @@
+"""Command line interface for rag-failcase-kit."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Any, Dict, List, Optional, Union
+
+from rag_failcase_kit.report import failure_records, find_record, load_records, summarize
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the CLI argument parser."""
+
+    parser = argparse.ArgumentParser(
+        prog="rag-failcase",
+        description="Inspect rag-failcase-kit JSONL trace logs.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    summary_parser = subparsers.add_parser("summary", help="Show aggregate log statistics.")
+    summary_parser.add_argument("log_path")
+
+    failures_parser = subparsers.add_parser("failures", help="List failed traces.")
+    failures_parser.add_argument("log_path")
+
+    inspect_parser = subparsers.add_parser("inspect", help="Show one trace by ID.")
+    inspect_parser.add_argument("log_path")
+    inspect_parser.add_argument("--trace-id", required=True)
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Run the command line interface."""
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        records = load_records(args.log_path)
+    except FileNotFoundError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+
+    if args.command == "summary":
+        _print_json(summarize(records))
+        return 0
+
+    if args.command == "failures":
+        _print_json(failure_records(records))
+        return 0
+
+    if args.command == "inspect":
+        record = find_record(records, args.trace_id)
+        if record is None:
+            print(f"Trace not found: {args.trace_id}", file=sys.stderr)
+            return 1
+        _print_json(record)
+        return 0
+
+    parser.print_help()
+    return 1
+
+
+def _print_json(value: Union[Dict[str, Any], List[Dict[str, Any]]]) -> None:
+    """Print JSON output for humans and scripts."""
+
+    print(json.dumps(value, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rag_failcase_kit-0.1.0/src/rag_failcase_kit/logger.py

@@ -0,0 +1,28 @@
+"""Public logger facade for creating RAG traces."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Union
+
+from rag_failcase_kit.trace import RagTrace
+from rag_failcase_kit.writer import JsonlWriter
+
+
+class RagLogger:
+    """Create structured JSONL traces for RAG runs.
+
+    Args:
+        project: Human-readable project or application name.
+        log_path: JSONL file path where traces will be appended.
+    """
+
+    def __init__(self, project: str, log_path: Union[str, Path] = "logs/rag_traces.jsonl") -> None:
+        self.project = project
+        self.log_path = Path(log_path)
+        self.writer = JsonlWriter(self.log_path)
+
+    def trace(self, query: str) -> RagTrace:
+        """Create a context manager for one RAG query execution."""
+
+        return RagTrace(project=self.project, query=query, writer=self.writer)

rag_failcase_kit-0.1.0/src/rag_failcase_kit/py.typed

@@ -0,0 +1 @@
+

rag_failcase_kit-0.1.0/src/rag_failcase_kit/report.py

@@ -0,0 +1,59 @@
+"""Read and summarize JSONL trace logs."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Union
+
+
+def load_records(log_path: Union[str, Path]) -> List[Dict[str, Any]]:
+    """Load trace records from a JSONL file."""
+
+    path = Path(log_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Log file not found: {path}")
+
+    records: List[Dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            if line.strip():
+                records.append(json.loads(line))
+    return records
+
+
+def summarize(records: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
+    """Return aggregate statistics for trace records."""
+
+    items = list(records)
+    total = len(items)
+    failures = [record for record in items if record.get("status") != "success"]
+    fallback_used = [record for record in items if record.get("fallback", {}).get("used")]
+    latencies = [
+        record["latency_ms"]
+        for record in items
+        if isinstance(record.get("latency_ms"), (int, float))
+    ]
+
+    return {
+        "total_traces": total,
+        "successes": total - len(failures),
+        "failures": len(failures),
+        "fallback_used": len(fallback_used),
+        "avg_latency_ms": round(sum(latencies) / len(latencies), 3) if latencies else None,
+    }
+
+
+def failure_records(records: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Return records that indicate a failed or errored trace."""
+
+    return [record for record in records if record.get("status") != "success"]
+
+
+def find_record(records: Iterable[Dict[str, Any]], trace_id: str) -> Optional[Dict[str, Any]]:
+    """Find a trace by ID."""
+
+    for record in records:
+        if record.get("trace_id") == trace_id:
+            return record
+    return None

rag_failcase_kit-0.1.0/src/rag_failcase_kit/schema.py

@@ -0,0 +1,120 @@
+"""Schema helpers for trace records.
+
+The project intentionally uses plain dictionaries instead of a validation
+dependency. This keeps the MVP lightweight and easy to inspect.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+
+TraceRecord = Dict[str, Any]
+DocumentRecord = Dict[str, Any]
+
+
+def utc_now_iso() -> str:
+    """Return the current UTC time as an ISO-8601 string."""
+
+    return datetime.now(timezone.utc).isoformat()
+
+
+def new_trace_record(
+    *,
+    trace_id: str,
+    project: str,
+    query: str,
+    created_at: Optional[str] = None,
+) -> TraceRecord:
+    """Create the default JSON-serializable record for one RAG trace."""
+
+    return {
+        "trace_id": trace_id,
+        "project": project,
+        "query": query,
+        "retrieval": {
+            "documents": [],
+            "top_k": None,
+            "min_score": None,
+        },
+        "retrieved_count": 0,
+        "fallback": {
+            "used": False,
+            "strategy": None,
+            "reason": None,
+        },
+        "answer": None,
+        "answer_preview": None,
+        "latency_ms": None,
+        "status": "running",
+        "error": None,
+        "created_at": created_at or utc_now_iso(),
+    }
+
+
+def preview_text(value: Any, *, limit: int = 500) -> Optional[str]:
+    """Convert a value to a bounded text preview."""
+
+    if value is None:
+        return None
+
+    text = str(value)
+    if len(text) <= limit:
+        return text
+    return text[:limit] + "..."
+
+
+def extract_documents(documents: List[Any], *, content_limit: int = 500) -> List[DocumentRecord]:
+    """Normalize dictionaries or document-like objects into safe log records.
+
+    Supported object attributes are intentionally broad enough for common RAG
+    libraries without importing them: page_content, metadata, source, and score.
+    """
+
+    return [extract_document(document, content_limit=content_limit) for document in documents]
+
+
+def extract_document(document: Any, *, content_limit: int = 500) -> DocumentRecord:
+    """Normalize one document-like value into a JSON-serializable dictionary."""
+
+    if isinstance(document, dict):
+        content = (
+            document.get("content")
+            or document.get("page_content")
+            or document.get("text")
+            or document.get("body")
+        )
+        metadata = document.get("metadata") or {}
+        source = document.get("source")
+        if source is None and isinstance(metadata, dict):
+            source = metadata.get("source")
+        score = document.get("score")
+        if score is None:
+            score = document.get("similarity_score")
+    else:
+        content = getattr(document, "page_content", None) or getattr(document, "content", None)
+        metadata = getattr(document, "metadata", {}) or {}
+        source = getattr(document, "source", None)
+        if source is None and isinstance(metadata, dict):
+            source = metadata.get("source")
+        score = getattr(document, "score", None)
+
+    return {
+        "content_preview": preview_text(content, limit=content_limit),
+        "metadata": make_json_safe(metadata),
+        "source": source,
+        "score": score,
+    }
+
+
+def make_json_safe(value: Any) -> Any:
+    """Convert common Python values into JSON-safe structures."""
+
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    if isinstance(value, dict):
+        return {str(key): make_json_safe(item) for key, item in value.items()}
+    if isinstance(value, (list, tuple, set)):
+        return [make_json_safe(item) for item in value]
+    return str(value)

rag_failcase_kit-0.1.0/src/rag_failcase_kit/trace.py

@@ -0,0 +1,112 @@
+"""Trace object used to record one RAG execution."""
+
+from __future__ import annotations
+
+import traceback
+from time import perf_counter
+from typing import Any, List, Optional
+from uuid import uuid4
+
+from rag_failcase_kit.schema import extract_documents, new_trace_record, preview_text
+from rag_failcase_kit.writer import JsonlWriter
+
+
+class RagTrace:
+    """Collect structured events for one RAG pipeline run."""
+
+    def __init__(self, *, project: str, query: str, writer: JsonlWriter) -> None:
+        self.project = project
+        self.query = query
+        self.writer = writer
+        self.trace_id = str(uuid4())
+        self.record = new_trace_record(
+            trace_id=self.trace_id,
+            project=self.project,
+            query=self.query,
+        )
+        self._started_at = perf_counter()
+        self._finished = False
+
+    def __enter__(self) -> "RagTrace":
+        """Enter the trace context manager."""
+
+        return self
+
+    def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
+        """Finish the trace when leaving a context manager.
+
+        Exceptions are logged and then allowed to propagate.
+        """
+
+        if exc is not None:
+            self.log_error(exc)
+            self.finish(status="error")
+            return False
+
+        if not self._finished:
+            self.finish()
+        return False
+
+    def log_retrieval(
+        self,
+        documents: List[Any],
+        top_k: Optional[int] = None,
+        min_score: Optional[float] = None,
+    ) -> None:
+        """Log retrieved documents and retrieval parameters."""
+
+        normalized_documents = extract_documents(documents)
+        self.record["retrieval"] = {
+            "documents": normalized_documents,
+            "top_k": top_k,
+            "min_score": min_score,
+        }
+        self.record["retrieved_count"] = len(normalized_documents)
+
+    def log_fallback(self, strategy: str, reason: Optional[str] = None) -> None:
+        """Log that fallback behavior was used."""
+
+        self.record["fallback"] = {
+            "used": True,
+            "strategy": strategy,
+            "reason": reason,
+        }
+
+    def log_answer(self, answer: Any) -> None:
+        """Log the generated answer and a bounded preview."""
+
+        self.record["answer"] = answer
+        self.record["answer_preview"] = preview_text(answer)
+
+    def log_error(self, error: Any) -> None:
+        """Log an error raised or observed during the RAG run."""
+
+        if isinstance(error, BaseException):
+            formatted_traceback = None
+            if error.__traceback__ is not None:
+                formatted_traceback = "".join(
+                    traceback.format_exception(type(error), error, error.__traceback__)
+                )
+            self.record["error"] = {
+                "type": error.__class__.__name__,
+                "message": str(error),
+                "traceback": formatted_traceback,
+            }
+        else:
+            self.record["error"] = {
+                "type": type(error).__name__,
+                "message": str(error),
+                "traceback": None,
+            }
+
+    def finish(self, status: str = "success") -> None:
+        """Finalize and write the trace exactly once."""
+
+        if self._finished:
+            return
+
+        elapsed_ms = (perf_counter() - self._started_at) * 1000
+        self.record["latency_ms"] = round(elapsed_ms, 3)
+        self.record["status"] = status
+        self.writer.write(self.record)
+        self._finished = True

rag_failcase_kit-0.1.0/src/rag_failcase_kit/writer.py

@@ -0,0 +1,22 @@
+"""JSONL writer for trace records."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Union
+
+
+class JsonlWriter:
+    """Append JSON records to a JSONL file."""
+
+    def __init__(self, log_path: Union[str, Path]) -> None:
+        self.log_path = Path(log_path)
+        self.log_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def write(self, record: Dict[str, Any]) -> None:
+        """Append one JSON-serializable record as a single line."""
+
+        with self.log_path.open("a", encoding="utf-8") as file:
+            json.dump(record, file, ensure_ascii=False, default=str)
+            file.write("\n")

rag_failcase_kit-0.1.0/tests/test_cli.py

@@ -0,0 +1,26 @@
+import json
+
+from rag_failcase_kit.cli import main
+
+
+def test_cli_summary_outputs_counts(tmp_path, capsys):
+    log_path = tmp_path / "traces.jsonl"
+    records = [
+        {"trace_id": "1", "status": "success", "fallback": {"used": False}, "latency_ms": 10},
+        {"trace_id": "2", "status": "error", "fallback": {"used": True}, "latency_ms": 30},
+    ]
+    log_path.write_text(
+        "\n".join(json.dumps(record) for record in records) + "\n",
+        encoding="utf-8",
+    )
+
+    exit_code = main(["summary", str(log_path)])
+
+    captured = capsys.readouterr()
+    summary = json.loads(captured.out)
+    assert exit_code == 0
+    assert summary["total_traces"] == 2
+    assert summary["successes"] == 1
+    assert summary["failures"] == 1
+    assert summary["fallback_used"] == 1
+    assert summary["avg_latency_ms"] == 20

rag_failcase_kit-0.1.0/tests/test_logger.py

@@ -0,0 +1,11 @@
+from rag_failcase_kit import RagLogger
+
+
+def test_logger_initialization_creates_log_directory(tmp_path):
+    log_path = tmp_path / "nested" / "traces.jsonl"
+
+    logger = RagLogger(project="test-project", log_path=log_path)
+
+    assert logger.project == "test-project"
+    assert logger.log_path == log_path
+    assert log_path.parent.exists()

rag_failcase_kit-0.1.0/tests/test_trace.py

@@ -0,0 +1,77 @@
+import json
+
+from rag_failcase_kit import RagLogger
+
+
+def read_first_record(log_path):
+    return json.loads(log_path.read_text(encoding="utf-8").splitlines()[0])
+
+
+def test_trace_saves_jsonl_record(tmp_path):
+    log_path = tmp_path / "traces.jsonl"
+    logger = RagLogger(project="demo", log_path=log_path)
+
+    with logger.trace("hello") as trace:
+        trace.log_answer("world")
+
+    record = read_first_record(log_path)
+    assert record["project"] == "demo"
+    assert record["query"] == "hello"
+    assert record["answer_preview"] == "world"
+    assert record["status"] == "success"
+    assert record["trace_id"]
+    assert isinstance(record["latency_ms"], float)
+
+
+def test_retrieval_logging_handles_dicts_and_objects(tmp_path):
+    class Document:
+        page_content = "object document content"
+        metadata = {"source": "object.md"}
+        score = 0.81
+
+    log_path = tmp_path / "traces.jsonl"
+    logger = RagLogger(project="demo", log_path=log_path)
+
+    with logger.trace("query") as trace:
+        trace.log_retrieval(
+            [
+                {"content": "dict document content", "source": "dict.md", "score": 0.95},
+                Document(),
+            ],
+            top_k=2,
+            min_score=0.8,
+        )
+
+    record = read_first_record(log_path)
+    assert record["retrieved_count"] == 2
+    assert record["retrieval"]["top_k"] == 2
+    assert record["retrieval"]["min_score"] == 0.8
+    assert record["retrieval"]["documents"][0]["content_preview"] == "dict document content"
+    assert record["retrieval"]["documents"][1]["source"] == "object.md"
+
+
+def test_fallback_logging(tmp_path):
+    log_path = tmp_path / "traces.jsonl"
+    logger = RagLogger(project="demo", log_path=log_path)
+
+    with logger.trace("query") as trace:
+        trace.log_fallback("no_results", reason="Retriever returned nothing.")
+
+    record = read_first_record(log_path)
+    assert record["fallback"]["used"] is True
+    assert record["fallback"]["strategy"] == "no_results"
+    assert record["fallback"]["reason"] == "Retriever returned nothing."
+
+
+def test_error_logging_with_explicit_finish(tmp_path):
+    log_path = tmp_path / "traces.jsonl"
+    logger = RagLogger(project="demo", log_path=log_path)
+
+    with logger.trace("query") as trace:
+        trace.log_error(ValueError("bad answer"))
+        trace.finish(status="error")
+
+    record = read_first_record(log_path)
+    assert record["status"] == "error"
+    assert record["error"]["type"] == "ValueError"
+    assert record["error"]["message"] == "bad answer"

rag_failcase_kit-0.1.0/tests/test_writer.py

@@ -0,0 +1,14 @@
+import json
+
+from rag_failcase_kit.writer import JsonlWriter
+
+
+def test_writer_appends_single_json_line(tmp_path):
+    log_path = tmp_path / "logs" / "trace.jsonl"
+    writer = JsonlWriter(log_path)
+
+    writer.write({"trace_id": "abc", "status": "success"})
+
+    lines = log_path.read_text(encoding="utf-8").splitlines()
+    assert len(lines) == 1
+    assert json.loads(lines[0]) == {"trace_id": "abc", "status": "success"}