finobs 0.0.1__tar.gz

finobs-0.0.1/.gitignore

@@ -0,0 +1,111 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+
+/src/generated/prisma
+
+.vercel
+
+# Documentation
+/docs/
+# SQLite databases
+*.db
+/intelligence_engine/data/
+/intelligence_engine/**/data/
+
+# Agent & Tools Clutter
+.agents/
+.playwright-mcp/
+playwright-report/
+test-results/
+.gemini/
+*.log
+*.resolved
+
+# Python
+__pycache__/
+*.pyc
+
+# Temporary Scripts
+fetch_leads.js
+fix_db.js
+save_intelligence.js
+tmp_*.js
+scripts/inject-test-lead.ts
+
+node_modules/
+.env
+.next
+fix-*.js
+
+# Virtual Environments
+venv/
+.venv/
+env/
+
+# Claude Code
+.claude/
+
+# Local Deployment & Platform
+/agents-platform/
+launch_agencia_ai.ps1
+
+# Local Strategy Assets (opcional)
+# Strategy/
+
+# Backups & Data
+*.sql
+*.tar.gz
+*.zip
+
+# Agent & Platform Tools
+.mcp.json
+.agent/
+.agents/
+.gemini/
+.claude/
+
+# Logs & Debug
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*

finobs-0.0.1/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: finobs
+Version: 0.0.1
+Summary: CLI tool to monitor, analyze and optimize LLM agents in production
+Project-URL: Repository, https://github.com/fmonfasani/finobs
+License: MIT
+Keywords: agents,cli,llm,monitoring,observability
+Requires-Python: >=3.11
+Requires-Dist: anthropic>=0.25
+Requires-Dist: httpx>=0.27
+Requires-Dist: numpy>=1.26
+Requires-Dist: rich>=13
+Requires-Dist: typer[all]>=0.12
+Provides-Extra: dev
+Requires-Dist: hatch; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.110; extra == 'fastapi'
+Requires-Dist: uvicorn[standard]; extra == 'fastapi'
+Description-Content-Type: text/markdown
+
+# finobs — LLM Agent Observability Suite
+
+Monitor, analyze and optimize LLM agents in production.
+
+```bash
+pip install finobs
+```
+
+## Commands
+
+```bash
+finobs audit run_123           # Full observability audit (5 agents in parallel)
+finobs trace run_123           # Visualize execution trace
+finobs loops run_123           # Detect loops and redundant steps
+finobs cost run_123            # Cost breakdown by run
+finobs cost tenant_abc         # Cost breakdown by tenant
+finobs performance run_123     # Latency analysis (p50/p95/p99)
+finobs debug run_123           # Debug failed steps
+finobs debug run_123 --llm     # Debug with Claude semantic analysis (Phase 4)
+```
+
+## Instrument your agent
+
+```python
+from finobs import traced, flush_trace
+
+@traced("search_documents")
+def search_documents(query: str) -> str:
+    ...
+
+@traced("llm_call")
+def call_llm(prompt: str) -> str:
+    ...
+
+# At end of run:
+flush_trace(run_id="run_123", tenant="my_tenant")
+```
+
+## Generate test traces
+
+```python
+from finobs.scripts.trace_generator import generate_trace, Scenario
+
+generate_trace(Scenario.LOOP)           # loop detection
+generate_trace(Scenario.LATENCY_SPIKE)  # p95 spike
+generate_trace(Scenario.TOKEN_WASTE)    # cost waste
+generate_trace(Scenario.MULTI_FAILURE)  # combined issues
+```
+
+Then audit:
+```bash
+finobs audit loop_xxxxxx
+```
+
+## Scoring
+
+| Dimension   | Weight |
+|-------------|--------|
+| Performance | 25%    |
+| Quality     | 25%    |
+| Cost        | 20%    |
+| Stability   | 15%    |
+| RAG         | 15%    |
+
+Bands: 🟢 85–100 HEALTHY · 🟡 65–84 DEGRADED · 🔴 0–64 CRITICAL
+
+## Development
+
+```bash
+git clone https://github.com/tu-org/finobs
+cd finobs
+pip install -e ".[dev,fastapi]"
+pytest tests/ -v
+```
+
+## Roadmap
+
+- [x] Phase 1 — CLI + trace interceptor + 5 analysis agents
+- [x] Phase 2 — Scoring system + report generation
+- [x] Phase 3 — Deterministic scripts (loop detector, cost analyzer, metrics)
+- [ ] Phase 4 — Claude LLM semantic analysis (`--llm` flag)
+- [ ] Phase 5 — Self-healing + A/B prompt comparison + degradation detection

finobs-0.0.1/README.md

@@ -0,0 +1,82 @@
+# finobs — LLM Agent Observability Suite
+
+Monitor, analyze and optimize LLM agents in production.
+
+```bash
+pip install finobs
+```
+
+## Commands
+
+```bash
+finobs audit run_123           # Full observability audit (5 agents in parallel)
+finobs trace run_123           # Visualize execution trace
+finobs loops run_123           # Detect loops and redundant steps
+finobs cost run_123            # Cost breakdown by run
+finobs cost tenant_abc         # Cost breakdown by tenant
+finobs performance run_123     # Latency analysis (p50/p95/p99)
+finobs debug run_123           # Debug failed steps
+finobs debug run_123 --llm     # Debug with Claude semantic analysis (Phase 4)
+```
+
+## Instrument your agent
+
+```python
+from finobs import traced, flush_trace
+
+@traced("search_documents")
+def search_documents(query: str) -> str:
+    ...
+
+@traced("llm_call")
+def call_llm(prompt: str) -> str:
+    ...
+
+# At end of run:
+flush_trace(run_id="run_123", tenant="my_tenant")
+```
+
+## Generate test traces
+
+```python
+from finobs.scripts.trace_generator import generate_trace, Scenario
+
+generate_trace(Scenario.LOOP)           # loop detection
+generate_trace(Scenario.LATENCY_SPIKE)  # p95 spike
+generate_trace(Scenario.TOKEN_WASTE)    # cost waste
+generate_trace(Scenario.MULTI_FAILURE)  # combined issues
+```
+
+Then audit:
+```bash
+finobs audit loop_xxxxxx
+```
+
+## Scoring
+
+| Dimension   | Weight |
+|-------------|--------|
+| Performance | 25%    |
+| Quality     | 25%    |
+| Cost        | 20%    |
+| Stability   | 15%    |
+| RAG         | 15%    |
+
+Bands: 🟢 85–100 HEALTHY · 🟡 65–84 DEGRADED · 🔴 0–64 CRITICAL
+
+## Development
+
+```bash
+git clone https://github.com/tu-org/finobs
+cd finobs
+pip install -e ".[dev,fastapi]"
+pytest tests/ -v
+```
+
+## Roadmap
+
+- [x] Phase 1 — CLI + trace interceptor + 5 analysis agents
+- [x] Phase 2 — Scoring system + report generation
+- [x] Phase 3 — Deterministic scripts (loop detector, cost analyzer, metrics)
+- [ ] Phase 4 — Claude LLM semantic analysis (`--llm` flag)
+- [ ] Phase 5 — Self-healing + A/B prompt comparison + degradation detection

finobs-0.0.1/examples/sample_agent/main.py

@@ -0,0 +1,83 @@
+"""
+Sample FastAPI agent instrumented with finobs @traced decorator.
+Run: uvicorn examples.sample_agent.main:app --reload --port 8001
+"""
+import uuid
+import random
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+from finobs import traced, flush_trace
+from finobs.scripts.trace_interceptor import set_run_id
+
+app = FastAPI(title="Sample LLM Agent")
+
+
+class TaskRequest(BaseModel):
+    task: str
+    tenant: str = "default"
+
+
+class TaskResponse(BaseModel):
+    run_id: str
+    result: str
+    trace_path: str
+
+
+@traced("search_documents")
+def search_documents(query: str) -> str:
+    # Simulate RAG retrieval
+    import time; time.sleep(random.uniform(0.1, 0.5))
+    return f"Found 3 documents relevant to: {query}"
+
+
+@traced("llm_call")
+def llm_call(prompt: str) -> str:
+    # Simulate LLM call latency
+    import time; time.sleep(random.uniform(0.3, 1.2))
+    return f"LLM response for: {prompt[:50]}"
+
+
+@traced("validate_output")
+def validate_output(output: str) -> str:
+    import time; time.sleep(0.05)
+    if random.random() < 0.1:
+        raise ValueError("Validation failed")
+    return "valid"
+
+
+@traced("write_file")
+def write_file(content: str) -> str:
+    import time; time.sleep(0.05)
+    return f"Written {len(content)} chars"
+
+
+@app.post("/invoke", response_model=TaskResponse)
+async def invoke(request: TaskRequest):
+    run_id = str(uuid.uuid4())[:8]
+    set_run_id(run_id)
+
+    # Simulate agent execution
+    docs = search_documents(request.task)
+    response = llm_call(f"Given: {docs}. Task: {request.task}")
+    search_documents(request.task)   # Intentional loop for demo
+    llm_call(f"Refine: {response}")
+    search_documents(request.task)   # Loop again
+    try:
+        validate_output(response)
+    except ValueError:
+        pass  # captured in trace as error
+    write_file(response)
+
+    trace_path = flush_trace(run_id=run_id, tenant=request.tenant)
+
+    return TaskResponse(
+        run_id=run_id,
+        result=response,
+        trace_path=trace_path,
+    )
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}

finobs-0.0.1/finobs/__init__.py

@@ -0,0 +1,7 @@
+"""finobs — LLM Agent Observability Suite"""
+
+__version__ = "0.1.0"
+
+from finobs.scripts.trace_interceptor import traced, flush_trace
+
+__all__ = ["traced", "flush_trace"]

finobs-0.0.1/finobs/agents/__init__.py

@@ -0,0 +1,3 @@
+from finobs.agents import latency, quality, cost, behavior, rag
+
+__all__ = ["latency", "quality", "cost", "behavior", "rag"]

finobs-0.0.1/finobs/agents/behavior.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass, field
+from finobs.scripts.trace_parser import Trace
+from finobs.scripts.loop_detector import detect_loops
+from finobs.scripts.scoring import score_stability
+
+
+@dataclass
+class AgentResult:
+    agent: str
+    score: int
+    summary: str
+    issues: list[dict] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+
+
+async def analyze(trace: Trace) -> AgentResult:
+    steps = trace.steps
+    loops = detect_loops(steps)
+    error_rate = sum(1 for s in steps if s.status == "error") / max(len(steps), 1)
+    score = score_stability(len(loops), error_rate, len(steps))
+
+    issues = []
+    for loop in loops:
+        issues.append({
+            "severity": "high",
+            "description": (
+                f"Loop detected: tool='{loop['tool']}' "
+                f"repeated {loop['occurrences']}x at steps {loop['step_indices']}"
+            ),
+            "recommendation": "Add loop guard — check result before re-invoking same tool with same input",
+        })
+
+    if len(steps) > 40:
+        issues.append({
+            "severity": "medium",
+            "description": f"Excessive step count: {len(steps)} steps",
+            "recommendation": "Review planner — may be generating redundant sub-tasks",
+        })
+
+    metrics = {
+        "total_steps": len(steps),
+        "loops_detected": len(loops),
+        "loop_detail": loops,
+        "error_rate": round(error_rate, 3),
+    }
+
+    summary = f"{len(loops)} loops" if loops else "no loops"
+    return AgentResult(agent="behavior", score=score, summary=summary, issues=issues, metrics=metrics)

finobs-0.0.1/finobs/agents/cost.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass, field
+from finobs.scripts.trace_parser import Trace
+from finobs.scripts.cost_analyzer import compute_cost
+from finobs.scripts.scoring import score_cost
+
+
+@dataclass
+class AgentResult:
+    agent: str
+    score: int
+    summary: str
+    issues: list[dict] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+
+
+async def analyze(trace: Trace) -> AgentResult:
+    result = compute_cost(trace.steps)
+    score = score_cost(result["total_usd"], len(result["waste_steps"]))
+
+    issues = []
+    from finobs.scripts.scoring import THRESHOLDS
+    if result["total_usd"] > THRESHOLDS["cost_per_run_usd"]:
+        issues.append({
+            "severity": "high",
+            "description": f"Run cost ${result['total_usd']:.4f} exceeds budget (${THRESHOLDS['cost_per_run_usd']})",
+            "recommendation": "Consider using a cheaper model for non-critical steps",
+        })
+    if result["waste_steps"]:
+        issues.append({
+            "severity": "medium",
+            "description": f"{len(result['waste_steps'])} steps with high input/output token ratio",
+            "recommendation": "Trim context window — avoid re-injecting full history each step",
+        })
+
+    summary = f"${result['total_usd']:.4f}"
+    return AgentResult(agent="cost", score=score, summary=summary, issues=issues, metrics=result)

finobs-0.0.1/finobs/agents/latency.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass, field
+from finobs.scripts.trace_parser import Trace
+from finobs.scripts.metrics_aggregator import compute_latency_metrics
+from finobs.scripts.scoring import score_latency
+
+
+@dataclass
+class AgentResult:
+    agent: str
+    score: int
+    summary: str
+    issues: list[dict] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+
+
+async def analyze(trace: Trace) -> AgentResult:
+    metrics = compute_latency_metrics(trace.steps)
+    error_rate = sum(1 for s in trace.steps if s.status == "error") / max(len(trace.steps), 1)
+
+    score = score_latency(metrics["p95"], metrics["p99"], error_rate)
+    issues = []
+
+    from finobs.scripts.scoring import THRESHOLDS
+    if metrics["p95"] > THRESHOLDS["latency_p95_ms"]:
+        issues.append({
+            "severity": "high",
+            "description": f"p95 latency {metrics['p95']:.0f}ms exceeds threshold ({THRESHOLDS['latency_p95_ms']}ms)",
+            "recommendation": "Profile slowest steps and consider caching or parallelization",
+        })
+
+    if error_rate > THRESHOLDS["error_rate_max"]:
+        issues.append({
+            "severity": "high",
+            "description": f"Error rate {error_rate:.1%} exceeds threshold",
+            "recommendation": "Review failed steps and add retry logic",
+        })
+
+    summary = f"p95: {metrics['p95']:.0f}ms"
+    return AgentResult(agent="latency", score=score, summary=summary, issues=issues, metrics=metrics)

finobs-0.0.1/finobs/agents/quality.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass, field
+from finobs.scripts.trace_parser import Trace
+
+
+@dataclass
+class AgentResult:
+    agent: str
+    score: int
+    summary: str
+    issues: list[dict] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+
+
+async def analyze(trace: Trace) -> AgentResult:
+    steps = trace.steps
+    total = len(steps)
+    errors = [s for s in steps if s.status == "error"]
+    error_rate = len(errors) / max(total, 1)
+
+    # Detectar pasos con output muy bajo (posible respuesta vacía)
+    low_output = [s for s in steps if s.output_tokens < 10]
+    low_output_rate = len(low_output) / max(total, 1)
+
+    score = 100
+    score -= int(error_rate * 100 * 1.5)
+    score -= int(low_output_rate * 100 * 0.8)
+    score = max(0, score)
+
+    issues = []
+    if error_rate > 0.05:
+        issues.append({
+            "severity": "high",
+            "description": f"High failure rate: {error_rate:.1%} of steps failed",
+            "recommendation": "Add input validation before tool calls",
+        })
+    if low_output_rate > 0.10:
+        issues.append({
+            "severity": "medium",
+            "description": f"{len(low_output)} steps produced near-empty outputs",
+            "recommendation": "Check prompt clarity and context sufficiency",
+        })
+
+    metrics = {
+        "total_steps": total,
+        "error_count": len(errors),
+        "error_rate": round(error_rate, 3),
+        "low_output_steps": len(low_output),
+    }
+
+    summary = f"error_rate: {error_rate:.1%}"
+    return AgentResult(agent="quality", score=score, summary=summary, issues=issues, metrics=metrics)

finobs-0.0.1/finobs/agents/rag.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass, field
+from finobs.scripts.trace_parser import Trace
+from finobs.scripts.scoring import score_rag
+
+
+@dataclass
+class AgentResult:
+    agent: str
+    score: int
+    summary: str
+    issues: list[dict] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+
+
+async def analyze(trace: Trace) -> AgentResult:
+    rag_steps = [s for s in trace.steps if "rag" in s.tool.lower() or "retrieve" in s.tool.lower()]
+
+    if not rag_steps:
+        return AgentResult(
+            agent="rag",
+            score=100,
+            summary="no RAG steps",
+            metrics={"rag_steps": 0},
+        )
+
+    # Proxy de relevancia: ratio output/input tokens (bajo = recuperó pero no usó)
+    relevance_scores = []
+    for s in rag_steps:
+        ratio = s.output_tokens / max(s.input_tokens, 1)
+        relevance_scores.append(min(1.0, ratio * 2))
+
+    avg_relevance = sum(relevance_scores) / len(relevance_scores)
+    empty_steps = [s for s in rag_steps if s.output_tokens < 15]
+    empty_rate = len(empty_steps) / len(rag_steps)
+
+    score = score_rag(avg_relevance, empty_rate)
+    issues = []
+
+    from finobs.scripts.scoring import THRESHOLDS
+    if avg_relevance < THRESHOLDS["rag_relevance_min"]:
+        issues.append({
+            "severity": "high",
+            "description": f"Low RAG utilization: avg relevance proxy {avg_relevance:.2f}",
+            "recommendation": "Review chunking strategy and embedding model",
+        })
+    if empty_rate > 0.2:
+        issues.append({
+            "severity": "medium",
+            "description": f"{len(empty_steps)}/{len(rag_steps)} retrievals returned near-empty results",
+            "recommendation": "Check retrieval thresholds and fallback strategy",
+        })
+
+    metrics = {
+        "rag_steps": len(rag_steps),
+        "avg_relevance_proxy": round(avg_relevance, 3),
+        "empty_retrieval_rate": round(empty_rate, 3),
+    }
+
+    summary = f"relevance: {avg_relevance:.2f}"
+    return AgentResult(agent="rag", score=score, summary=summary, issues=issues, metrics=metrics)

finobs-0.0.1/finobs/cli.py

@@ -0,0 +1,19 @@
+import typer
+from finobs.commands import audit, trace, loops, cost, performance, debug
+
+app = typer.Typer(
+    name="finobs",
+    help="finobs — LLM Agent Observability Suite",
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+)
+
+app.add_typer(audit.app,       name="audit",       help="Full observability audit of an agent run")
+app.add_typer(trace.app,       name="trace",        help="Reconstruct and visualize execution trace")
+app.add_typer(loops.app,       name="loops",        help="Detect loops and redundant steps")
+app.add_typer(cost.app,        name="cost",         help="Cost analysis by run or tenant")
+app.add_typer(performance.app, name="performance",  help="Latency and throughput analysis")
+app.add_typer(debug.app,       name="debug",        help="Deep debug of a failed run")
+
+if __name__ == "__main__":
+    app()