swarmtrace 0.1.0__tar.gz

swarmtrace-0.1.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: swarmtrace
+Version: 0.1.0
+Summary: pytest for AI agents — trace, debug and catch regressions in LLM swarms
+Author: Ravi
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: litai
+Requires-Dist: click
+Requires-Dist: rich
+Dynamic: author
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Tracely
+> Open-source AI observability framework. Trace, debug and detect regressions in LLM apps with one decorator.
+
+## Install
+pip install tracely
+
+## Usage
+from tracely import observe
+
+@observe
+def my_agent(question):
+    return llm.chat(question)
+
+## Features
+- One decorator, zero config
+- Latency tracking
+- Error capture
+- AI-powered regression detection
+- CLI: tracely view
+- Works with any LLM
+
+## Why Not LangSmith?
+| | LangSmith | Tracely |
+|---|---|---|
+| Open Source | No | Yes |
+| Any Framework | No | Yes |
+| Self-hosted | No | Yes |
+| Regression Detection | No | Yes |
+| Setup | Complex | One decorator |

swarmtrace-0.1.0/README.md

@@ -0,0 +1,29 @@
+# Tracely
+> Open-source AI observability framework. Trace, debug and detect regressions in LLM apps with one decorator.
+
+## Install
+pip install tracely
+
+## Usage
+from tracely import observe
+
+@observe
+def my_agent(question):
+    return llm.chat(question)
+
+## Features
+- One decorator, zero config
+- Latency tracking
+- Error capture
+- AI-powered regression detection
+- CLI: tracely view
+- Works with any LLM
+
+## Why Not LangSmith?
+| | LangSmith | Tracely |
+|---|---|---|
+| Open Source | No | Yes |
+| Any Framework | No | Yes |
+| Self-hosted | No | Yes |
+| Regression Detection | No | Yes |
+| Setup | Complex | One decorator |

swarmtrace-0.1.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

swarmtrace-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

swarmtrace-0.1.0/setup.py

@@ -0,0 +1,14 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="swarmtrace",
+    version="0.1.0",
+    description="pytest for AI agents — trace, debug and catch regressions in LLM swarms",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    author="Ravi",
+    packages=find_packages(),
+    install_requires=["litai", "click", "rich"],
+    entry_points={"console_scripts": ["swarmtrace=tracely.cli:view"]},
+    python_requires=">=3.8",
+)

swarmtrace-0.1.0/swarmtrace.egg-info/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: swarmtrace
+Version: 0.1.0
+Summary: pytest for AI agents — trace, debug and catch regressions in LLM swarms
+Author: Ravi
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: litai
+Requires-Dist: click
+Requires-Dist: rich
+Dynamic: author
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Tracely
+> Open-source AI observability framework. Trace, debug and detect regressions in LLM apps with one decorator.
+
+## Install
+pip install tracely
+
+## Usage
+from tracely import observe
+
+@observe
+def my_agent(question):
+    return llm.chat(question)
+
+## Features
+- One decorator, zero config
+- Latency tracking
+- Error capture
+- AI-powered regression detection
+- CLI: tracely view
+- Works with any LLM
+
+## Why Not LangSmith?
+| | LangSmith | Tracely |
+|---|---|---|
+| Open Source | No | Yes |
+| Any Framework | No | Yes |
+| Self-hosted | No | Yes |
+| Regression Detection | No | Yes |
+| Setup | Complex | One decorator |

swarmtrace-0.1.0/swarmtrace.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+setup.py
+swarmtrace.egg-info/PKG-INFO
+swarmtrace.egg-info/SOURCES.txt
+swarmtrace.egg-info/dependency_links.txt
+swarmtrace.egg-info/entry_points.txt
+swarmtrace.egg-info/requires.txt
+swarmtrace.egg-info/top_level.txt
+tracely/__init__.py
+tracely/cli.py
+tracely/regression.py
+tracely/replay.py
+tracely/storage.py
+tracely/tracer.py

swarmtrace-0.1.0/swarmtrace.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

swarmtrace-0.1.0/swarmtrace.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+swarmtrace = tracely.cli:view

swarmtrace-0.1.0/swarmtrace.egg-info/requires.txt

@@ -0,0 +1,3 @@
+litai
+click
+rich

swarmtrace-0.1.0/swarmtrace.egg-info/top_level.txt

@@ -0,0 +1 @@
+tracely

swarmtrace-0.1.0/tracely/__init__.py

@@ -0,0 +1,5 @@
+from tracely.tracer import observe
+from tracely.storage import get_traces, save_trace
+
+__version__ = "0.1.0"
+__all__ = ["observe", "get_traces"]

swarmtrace-0.1.0/tracely/cli.py

@@ -0,0 +1,53 @@
+import sqlite3
+import os
+import sys
+
+DB_PATH = os.path.expanduser("~/.tracely.db")
+
+def get_traces():
+    conn = sqlite3.connect(DB_PATH)
+    rows = conn.execute("SELECT * FROM traces ORDER BY timestamp DESC LIMIT 20").fetchall()
+    conn.close()
+    return rows
+
+def print_tree(traces, parent_id=None, indent=0):
+    children = [t for t in traces if t[1] == parent_id]
+    for t in children:
+        id_, par, func, args, output, latency, error, timestamp, in_tok, out_tok, cost = t
+        status = "ERROR" if error else "OK"
+        prefix = "    " * indent + ("└── " if indent > 0 else "")
+        print(f"{prefix}{func}() [{id_}] {latency}s | {in_tok}in/{out_tok}out tokens | ${cost} | {status}")
+        print_tree(traces, id_, indent + 1)
+
+def view():
+    try:
+        traces = get_traces()
+    except:
+        traces = []
+
+    if not traces:
+        print("No traces found.")
+        return
+
+    total_cost = sum(t[10] for t in traces if t[10])
+    total_tokens = sum((t[8] or 0) + (t[9] or 0) for t in traces)
+
+    print("\n=== Tracely Trace View ===")
+    print(f"{'ID':<10} {'FUNCTION':<20} {'LATENCY':<10} {'TOKENS':<15} {'COST':<12} {'STATUS'}")
+    print("-" * 80)
+    for t in traces:
+        id_, parent_id, func, args, output, latency, error, timestamp, in_tok, out_tok, cost = t
+        status = "ERROR" if error else "OK"
+        tokens_str = f"{in_tok or 0}in/{out_tok or 0}out"
+        print(f"{id_:<10} {func:<20} {str(latency)+'s':<10} {tokens_str:<15} ${cost or 0:<11} {status}")
+
+    print("\n=== Tree View ===")
+    print_tree(traces)
+
+    print("\n=== Summary ===")
+    print(f"Total traces : {len(traces)}")
+    print(f"Total tokens : {total_tokens}")
+    print(f"Total cost   : ${round(total_cost, 6)}")
+
+if __name__ == "__main__":
+    view()

swarmtrace-0.1.0/tracely/regression.py

@@ -0,0 +1,68 @@
+import sys
+sys.path.insert(0, "/teamspace/studios/this_studio/tracely")
+
+import os
+from litai import LLM
+from tracely.storage import get_traces, save_trace
+import uuid
+from datetime import datetime
+
+llm = LLM(model="anthropic/claude-haiku-4-5-20251001", api_key=os.environ.get("LIGHTNING_API_KEY"))
+
+def score_similarity(output_a: str, output_b: str) -> float:
+    """Use AI to score how similar two outputs are. Returns 0.0 to 1.0"""
+    prompt = f"""Compare these two AI outputs and return ONLY a number between 0.0 and 1.0.
+1.0 = identical meaning. 0.0 = completely different.
+
+Output A: {output_a[:300]}
+Output B: {output_b[:300]}
+
+Reply with just the number, nothing else."""
+
+    score = llm.chat(prompt).strip()
+    try:
+        return float(score)
+    except:
+        return 0.5
+
+def compare(func, inputs: list, version_a_prompt: str, version_b_prompt: str):
+    """
+    Compare two prompt versions against the same inputs.
+    Detects regressions automatically.
+    """
+    print(f"\n[Tracely Regression] Comparing v1 vs v2 on {len(inputs)} inputs...\n")
+    print(f"{'INPUT':<30} {'V1 LATENCY':<12} {'V2 LATENCY':<12} {'SIMILARITY':<12} {'REGRESSION?'}")
+    print("-" * 85)
+
+    regressions = 0
+
+    for input_text in inputs:
+        # Run v1
+        import time
+        start = time.time()
+        out_a = func(input_text, version_a_prompt)
+        lat_a = round(time.time() - start, 2)
+
+        # Run v2
+        start = time.time()
+        out_b = func(input_text, version_b_prompt)
+        lat_b = round(time.time() - start, 2)
+
+        # Score similarity
+        similarity = score_similarity(out_a, out_b)
+        regressed = similarity < 0.6
+
+        if regressed:
+            regressions += 1
+
+        flag = "🔴 YES" if regressed else "✅ NO"
+        short_input = input_text[:28] + ".." if len(input_text) > 28 else input_text
+        print(f"{short_input:<30} {str(lat_a)+'s':<12} {str(lat_b)+'s':<12} {str(similarity):<12} {flag}")
+
+    print(f"\n{'='*85}")
+    print(f"Result: {regressions}/{len(inputs)} regressions detected")
+    if regressions > 0:
+        print("⚠️  WARNING: Your new prompt may have regressed!")
+    else:
+        print("✅ No regressions. Safe to ship.")
+    print()

swarmtrace-0.1.0/tracely/replay.py

@@ -0,0 +1,40 @@
+import sys
+sys.path.insert(0, "/teamspace/studios/this_studio/tracely")
+
+from tracely.storage import get_by_id, get_traces
+
+def replay(trace_id: str):
+    trace = get_by_id(trace_id)
+    if not trace:
+        print(f"Trace {trace_id} not found.")
+        return
+
+    id_, parent_id, func, args, output, latency, error, timestamp, in_tok, out_tok, cost = trace
+
+    print("\n=== Tracely Replay: " + trace_id + " ===")
+    print("Function  : " + str(func))
+    print("Timestamp : " + str(timestamp))
+    print("Args      : " + str(args))
+    print("Output    : " + str(output))
+    print("Latency   : " + str(latency) + "s")
+    print("Tokens    : " + str(in_tok) + " in / " + str(out_tok) + " out")
+    print("Cost      : $" + str(cost))
+    print("Error     : " + str(error if error else "None"))
+    print("Parent    : " + str(parent_id if parent_id else "root"))
+
+def show_failures():
+    traces = get_traces(limit=50)
+    failed = [t for t in traces if t[6]]
+
+    if not failed:
+        print("No failures found.")
+        return
+
+    print("\n=== Failed Traces ===")
+    print(f"{'ID':<10} {'FUNCTION':<20} {'ERROR':<40} {'TIMESTAMP'}")
+    print("-" * 90)
+    for t in failed:
+        id_, parent_id, func, args, output, latency, error, timestamp, in_tok, out_tok, cost = t
+        print(f"{id_:<10} {func:<20} {str(error)[:38]:<40} {timestamp}")
+    print(f"\nTotal failures: {len(failed)}")
+    print("\nReplay any failure: from tracely.replay import replay; replay('id')")

swarmtrace-0.1.0/tracely/storage.py

@@ -0,0 +1,63 @@
+import sqlite3
+import os
+
+DB_PATH = os.path.expanduser("~/.tracely.db")
+
+def init_db():
+    conn = sqlite3.connect(DB_PATH)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS traces (
+            id TEXT PRIMARY KEY,
+            parent_id TEXT,
+            function TEXT,
+            args TEXT,
+            output TEXT,
+            latency_sec REAL,
+            error TEXT,
+            timestamp TEXT,
+            input_tokens INTEGER,
+            output_tokens INTEGER,
+            cost_usd REAL
+        )
+    """)
+    conn.commit()
+    conn.close()
+
+def save_trace(trace: dict):
+    init_db()
+    conn = sqlite3.connect(DB_PATH)
+    conn.execute("""
+        INSERT OR REPLACE INTO traces
+        VALUES (:id, :parent_id, :function, :args, :output,
+                :latency_sec, :error, :timestamp,
+                :input_tokens, :output_tokens, :cost_usd)
+    """, trace)
+    conn.commit()
+    conn.close()
+
+def get_traces(limit=20):
+    init_db()
+    conn = sqlite3.connect(DB_PATH)
+    rows = conn.execute(
+        "SELECT * FROM traces ORDER BY timestamp DESC LIMIT ?", (limit,)
+    ).fetchall()
+    conn.close()
+    return rows
+
+def get_by_id(trace_id):
+    init_db()
+    conn = sqlite3.connect(DB_PATH)
+    row = conn.execute(
+        "SELECT * FROM traces WHERE id = ?", (trace_id,)
+    ).fetchone()
+    conn.close()
+    return row
+
+def get_tree(parent_id=None):
+    init_db()
+    conn = sqlite3.connect(DB_PATH)
+    rows = conn.execute(
+        "SELECT * FROM traces WHERE parent_id IS ? ORDER BY timestamp ASC", (parent_id,)
+    ).fetchall()
+    conn.close()
+    return rows

swarmtrace-0.1.0/tracely/tracer.py

@@ -0,0 +1,137 @@
+import time
+import uuid
+import functools
+import threading
+import asyncio
+from datetime import datetime
+from tracely.storage import save_trace
+
+_local = threading.local()
+
+# Pricing per million tokens (Haiku 4.5)
+PRICING = {
+    "anthropic/claude-haiku-4-5-20251001": {"input": 0.80, "output": 4.00},
+    "openai/gpt-4o-mini":                  {"input": 0.15, "output": 0.60},
+    "openai/gpt-4o":                       {"input": 2.50, "output": 10.00},
+    "default":                             {"input": 1.00, "output": 3.00},
+}
+
+def get_cost(model, input_tokens, output_tokens):
+    price = PRICING.get(model, PRICING["default"])
+    return round(
+        (input_tokens * price["input"] / 1_000_000) +
+        (output_tokens * price["output"] / 1_000_000), 8
+    )
+
+def extract_real_tokens(result):
+    """Try to extract real token counts from LLM response objects."""
+    # Anthropic SDK response
+    if hasattr(result, "usage"):
+        usage = result.usage
+        return getattr(usage, "input_tokens", 0), getattr(usage, "output_tokens", 0)
+    # OpenAI SDK response
+    if hasattr(result, "usage") and hasattr(result.usage, "prompt_tokens"):
+        return result.usage.prompt_tokens, result.usage.completion_tokens
+    # Dict response
+    if isinstance(result, dict) and "usage" in result:
+        u = result["usage"]
+        return u.get("input_tokens") or u.get("prompt_tokens", 0),                u.get("output_tokens") or u.get("completion_tokens", 0)
+    return None, None
+
+def _get_parent_id():
+    return getattr(_local, "current_trace_id", None)
+
+def _set_parent_id(trace_id):
+    _local.current_trace_id = trace_id
+
+def _save(trace_id, parent_id, func_name, args, result, latency, error, model="default"):
+    # Try real tokens first, fall back to estimate
+    input_tokens, output_tokens = extract_real_tokens(result)
+    source = "exact"
+    if input_tokens is None:
+        input_tokens = max(1, len(str(args)) // 4)
+        output_tokens = max(1, len(str(result)) // 4) if result else 0
+        source = "estimated"
+
+    cost = get_cost(model, input_tokens, output_tokens)
+
+    indent = "  " if parent_id else ""
+    status = "✗ FAILED" if error else "✓ done"
+    print(f"[Tracely] {indent}{status}: {func_name} | {latency}s | {input_tokens}in/{output_tokens}out ({source}) | ${cost}")
+
+    save_trace({
+        "id": trace_id,
+        "parent_id": parent_id,
+        "function": func_name,
+        "args": str(args)[:200],
+        "output": str(result)[:200] if result else None,
+        "latency_sec": latency,
+        "error": str(error) if error else None,
+        "timestamp": datetime.utcnow().isoformat(),
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "cost_usd": cost
+    })
+
+def observe(func=None, model="default"):
+    """
+    Usage:
+        @observe
+        def my_agent(): ...
+
+        @observe(model="anthropic/claude-haiku-4-5-20251001")
+        def my_agent(): ...
+    """
+    def decorator(f):
+        if asyncio.iscoroutinefunction(f):
+            @functools.wraps(f)
+            async def async_wrapper(*args, **kwargs):
+                trace_id = str(uuid.uuid4())[:8]
+                parent_id = _get_parent_id()
+                prev = parent_id
+                _set_parent_id(trace_id)
+                start = time.time()
+                error = None
+                result = None
+                indent = "  " if parent_id else ""
+                print(f"[Tracely] {indent}▶ {f.__name__} started (id={trace_id})")
+                try:
+                    result = await f(*args, **kwargs)
+                except Exception as e:
+                    error = e
+                finally:
+                    latency = round(time.time() - start, 3)
+                    _save(trace_id, parent_id, f.__name__, args, result, latency, error, model)
+                    _set_parent_id(prev)
+                if error:
+                    raise error
+                return result
+            return async_wrapper
+
+        @functools.wraps(f)
+        def sync_wrapper(*args, **kwargs):
+            trace_id = str(uuid.uuid4())[:8]
+            parent_id = _get_parent_id()
+            prev = parent_id
+            _set_parent_id(trace_id)
+            start = time.time()
+            error = None
+            result = None
+            indent = "  " if parent_id else ""
+            print(f"[Tracely] {indent}▶ {f.__name__} started (id={trace_id})")
+            try:
+                result = f(*args, **kwargs)
+            except Exception as e:
+                error = e
+            finally:
+                latency = round(time.time() - start, 3)
+                _save(trace_id, parent_id, f.__name__, args, result, latency, error, model)
+                _set_parent_id(prev)
+            if error:
+                raise error
+            return result
+        return sync_wrapper
+
+    if func is not None:
+        return decorator(func)
+    return decorator