cortexops 0.1.0__tar.gz → 0.2.0__tar.gz

{cortexops-0.1.0 → cortexops-0.2.0}/.gitignore

@@ -1,34 +1,34 @@
-# Python
-__pycache__/
-*.py[cod]
-*.pyo
-.venv/
-venv/
-.env
-*.egg-info/
-dist/
-build/
-PKG-INFO
-*.whl
-*.tar.gz
-
-# Test / lint caches
-.pytest_cache/
-.ruff_cache/
-.mypy_cache/
-
-# Package managers
-uv.lock
-.python-version
-
-# Database
-*.db
-*.sqlite
-
-# IDE
-.vscode/
-.idea/
-
-# OS
-.DS_Store
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+.venv/
+venv/
+.env
+*.egg-info/
+dist/
+build/
+PKG-INFO
+*.whl
+*.tar.gz
+
+# Test / lint caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# Package managers
+uv.lock
+.python-version
+
+# Database
+*.db
+*.sqlite
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
 Thumbs.db

{cortexops-0.1.0 → cortexops-0.2.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: cortexops
-Version: 0.1.0
+Version: 0.2.0
 Summary: Reliability infrastructure for AI agents — evaluation, observability, and regression testing
-Project-URL: Homepage, https://cortexops.ai
+Project-URL: Homepage, https://getcortexops.com
 Project-URL: Repository, https://github.com/ashishodu2023/cortexops
-Project-URL: Documentation, https://docs.cortexops.ai
+Project-URL: Documentation, https://docs.getcortexops.com
 Project-URL: Bug Tracker, https://github.com/ashishodu2023/cortexops/issues
 Project-URL: Changelog, https://github.com/ashishodu2023/cortexops/releases
-Author-email: Ashish <ashishodu2023@gmail.com>
+Author-email: Ashish <ashish@getcortexops.com>
 License: MIT License
         
         Copyright (c) 2025 CortexOps Contributors
@@ -31,7 +31,7 @@ License: MIT License
         SOFTWARE.
 License-File: LICENSE
 Keywords: agents,ai,autogen,crewai,evaluation,langgraph,llm,observability,testing
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -46,7 +46,6 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: setuptools>=82.0.1
 Provides-Extra: all
 Requires-Dist: httpx>=0.27; extra == 'all'
 Provides-Extra: dev

{cortexops-0.1.0 → cortexops-0.2.0}/cortexops/__init__.py

@@ -27,13 +27,14 @@ from .models import (
     EvalSummary,
     FailureKind,
     RunStatus,
+    ToolCall,
     Trace,
     TraceNode,
-    ToolCall,
 )
+from .auth import cmd_login, cmd_logout, cmd_whoami, save_credentials, load_credentials
 from .tracer import CortexTracer
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 __all__ = [
     "CortexTracer",
@@ -55,4 +56,9 @@ __all__ = [
     "CaseResult",
     "FailureKind",
     "RunStatus",
-]
+    "cmd_login",
+    "cmd_logout",
+    "cmd_whoami",
+    "save_credentials",
+    "load_credentials",
+]

cortexops-0.2.0/cortexops/auth.py

@@ -0,0 +1,149 @@
+"""
+cortexops login / logout / whoami — credential management.
+Stores API key in ~/.cortexops/credentials (JSON).
+Called by CLI and importable for programmatic use.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+_CREDENTIALS_DIR  = Path.home() / ".cortexops"
+_CREDENTIALS_FILE = _CREDENTIALS_DIR / "credentials"
+_DEFAULT_API_URL  = "https://api.getcortexops.com"
+
+
+def save_credentials(api_key: str, project: str, api_url: str = _DEFAULT_API_URL) -> None:
+    """Persist credentials to ~/.cortexops/credentials."""
+    _CREDENTIALS_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
+    creds = {"api_key": api_key, "project": project, "api_url": api_url}
+    _CREDENTIALS_FILE.write_text(json.dumps(creds, indent=2))
+    _CREDENTIALS_FILE.chmod(0o600)  # owner read/write only — no secrets leak
+
+
+def load_credentials() -> dict | None:
+    """Load credentials from ~/.cortexops/credentials. Returns None if not found."""
+    if not _CREDENTIALS_FILE.exists():
+        return None
+    try:
+        return json.loads(_CREDENTIALS_FILE.read_text())
+    except Exception:
+        return None
+
+
+def clear_credentials() -> None:
+    """Remove stored credentials."""
+    if _CREDENTIALS_FILE.exists():
+        _CREDENTIALS_FILE.unlink()
+
+
+def verify_key(api_key: str, api_url: str = _DEFAULT_API_URL) -> dict | None:
+    """
+    Call GET /health with the key to verify it is valid.
+    Returns {"project": ..., "environment": ...} on success, None on failure.
+    """
+    try:
+        import httpx
+        r = httpx.get(
+            f"{api_url.rstrip('/')}/health",
+            headers={"X-API-Key": api_key},
+            timeout=8.0,
+        )
+        if r.status_code == 200:
+            return r.json()
+        return None
+    except Exception:
+        return None
+
+
+def cmd_login(api_key: str | None = None, project: str | None = None,
+              api_url: str = _DEFAULT_API_URL) -> int:
+    """
+    Interactive login flow. Called by `cortexops login`.
+
+    If api_key is not provided, prompts interactively.
+    Returns 0 on success, 1 on failure.
+    """
+    print("CortexOps Login")
+    print("Get your API key at https://getcortexops.com/#pricing\n")
+
+    if not api_key:
+        try:
+            import getpass
+            api_key = getpass.getpass("API key (cxo-...): ").strip()
+        except KeyboardInterrupt:
+            print("\nCancelled.")
+            return 1
+
+    if not api_key.startswith("cxo-"):
+        print("Error: API key must start with 'cxo-'", file=sys.stderr)
+        return 1
+
+    if not project:
+        project = input("Default project name: ").strip() or "my-agent"
+
+    print(f"\nVerifying key against {api_url}...")
+    info = verify_key(api_key, api_url)
+    if info is None:
+        print(
+            "Warning: Could not verify key (API may be unreachable).\n"
+            "Saving credentials anyway — verify manually with: cortexops whoami",
+            file=sys.stderr,
+        )
+
+    save_credentials(api_key, project, api_url)
+    masked = api_key[:8] + "..." + api_key[-4:]
+    print("\n✓ Logged in")
+    print(f"  Key:     {masked}")
+    print(f"  Project: {project}")
+    print("  Stored:  ~/.cortexops/credentials")
+    print("\nYou can now use CortexTracer without passing api_key:")
+    print(f"  tracer = CortexTracer(project=\"{project}\")")
+    return 0
+
+
+def cmd_logout() -> int:
+    """Called by `cortexops logout`."""
+    creds = load_credentials()
+    if not creds:
+        print("Not logged in.")
+        return 0
+    clear_credentials()
+    print("✓ Logged out — credentials removed from ~/.cortexops/credentials")
+    return 0
+
+
+def cmd_whoami(api_url: str | None = None) -> int:
+    """Called by `cortexops whoami`."""
+    # Check env var first
+    env_key = os.getenv("CORTEXOPS_API_KEY")
+    file_creds = load_credentials()
+
+    if not env_key and not file_creds:
+        print("Not logged in.\nRun: cortexops login", file=sys.stderr)
+        return 1
+
+    if env_key:
+        print("API key source : CORTEXOPS_API_KEY (env)")
+        masked = env_key[:8] + "..." + env_key[-4:]
+        print(f"Key            : {masked}")
+        url = api_url or os.getenv("CORTEXOPS_API_URL", _DEFAULT_API_URL)
+    else:
+        masked = file_creds["api_key"][:8] + "..." + file_creds["api_key"][-4:]
+        print("API key source : ~/.cortexops/credentials")
+        print(f"Key            : {masked}")
+        print(f"Project        : {file_creds.get('project', '—')}")
+        url = api_url or file_creds.get("api_url", _DEFAULT_API_URL)
+        env_key = file_creds["api_key"]
+
+    print(f"API URL        : {url}")
+    print("\nVerifying...")
+    info = verify_key(env_key, url)
+    if info:
+        print(f"✓ Key is valid  (API status: {info.get('status', 'ok')})")
+    else:
+        print("✗ Key verification failed — API unreachable or key invalid")
+        return 1
+    return 0

{cortexops-0.1.0 → cortexops-0.2.0}/cortexops/cli.py

@@ -76,7 +76,8 @@ def cmd_eval_diff(args: argparse.Namespace) -> int:
     regressions = diff.get("regressions", [])
     improvements = diff.get("improvements", [])
 
-    sign = lambda v: f"+{v:.1%}" if v >= 0 else f"{v:.1%}"
+    def sign(v):
+        return f"+{v:.1%}" if v >= 0 else f"{v:.1%}"
     print(f"Diff: {args.run_a[:8]} → {args.run_b[:8]}")
     print(f"  Task completion : {sign(delta_tc)}")
     print(f"  Tool accuracy   : {sign(delta_tool / 100)}")
@@ -137,6 +138,29 @@ def _load_agent(agent_path: str):
     return getattr(module, attr)
 
 
+
+
+def cmd_login(args: argparse.Namespace) -> int:
+    """cortexops login"""
+    from cortexops.auth import cmd_login as _login
+    return _login(
+        api_key=getattr(args, 'api_key', None),
+        project=getattr(args, 'project', None),
+        api_url=getattr(args, 'base_url', 'https://api.getcortexops.com'),
+    )
+
+
+def cmd_logout(args: argparse.Namespace) -> int:
+    """cortexops logout"""
+    from cortexops.auth import cmd_logout as _logout
+    return _logout()
+
+
+def cmd_whoami(args: argparse.Namespace) -> int:
+    """cortexops whoami"""
+    from cortexops.auth import cmd_whoami as _whoami
+    return _whoami(api_url=getattr(args, 'base_url', None))
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         prog="cortexops",
@@ -169,12 +193,26 @@ def main() -> None:
     fail_p.add_argument("--api-key", default=None)
     fail_p.add_argument("--base-url", default="https://api.cortexops.ai")
 
+
+    # ── login / logout / whoami ───────────────────────────────────────────
+    login_p = sub.add_parser("login", help="Save API key to ~/.cortexops/credentials")
+    login_p.add_argument("--api-key", default=None, help="cxo-... key (prompted if omitted)")
+    login_p.add_argument("--project", "-p", default=None, help="Default project name")
+    login_p.add_argument("--base-url", default="https://api.getcortexops.com")
+
+    sub.add_parser("logout", help="Remove stored credentials")
+    whoami_p = sub.add_parser("whoami", help="Show current credentials and verify key")
+    whoami_p.add_argument("--base-url", default=None)
+
     # ── version ───────────────────────────────────────────────────────────
     sub.add_parser("version", help="Print version and exit")
 
     args = parser.parse_args()
 
     handlers = {
+        ("login", None): cmd_login,
+        ("logout", None): cmd_logout,
+        ("whoami", None): cmd_whoami,
         ("eval", "run"): cmd_eval_run,
         ("eval", "diff"): cmd_eval_diff,
         ("failures", None): cmd_failures,
@@ -192,4 +230,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

{cortexops-0.1.0 → cortexops-0.2.0}/cortexops/client.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from typing import Any
 from urllib.parse import urljoin
 
 from .models import EvalSummary, Trace

{cortexops-0.1.0 → cortexops-0.2.0}/cortexops/judge.py

@@ -22,7 +22,6 @@ from typing import Any
 from .metrics import Metric
 from .models import EvalCase, FailureKind, Trace
 
-
 JUDGE_SYSTEM_PROMPT = """You are a strict but fair evaluator of AI agent outputs.
 You will be given:
   - The user's input to the agent

{cortexops-0.1.0/cortexops → cortexops-0.2.0}/cortexops/metrics.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import re
 from abc import ABC, abstractmethod
-from typing import Any
 
 from .models import CaseResult, EvalCase, FailureKind, Trace
 
@@ -139,7 +138,7 @@ class HallucinationMetric(Metric):
         return 100.0, None, None
 
 
-def compute_case_result(case: EvalCase, trace: Trace, extra_metrics: "list[Metric] | None" = None) -> CaseResult:
+def compute_case_result(case: EvalCase, trace: Trace, extra_metrics: list[Metric] | None = None) -> CaseResult:
     metrics: list[Metric] = [
         TaskCompletionMetric(),
         ToolAccuracyMetric(),

{cortexops-0.1.0/cortexops → cortexops-0.2.0}/cortexops/models.py

@@ -139,3 +139,6 @@ class EvalSummary(BaseModel):
             for r in failing:
                 lines.append(f"    - {r.case_id}: {r.failure_kind or 'unknown'} (score {r.score:.0f})")
         return "\n".join(lines)
+
+    def __str__(self) -> str:
+        return self.summary()

{cortexops-0.1.0/cortexops → cortexops-0.2.0}/cortexops/tracer.py

@@ -1,21 +1,74 @@
 from __future__ import annotations
 
+import os
 import time
 import uuid
+from collections.abc import Callable
 from contextlib import contextmanager
-from typing import Any, Callable
+from pathlib import Path
+from typing import Any
+
+from .models import FailureKind, RunStatus, ToolCall, ToolCallStatus, Trace, TraceNode
+
+# ── Key resolution order ───────────────────────────────────────────────────
+# 1. Explicit api_key argument
+# 2. CORTEXOPS_API_KEY environment variable
+# 3. ~/.cortexops/credentials file  (written by `cortexops login`)
+# 4. None → local-only mode, no hosted tracing
+
+_CREDENTIALS_FILE = Path.home() / ".cortexops" / "credentials"
+_DEFAULT_API_URL   = "https://api.getcortexops.com"
+_ENV_KEY           = "CORTEXOPS_API_KEY"
+_ENV_URL           = "CORTEXOPS_API_URL"
+_ENV_PROJECT       = "CORTEXOPS_PROJECT"
+_ENV_ENV           = "CORTEXOPS_ENVIRONMENT"
+
+
+def _resolve_api_key(explicit: str | None) -> str | None:
+    """Resolve API key from multiple sources in priority order."""
+    if explicit:
+        return explicit
+    # Environment variable
+    if env_key := os.getenv(_ENV_KEY):
+        return env_key
+    # Credentials file written by `cortexops login`
+    if _CREDENTIALS_FILE.exists():
+        try:
+            import json
+            creds = json.loads(_CREDENTIALS_FILE.read_text())
+            return creds.get("api_key")
+        except Exception:
+            pass
+    return None
 
-from .models import FailureKind, RunStatus, Trace, TraceNode, ToolCall, ToolCallStatus
+
+def _resolve_api_url(explicit: str) -> str:
+    """Resolve API URL — explicit arg > env var > default."""
+    if explicit != _DEFAULT_API_URL:
+        return explicit.rstrip("/")
+    return os.getenv(_ENV_URL, _DEFAULT_API_URL).rstrip("/")
 
 
 class CortexTracer:
     """Instruments AI agents with zero-refactor tracing.
 
-    Supports LangGraph StateGraph and CrewAI Crew out of the box.
-    Falls back to a generic callable wrapper for any other agent type.
+    API key resolution order (most to least specific):
+      1. api_key argument
+      2. CORTEXOPS_API_KEY environment variable
+      3. ~/.cortexops/credentials (written by `cortexops login`)
+      4. None — local-only mode, traces stored in memory only
 
     Usage:
+        # Explicit key
+        tracer = CortexTracer(project="payments-agent", api_key="cxo-...")
+
+        # From environment variable (recommended for CI)
+        # export CORTEXOPS_API_KEY=cxo-...
         tracer = CortexTracer(project="payments-agent")
+
+        # After `cortexops login` (recommended for local dev)
+        tracer = CortexTracer(project="payments-agent")
+
         graph = tracer.wrap(your_langgraph_app)
         result = graph.invoke({"messages": [...]})
         trace  = tracer.last_trace()
@@ -23,30 +76,52 @@ class CortexTracer:
 
     def __init__(
         self,
-        project: str,
+        project: str | None = None,
         api_key: str | None = None,
-        environment: str = "development",
+        api_url: str = _DEFAULT_API_URL,
+        environment: str | None = None,
         sample_rate: float = 1.0,
         local_store: bool = True,
     ) -> None:
-        self.project = project
-        self.api_key = api_key
-        self.environment = environment
+        # Project: arg > env var
+        self.project     = project or os.getenv(_ENV_PROJECT) or "default"
+        # Key: auto-resolved from all sources
+        self.api_key     = _resolve_api_key(api_key)
+        self.api_url     = _resolve_api_url(api_url)
+        # Environment: arg > env var > "development"
+        self.environment = environment or os.getenv(_ENV_ENV, "development")
         self.sample_rate = sample_rate
         self.local_store = local_store
         self._traces: list[Trace] = []
         self._current_trace: Trace | None = None
 
+        # Inform user where key came from — only in development
+        if self.environment == "development" and self.api_key:
+            source = "argument"
+            if not api_key:
+                if os.getenv(_ENV_KEY):
+                    source = f"env:{_ENV_KEY}"
+                elif _CREDENTIALS_FILE.exists():
+                    source = "~/.cortexops/credentials"
+            if source != "argument":
+                import logging
+                logging.getLogger(__name__).debug(
+                    "CortexTracer: api_key loaded from %s", source
+                )
+
+    @property
+    def is_hosted(self) -> bool:
+        """True if traces will be shipped to the hosted API."""
+        return bool(self.api_key)
+
     def wrap(self, agent: Any) -> Any:
         """Auto-detect agent type and return an instrumented wrapper."""
         agent_type = type(agent).__name__
 
         if agent_type == "CompiledStateGraph":
             return self._wrap_langgraph(agent)
-
         if agent_type == "Crew":
             return self._wrap_crewai(agent)
-
         if callable(agent) or hasattr(agent, "invoke"):
             return self._wrap_callable(agent)
 
@@ -62,8 +137,7 @@ class CortexTracer:
             def invoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
                 return tracer._run_traced(
                     fn=lambda: graph.invoke(input, config, **kwargs),
-                    input=input,
-                    framework="langgraph",
+                    input=input, framework="langgraph",
                 )
 
             async def ainvoke(self_, input: dict, config: dict | None = None, **kwargs) -> dict:
@@ -71,8 +145,7 @@ class CortexTracer:
                 return await asyncio.get_event_loop().run_in_executor(
                     None, lambda: tracer._run_traced(
                         fn=lambda: graph.invoke(input, config, **kwargs),
-                        input=input,
-                        framework="langgraph",
+                        input=input, framework="langgraph",
                     )
                 )
 
@@ -91,8 +164,7 @@ class CortexTracer:
             def kickoff(self_, inputs: dict | None = None) -> Any:
                 return tracer._run_traced(
                     fn=lambda: crew.kickoff(inputs=inputs),
-                    input=inputs or {},
-                    framework="crewai",
+                    input=inputs or {}, framework="crewai",
                 )
 
             def __getattr__(self_, name: str):
@@ -104,7 +176,6 @@ class CortexTracer:
         tracer = self
 
         if hasattr(fn, "invoke"):
-            # Object with .invoke() — wrap that method
             original_invoke = fn.invoke
 
             class InvokeWrapper:
@@ -121,7 +192,6 @@ class CortexTracer:
 
             return InvokeWrapper()
 
-        # Plain callable
         def wrapper(*args, **kwargs):
             input_data = {"args": list(args), "kwargs": kwargs}
             return tracer._run_traced(fn=lambda: fn(*args, **kwargs), input=input_data, framework="generic")
@@ -129,10 +199,11 @@ class CortexTracer:
         return wrapper
 
     def _run_traced(self, fn: Callable, input: dict, framework: str) -> Any:
-        trace = Trace(
-            project=self.project,
-            input=input,
-        )
+        import random
+        if self.sample_rate < 1.0 and random.random() > self.sample_rate:
+            return fn()
+
+        trace = Trace(project=self.project, input=input)
         self._current_trace = trace
         t0 = time.perf_counter()
 
@@ -176,12 +247,9 @@ class CortexTracer:
     ) -> ToolCall:
         """Manually record a tool call onto the current active trace."""
         tc = ToolCall(
-            name=name,
-            args=args or {},
-            result=result,
+            name=name, args=args or {}, result=result,
             status=ToolCallStatus.ERROR if error else ToolCallStatus.SUCCESS,
-            latency_ms=latency_ms,
-            error=error,
+            latency_ms=latency_ms, error=error,
         )
         if self._current_trace and self._current_trace.nodes:
             self._current_trace.nodes[-1].tool_calls.append(tc)
@@ -201,10 +269,10 @@ class CortexTracer:
         try:
             import httpx
             httpx.post(
-                "https://api.cortexops.ai/v1/traces",
+                f"{self.api_url}/v1/traces",
                 json=trace.model_dump(mode="json"),
-                headers={"Authorization": f"Bearer {self.api_key}"},
+                headers={"X-API-Key": self.api_key},
                 timeout=2.0,
             )
         except Exception:
-            pass  # non-blocking — tracing never breaks the agent
+            pass  # non-blocking — tracing never breaks the agent

{cortexops-0.1.0 → cortexops-0.2.0}/pyproject.toml

@@ -4,19 +4,19 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cortexops"
-version = "0.1.0"
+version = "0.2.0"
 description = "Reliability infrastructure for AI agents — evaluation, observability, and regression testing"
 readme = "README.md"
 license = { file = "LICENSE" }
 authors = [
-    { name = "Ashish", email = "ashishodu2023@gmail.com" },
+    { name = "Ashish", email = "ashish@getcortexops.com" },
 ]
 keywords = [
     "llm", "agents", "evaluation", "observability",
     "langgraph", "crewai", "autogen", "ai", "testing",
 ]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
@@ -33,7 +33,6 @@ requires-python = ">=3.10"
 dependencies = [
     "pydantic>=2.0",
     "pyyaml>=6.0",
-    "setuptools>=82.0.1",
 ]
 
 [project.optional-dependencies]
@@ -49,9 +48,9 @@ dev  = [
 ]
 
 [project.urls]
-Homepage      = "https://cortexops.ai"
+Homepage      = "https://getcortexops.com"
 Repository    = "https://github.com/ashishodu2023/cortexops"
-Documentation = "https://docs.cortexops.ai"
+Documentation = "https://docs.getcortexops.com"
 "Bug Tracker" = "https://github.com/ashishodu2023/cortexops/issues"
 Changelog     = "https://github.com/ashishodu2023/cortexops/releases"
 
@@ -71,7 +70,7 @@ include = [
 ]
 
 [tool.ruff]
-line-length = 100
+line-length = 121
 target-version = "py310"
 
 [tool.ruff.lint]
@@ -85,4 +84,4 @@ ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-testpaths = ["tests"]
+testpaths = ["tests"]