nabla-cli 0.1.0__py3-none-any.whl

nabla/schema_resolver.py

@@ -0,0 +1,309 @@
+"""T3 — output-schema resolution + verifiability classing. See cli-plan.md T3.
+
+Re-parses source from raw_site.file/span; never imports scanner or
+prompt_recon. The only dynamic import this module performs is of the
+target repo's own module, and only to resolve a Pydantic model's
+`.model_json_schema()` for the `.parse(response_format=Model)` shape —
+that import is fully guarded and degrades to (None, PYDANTIC_PARSE).
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib
+import sys
+from pathlib import Path
+from typing import Any, Callable
+
+from .contract import RawSite, RawSiteRef, Verifiability
+
+
+def resolve_schema(raw_site: RawSite, repo_path: str) -> tuple[dict[str, Any] | None, Verifiability]:
+    """Normalize the site's output declaration to one JSON Schema.
+
+    Handles: response_format json_schema (extract + inline $defs/$refs),
+    Pydantic model in .parse() (guarded dynamic import -> model_json_schema()),
+    tools function schemas, json_object (None, JSON_OBJECT_NO_SCHEMA),
+    nothing (None, FREE_TEXT), and wrapper-indirect sites (resolve the
+    caller's own schema argument back to a module constant).
+    """
+    repo = Path(repo_path)
+
+    if raw_site.wrapper is not None:
+        return _resolve_wrapper_site(raw_site, repo)
+
+    tree = _parse_file(repo, raw_site.file)
+    if tree is None:
+        return None, Verifiability.FREE_TEXT
+
+    call_node = _find_call_by_span(tree, raw_site.span, predicate=_is_openai_call)
+    if call_node is None:
+        return None, Verifiability.FREE_TEXT
+
+    module_consts = _module_level_assignments(tree)
+
+    tools_node = _get_kw(call_node, "tools")
+    if tools_node is not None:
+        return _resolve_tools(tools_node, module_consts)
+
+    rf_node = _get_kw(call_node, "response_format")
+    if rf_node is not None:
+        if isinstance(rf_node, ast.Dict):
+            try:
+                value = _eval_node(rf_node, module_consts)
+            except ValueError:
+                value = None
+            if isinstance(value, dict):
+                rf_type = value.get("type")
+                if rf_type == "json_schema":
+                    schema = (value.get("json_schema") or {}).get("schema")
+                    if isinstance(schema, dict):
+                        return _normalize_schema(schema), Verifiability.JSON_SCHEMA
+                    return None, Verifiability.JSON_SCHEMA
+                if rf_type == "json_object":
+                    return None, Verifiability.JSON_OBJECT_NO_SCHEMA
+        else:
+            # A Name/Attribute referencing a Pydantic model class, e.g.
+            # response_format=Order.
+            schema = _resolve_pydantic_model(rf_node, raw_site, repo)
+            return schema, Verifiability.PYDANTIC_PARSE
+
+    return None, Verifiability.FREE_TEXT
+
+
+# --------------------------------------------------------------------------
+# Wrapper-indirect sites: schema arrives as an argument to the wrapper call.
+# --------------------------------------------------------------------------
+
+def _resolve_wrapper_site(raw_site: RawSite, repo: Path) -> tuple[dict[str, Any] | None, Verifiability]:
+    tree = _parse_file(repo, raw_site.file)
+    if tree is None:
+        return None, Verifiability.FREE_TEXT
+
+    call_node = _find_call_by_span(tree, raw_site.span, predicate=None)
+    if call_node is None:
+        return None, Verifiability.FREE_TEXT
+
+    module_consts = _module_level_assignments(tree)
+
+    schema_node = _get_kw(call_node, "schema")
+    if schema_node is None and call_node.args:
+        schema_node = call_node.args[-1]
+    if schema_node is None:
+        return None, Verifiability.JSON_SCHEMA
+
+    try:
+        value = _eval_node(schema_node, module_consts)
+    except ValueError:
+        return None, Verifiability.JSON_SCHEMA
+
+    if isinstance(value, dict):
+        return _normalize_schema(value), Verifiability.JSON_SCHEMA
+    return None, Verifiability.JSON_SCHEMA
+
+
+# --------------------------------------------------------------------------
+# AST helpers
+# --------------------------------------------------------------------------
+
+def _parse_file(repo: Path, rel_file: str) -> ast.Module | None:
+    path = repo / rel_file
+    try:
+        source = path.read_text(encoding="utf-8")
+    except OSError:
+        return None
+    try:
+        return ast.parse(source, filename=str(path))
+    except SyntaxError:
+        return None
+
+
+def _is_openai_call(node: ast.Call) -> bool:
+    return isinstance(node.func, ast.Attribute) and node.func.attr in ("create", "parse")
+
+
+def _find_call_by_span(
+    tree: ast.AST, span: tuple[int, int], predicate: Callable[[ast.Call], bool] | None
+) -> ast.Call | None:
+    start, end = span
+    matches = [
+        node
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Call)
+        and node.lineno == start
+        and getattr(node, "end_lineno", node.lineno) == end
+    ]
+    if predicate is not None:
+        filtered = [n for n in matches if predicate(n)]
+        if filtered:
+            matches = filtered
+    return matches[0] if matches else None
+
+
+def _get_kw(call: ast.Call, name: str) -> ast.AST | None:
+    for kw in call.keywords:
+        if kw.arg == name:
+            return kw.value
+    return None
+
+
+def _module_level_assignments(tree: ast.Module) -> dict[str, ast.AST]:
+    consts: dict[str, ast.AST] = {}
+    for node in tree.body:
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name):
+                    consts[target.id] = node.value
+        elif isinstance(node, ast.AnnAssign) and node.value is not None and isinstance(node.target, ast.Name):
+            consts[node.target.id] = node.value
+    return consts
+
+
+def _eval_node(node: ast.AST, consts: dict[str, ast.AST], _stack: frozenset[str] = frozenset()) -> Any:
+    """Restricted static evaluator: literals, dict/list/tuple literals, and
+    Name references resolved against module-level constants. Never executes
+    arbitrary code."""
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.Dict):
+        result: dict[Any, Any] = {}
+        for k, v in zip(node.keys, node.values):
+            if k is None:  # dict unpacking (**x) — unsupported, bail
+                raise ValueError("dict unpacking not supported")
+            result[_eval_node(k, consts, _stack)] = _eval_node(v, consts, _stack)
+        return result
+    if isinstance(node, (ast.List, ast.Tuple)):
+        return [_eval_node(elt, consts, _stack) for elt in node.elts]
+    if isinstance(node, ast.Name):
+        if node.id in _stack:
+            raise ValueError(f"circular reference resolving {node.id!r}")
+        if node.id not in consts:
+            raise ValueError(f"unresolved name {node.id!r}")
+        return _eval_node(consts[node.id], consts, _stack | {node.id})
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
+        return -_eval_node(node.operand, consts, _stack)
+    raise ValueError(f"cannot statically evaluate {ast.dump(node)}")
+
+
+# --------------------------------------------------------------------------
+# tools=[...] resolution
+# --------------------------------------------------------------------------
+
+def _resolve_tools(
+    tools_node: ast.AST, consts: dict[str, ast.AST]
+) -> tuple[dict[str, Any] | None, Verifiability]:
+    try:
+        tools = _eval_node(tools_node, consts)
+    except ValueError:
+        return None, Verifiability.TOOL_CALL
+    if not isinstance(tools, list) or not tools:
+        return None, Verifiability.TOOL_CALL
+
+    if len(tools) == 1:
+        params = _tool_parameters(tools[0])
+        if isinstance(params, dict):
+            return _normalize_schema(params), Verifiability.TOOL_CALL
+        return None, Verifiability.TOOL_CALL
+
+    schemas: dict[str, Any] = {}
+    for tool in tools:
+        fn = tool.get("function", {}) if isinstance(tool, dict) else {}
+        name = fn.get("name")
+        params = fn.get("parameters")
+        if name and isinstance(params, dict):
+            schemas[name] = _normalize_schema(params)
+    return (schemas or None), Verifiability.TOOL_CALL
+
+
+def _tool_parameters(tool: Any) -> dict[str, Any] | None:
+    if not isinstance(tool, dict):
+        return None
+    fn = tool.get("function", {})
+    if not isinstance(fn, dict):
+        return None
+    params = fn.get("parameters")
+    return params if isinstance(params, dict) else None
+
+
+# --------------------------------------------------------------------------
+# Pydantic .parse(response_format=Model) resolution — guarded dynamic import
+# --------------------------------------------------------------------------
+
+def _resolve_pydantic_model(node: ast.AST, raw_site: RawSite, repo: Path) -> dict[str, Any] | None:
+    class_name: str | None = None
+    if isinstance(node, ast.Name):
+        class_name = node.id
+    elif isinstance(node, ast.Attribute):
+        class_name = node.attr
+
+    if class_name is None:
+        return None
+
+    module_dotted = _dotted_module(raw_site.file)
+    if module_dotted is None:
+        return None
+
+    repo_str = str(repo.resolve())
+    added = False
+    try:
+        if repo_str not in sys.path:
+            sys.path.insert(0, repo_str)
+            added = True
+        try:
+            module = importlib.import_module(module_dotted)
+            model_cls = getattr(module, class_name)
+            schema = model_cls.model_json_schema()
+        finally:
+            if added:
+                try:
+                    sys.path.remove(repo_str)
+                except ValueError:
+                    pass
+    except Exception:
+        return None
+
+    if not isinstance(schema, dict):
+        return None
+    return _normalize_schema(schema)
+
+
+def _dotted_module(rel_file: str) -> str | None:
+    p = Path(rel_file)
+    if p.suffix != ".py":
+        return None
+    parts = list(p.with_suffix("").parts)
+    if parts and parts[-1] == "__init__":
+        parts = parts[:-1]
+    return ".".join(parts) if parts else None
+
+
+# --------------------------------------------------------------------------
+# Schema normalization: inline $defs/$refs so the schema is self-contained.
+# --------------------------------------------------------------------------
+
+def _normalize_schema(schema: dict[str, Any]) -> dict[str, Any]:
+    defs = schema.get("$defs", {}) if isinstance(schema.get("$defs"), dict) else {}
+
+    def _inline(node: Any, seen: frozenset[str]) -> Any:
+        if isinstance(node, dict):
+            if "$ref" in node and isinstance(node["$ref"], str) and node["$ref"].startswith("#/$defs/"):
+                ref_name = node["$ref"].split("/")[-1]
+                if ref_name in seen or ref_name not in defs:
+                    # Cycle or dangling ref: leave as-is rather than recurse forever.
+                    return {k: v for k, v in node.items()}
+                resolved = _inline(defs[ref_name], seen | {ref_name})
+                merged = dict(resolved) if isinstance(resolved, dict) else {}
+                for k, v in node.items():
+                    if k != "$ref":
+                        merged[k] = _inline(v, seen)
+                return merged
+            return {k: _inline(v, seen) for k, v in node.items() if k != "$defs"}
+        if isinstance(node, list):
+            return [_inline(item, seen) for item in node]
+        return node
+
+    result = _inline(schema, frozenset())
+    if isinstance(result, dict):
+        result.pop("$defs", None)
+        return result
+    return schema

nabla/schemas/site_profile.schema.json

@@ -0,0 +1,136 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "nabla/site_profile/0.1",
+  "title": "Nabla site profile — the Phase 0 handoff artifact",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["profile_version", "site_id", "source", "classification", "prompt",
+               "output_schema", "sampling", "examples", "goal", "baseline", "provenance"],
+  "properties": {
+    "profile_version": { "const": "0.1" },
+    "site_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["file", "span", "symbol"],
+      "properties": {
+        "file": { "type": "string" },
+        "span": { "type": "array", "items": { "type": "integer", "minimum": 1 }, "minItems": 2, "maxItems": 2 },
+        "symbol": { "type": "string" }
+      }
+    },
+    "classification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["supported", "verifiability", "flags"],
+      "properties": {
+        "supported": { "type": "boolean" },
+        "verifiability": { "enum": ["json_schema", "pydantic_parse", "tool_call", "json_object_no_schema", "free_text"] },
+        "flags": {
+          "type": "array",
+          "items": { "enum": ["streaming", "async", "multi_turn", "vision", "n_gt_1", "wrapper_indirect"] }
+        }
+      }
+    },
+    "prompt": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["template", "slots"],
+      "properties": {
+        "template": { "type": "string" },
+        "partial": { "type": "boolean" },
+        "slots": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["name"],
+            "properties": {
+              "name": { "type": "string" },
+              "observed_values": { "type": "array", "items": { "type": "string" } },
+              "inferred_type": { "type": "string" }
+            }
+          }
+        },
+        "system_prompt": { "type": ["string", "null"] },
+        "embedded_few_shot": {
+          "type": "array",
+          "items": { "type": "object" }
+        }
+      }
+    },
+    "output_schema": { "type": ["object", "null"] },
+    "sampling": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["model"],
+      "properties": {
+        "model": { "type": "string" },
+        "temperature": { "type": ["number", "null"] },
+        "max_tokens": { "type": ["integer", "null"] },
+        "top_p": { "type": ["number", "null"] },
+        "stop": { "type": ["array", "string", "null"] }
+      }
+    },
+    "examples": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["input_slots", "prompt", "completion", "tokens", "latency_ms"],
+        "properties": {
+          "input_slots": { "type": "object" },
+          "prompt": { "type": "string" },
+          "completion": { "type": "string" },
+          "tokens": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["in", "out"],
+            "properties": { "in": { "type": "integer" }, "out": { "type": "integer" } }
+          },
+          "latency_ms": { "type": "integer" }
+        }
+      }
+    },
+    "goal": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["description", "constraints", "difficulty_dimensions"],
+      "properties": {
+        "description": { "type": "string", "minLength": 1 },
+        "constraints": { "type": "array", "items": { "type": "string" } },
+        "difficulty_dimensions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["name", "levels", "evidence"],
+            "properties": {
+              "name": { "type": "string" },
+              "levels": { "type": "array", "items": { "type": "string" }, "minItems": 2 },
+              "evidence": { "type": "string" }
+            }
+          }
+        }
+      }
+    },
+    "baseline": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["cost_per_call_usd", "p50_latency_ms", "calls_observed"],
+      "properties": {
+        "cost_per_call_usd": { "type": "number", "minimum": 0 },
+        "p50_latency_ms": { "type": "integer", "minimum": 0 },
+        "calls_observed": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "provenance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["recorded_from"],
+      "properties": {
+        "recorded_from": { "enum": ["proxy", "samples", "logs", "none"] }
+      }
+    }
+  }
+}

nabla_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,82 @@
+Metadata-Version: 2.4
+Name: nabla-cli
+Version: 0.1.0
+Summary: Call-site harvester: scan OpenAI call sites, record real traffic, emit site profiles
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: jsonschema
+Requires-Dist: pydantic>=2
+Requires-Dist: openai>=1.40
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+
+# nabla — OpenAI call-site harvester
+
+`nabla` finds the OpenAI call sites in a Python repo, records real
+(prompt, completion) traffic for one of them, and emits a single
+self-contained **site profile** (`site_profile.json`) describing that
+call site: prompt template + slots, resolved output JSON Schema,
+sampling parameters, recorded examples, an inferred goal with difficulty
+dimensions, and a cost/latency baseline.
+
+The profile is the handoff artifact for training a small, cheap
+replacement model for that one call site — everything downstream codes
+against the profile, never against your repo.
+
+## Install
+
+```bash
+pip install nabla-cli
+# or, isolated:
+pipx install nabla-cli
+```
+
+Requires Python ≥ 3.11.
+
+## Quickstart
+
+```bash
+# 1. Find call sites (pure static analysis — no network, no API key)
+nabla scan path/to/repo
+
+# 2. Record real traffic for one site, driven by sample inputs
+#    (JSONL lines of {"input_slots": {...}} matching the site's function kwargs)
+nabla record path/to/repo --site my_function --samples samples.jsonl --out recorded.jsonl
+
+# 3. Assemble + validate the profile (one LLM call for goal inference)
+nabla profile path/to/repo --site my_function --examples recorded.jsonl --out site_profile.json
+```
+
+## What gets detected
+
+`scan` classifies every `chat.completions.create` / `beta...parse` call
+by verifiability: `json_schema` and `pydantic_parse` sites are supported
+harvest targets; `tool_call`, `json_object_no_schema`, and `free_text`
+sites are detected and reported but not harvested. Streaming,
+multi-turn, vision, and `n>1` sites are flagged unsupported. Generic
+wrapper functions are expanded to their callers; functions that own
+their prompt template are kept as the site.
+
+## Environment variables
+
+| Variable | Purpose |
+|---|---|
+| `OPENAI_API_KEY` | Default upstream for `record` (auth passthrough). |
+| `NABLA_UPSTREAM_BASE_URL` | Record through any OpenAI-compatible stand-in oracle instead. |
+| `NABLA_UPSTREAM_API_KEY` | Key for the stand-in oracle. |
+| `NABLA_UPSTREAM_MODEL` | Rewrite the model for the stand-in only (the target repo's own model still lands in the profile). |
+| `NABLA_RECORD_DELAY_MS` | Pacing between samples for rate-limited upstreams (default 0). |
+| `NABLA_SAMPLE_TIMEOUT` | Per-sample subprocess timeout in seconds (default 300). |
+| `GEMINI_API_KEY` | Key for the default goal-inference provider (Gemini free tier). |
+| `NABLA_INDUCE_BASE_URL` / `NABLA_INDUCE_API_KEY_ENV` / `NABLA_INDUCE_MODEL` | Swap the goal-inference provider/model. |
+
+## Notes
+
+- `record` runs the site's enclosing function in a subprocess per sample,
+  with `OPENAI_BASE_URL` pointed at a local recording proxy. If the
+  target's tests mock the SDK, the proxy path reports zero traffic
+  explicitly instead of writing an empty profile.
+- Recorded prompts/completions are real data from your repo and leave
+  the machine only via the goal-inference call. Review before sharing
+  profiles.

nabla_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+nabla/__init__.py,sha256=9PXtjQRdKH4vIYDFe6pVoZDFWt_b826GKWZcZhF463U,78
+nabla/cli.py,sha256=oD-brSXwHKlSGoZaMrzFPYUVxjEC8eI6Rr2jrALyzYY,4829
+nabla/contract.py,sha256=2BV0aF1PRG607uVX93snIkGUjUG_VBPWzObHWV7Gt6Q,4785
+nabla/induce.py,sha256=OHrxNPeMmOjR_FIEu9_f-FT2KSPMSLb4qHkzXAx8GMY,7692
+nabla/profile.py,sha256=QLoRqGUHdLyOTNPMCGOo7X8UtVCW1LPPDwC8VgrQg2U,5379
+nabla/prompt_recon.py,sha256=ikFsUoyImJw3K8OVdX9VkKfs5qRE85pEZ9io8pKgsuw,20442
+nabla/recorder.py,sha256=Y9zy9MnwYRUhghrIl5UdYavITmL8d6BGivwyEurhXAw,24746
+nabla/scanner.py,sha256=kqk-3UePafLxKdMWYB2VeYZMUrog0Vexh-OwHd8Zw-Y,16956
+nabla/schema_resolver.py,sha256=6IBHi1sTm8oa-4pYtEPiAR9PDWYW_muCJqth_Gy7ynI,11509
+nabla/schemas/site_profile.schema.json,sha256=1nhZty8GcJePhFFR45JzzbCX9ZEJ33ZPwBSWt391Q6c,4727
+nabla_cli-0.1.0.dist-info/METADATA,sha256=UX1jYnRBOboWfHtdNj8Z2jQuQRfFmiGos30ve-fEw-8,3396
+nabla_cli-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+nabla_cli-0.1.0.dist-info/entry_points.txt,sha256=I48El-2J50j-3S97L11RWrgdfOLhwRAv_NWSDD1h2IA,41
+nabla_cli-0.1.0.dist-info/top_level.txt,sha256=dhw4W9HA8wn3H0DYFF4Pb9S2DPbnqFF1cS7DdHcIdhs,6
+nabla_cli-0.1.0.dist-info/RECORD,,

nabla_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nabla_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nabla = nabla.cli:main

nabla_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+nabla