nabla-cli 0.1.0__py3-none-any.whl

nabla/__init__.py

@@ -0,0 +1,3 @@
+"""nabla — call-site harvester CLI (pre-Phase-0)."""
+
+__version__ = "0.1.0"

nabla/cli.py

@@ -0,0 +1,121 @@
+"""nabla CLI entrypoint — T7. Subcommands: scan, record, profile."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .contract import Example
+from .profile import build_profile, find_site
+from .recorder import RecordSource, ZeroTrafficError, record
+from .scanner import scan
+from .schema_resolver import resolve_schema
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="nabla", description="Call-site harvester")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_scan = sub.add_parser("scan", help="find + classify call sites")
+    p_scan.add_argument("repo", help="path to target repo")
+    p_scan.add_argument("--json", action="store_true", help="machine-readable output")
+
+    p_record = sub.add_parser("record", help="capture (prompt, completion) pairs")
+    p_record.add_argument("repo")
+    p_record.add_argument("--site", required=True, help="site symbol")
+    p_record.add_argument("--samples", help="JSONL sample inputs")
+    p_record.add_argument("--logs", help="JSONL of past request/response pairs")
+    p_record.add_argument("--proxy-cmd", help="command to run under the recording proxy")
+    p_record.add_argument("--out", default="recorded.jsonl")
+
+    p_profile = sub.add_parser("profile", help="assemble site_profile.json")
+    p_profile.add_argument("repo")
+    p_profile.add_argument("--site", required=True)
+    p_profile.add_argument("--examples", required=True, help="recorded JSONL from `nabla record`")
+    p_profile.add_argument("--out", default="site_profile.json")
+
+    args = parser.parse_args(argv)
+    return {"scan": _cmd_scan, "record": _cmd_record, "profile": _cmd_profile}[args.command](args)
+
+
+def _cmd_scan(args) -> int:
+    sites = scan(args.repo)
+    rows = []
+    for s in sites:
+        _, verifiability = resolve_schema(s, args.repo)
+        rows.append({**s.to_dict(), "verifiability": verifiability.value})
+    if args.json:
+        print(json.dumps(rows, indent=2))
+        return 0
+    print(f"{'SYMBOL':<22} {'FILE':<28} {'KIND':<6} {'VERIFIABILITY':<22} FLAGS")
+    for r in rows:
+        print(f"{r['symbol']:<22} {r['file']:<28} {r['kind']:<6} "
+              f"{r['verifiability']:<22} {','.join(r['flags']) or '-'}")
+    print(f"\n{len(rows)} call sites found.")
+    return 0
+
+
+def _cmd_record(args) -> int:
+    sites = scan(args.repo)
+    site = find_site(sites, args.site)
+    if args.samples:
+        source = RecordSource(kind="samples", samples_file=args.samples)
+    elif args.logs:
+        source = RecordSource(kind="logs", logs_file=args.logs)
+    elif args.proxy_cmd:
+        source = RecordSource(kind="proxy", proxy_command=args.proxy_cmd.split())
+    else:
+        print("nabla record: one of --samples/--logs/--proxy-cmd is required", file=sys.stderr)
+        return 2
+    try:
+        examples = record(site, source, args.repo)
+    except ZeroTrafficError as exc:
+        print(f"nabla record: {exc}", file=sys.stderr)
+        return 1
+    with open(args.out, "w", encoding="utf-8") as f:
+        for ex in examples:
+            f.write(json.dumps({**ex.to_dict(), "_source_kind": source.kind}) + "\n")
+    print(f"nabla record: {len(examples)} examples -> {args.out}")
+    return 0
+
+
+def _cmd_profile(args) -> int:
+    from .induce import induce
+    from .prompt_recon import reconstruct
+    from .schema_resolver import resolve_schema as _resolve
+
+    sites = scan(args.repo)
+    site = find_site(sites, args.site)
+
+    examples, recorded_from = [], "samples"
+    for line in Path(args.examples).read_text(encoding="utf-8").splitlines():
+        if not line.strip():
+            continue
+        d = json.loads(line)
+        recorded_from = d.get("_source_kind", recorded_from)
+        examples.append(Example(
+            input_slots=d.get("input_slots", {}), prompt=d["prompt"],
+            completion=d["completion"], tokens_in=d.get("tokens", {}).get("in", 0),
+            tokens_out=d.get("tokens", {}).get("out", 0),
+            latency_ms=d.get("latency_ms", 0)))
+
+    template = reconstruct(site, args.repo)
+    schema, _ = _resolve(site, args.repo)
+    goal = induce(template, schema, examples)
+
+    profile = build_profile(site, args.repo, examples, goal, recorded_from=recorded_from)
+    Path(args.out).write_text(json.dumps(profile, indent=2), encoding="utf-8")
+    supported = profile["classification"]["supported"]
+    print(f"nabla profile: wrote {args.out} "
+          f"(site {profile['site_id'][:12]}…, supported={supported}, "
+          f"{len(examples)} examples)")
+    if len(examples) < 20:
+        print(f"nabla profile: WARNING — {len(examples)} examples < 20; "
+              "Phase-0-ready profiles need >=20 (cli-plan §5.4)", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

nabla/contract.py

@@ -0,0 +1,132 @@
+"""Shared contract for all nabla CLI workstreams (T0).
+
+Every module codes against these types and nothing else from its siblings.
+RawSite is deliberately serializable and AST-free: downstream consumers
+(prompt reconstruction, schema resolution) re-parse the source file from
+`file` + `span` so workstreams never import each other's internals.
+
+Profile shape mirrors cli-plan.md §2 and nabla/schemas/site_profile.schema.json.
+The JSON schema is the source of truth; these dataclasses are the typed
+convenience layer. Bump PROFILE_VERSION on any breaking change to either.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+PROFILE_VERSION = "0.1"
+
+
+class Verifiability(str, Enum):
+    JSON_SCHEMA = "json_schema"          # response_format json_schema — free verifier
+    PYDANTIC_PARSE = "pydantic_parse"    # beta .parse(response_format=Model)
+    TOOL_CALL = "tool_call"              # tools=[...] function schemas
+    JSON_OBJECT_NO_SCHEMA = "json_object_no_schema"  # json_object mode, no schema
+    FREE_TEXT = "free_text"              # nothing checkable
+
+
+# Site flags. A site carrying any of UNSUPPORTED_FLAGS is supported=False.
+FLAG_STREAMING = "streaming"
+FLAG_ASYNC = "async"
+FLAG_MULTI_TURN = "multi_turn"
+FLAG_VISION = "vision"
+FLAG_N_GT_1 = "n_gt_1"
+FLAG_WRAPPER_INDIRECT = "wrapper_indirect"  # informational, does not unsupport
+
+UNSUPPORTED_FLAGS = {FLAG_STREAMING, FLAG_MULTI_TURN, FLAG_VISION, FLAG_N_GT_1}
+SUPPORTED_VERIFIABILITY = {Verifiability.JSON_SCHEMA, Verifiability.PYDANTIC_PARSE}
+
+
+@dataclass
+class RawSite:
+    """Output of the scanner (T1). Serializable; no AST nodes."""
+    file: str                  # repo-relative, forward slashes
+    span: tuple[int, int]      # 1-indexed inclusive line range of the call expr
+    symbol: str                # enclosing function/method name, bare, e.g. "enrich" or "run" (no parens — matches manifest)
+    kind: str                  # "create" | "parse"
+    flags: list[str] = field(default_factory=list)
+    # For a wrapper function's callers: the wrapper's own site info.
+    wrapper: RawSiteRef | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = {
+            "file": self.file,
+            "span": list(self.span),
+            "symbol": self.symbol,
+            "kind": self.kind,
+            "flags": sorted(self.flags),
+        }
+        if self.wrapper:
+            d["wrapper"] = {"file": self.wrapper.file, "span": list(self.wrapper.span),
+                            "symbol": self.wrapper.symbol}
+        return d
+
+
+@dataclass
+class RawSiteRef:
+    file: str
+    span: tuple[int, int]
+    symbol: str
+
+
+@dataclass
+class Slot:
+    name: str
+    observed_values: list[str] = field(default_factory=list)
+    inferred_type: str = "unknown"
+
+
+@dataclass
+class PromptTemplate:
+    """Output of prompt reconstruction (T2)."""
+    template: str                      # user-message template with {slot} markers
+    slots: list[Slot] = field(default_factory=list)
+    system_prompt: str | None = None
+    embedded_few_shot: list[dict[str, str]] = field(default_factory=list)
+    partial: bool = False              # True when fragments were unresolvable
+    # Unresolvable fragments appear in template as {UNRESOLVED_n} and are
+    # listed in slots with inferred_type="unresolved".
+
+
+@dataclass
+class Example:
+    """One recorded (prompt, completion) pair (T4)."""
+    input_slots: dict[str, str]
+    prompt: str
+    completion: str
+    tokens_in: int = 0
+    tokens_out: int = 0
+    latency_ms: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "input_slots": self.input_slots,
+            "prompt": self.prompt,
+            "completion": self.completion,
+            "tokens": {"in": self.tokens_in, "out": self.tokens_out},
+            "latency_ms": self.latency_ms,
+        }
+
+
+@dataclass
+class GoalBlock:
+    """Output of goal inference / difficulty induction (T6)."""
+    description: str
+    constraints: list[str] = field(default_factory=list)
+    difficulty_dimensions: list[dict[str, Any]] = field(default_factory=list)
+    # each dimension: {"name": str, "levels": [str], "evidence": str}
+
+
+def compute_site_id(template: str, schema: dict[str, Any] | None) -> str:
+    """Stable site identity: hash of normalized template + schema, not file:line."""
+    norm_template = " ".join(template.split())
+    norm_schema = json.dumps(schema, sort_keys=True, separators=(",", ":")) if schema else ""
+    return hashlib.sha256(f"{norm_template}\x00{norm_schema}".encode()).hexdigest()
+
+
+def is_supported(verifiability: Verifiability, flags: list[str]) -> bool:
+    return verifiability in SUPPORTED_VERIFIABILITY and not (set(flags) & UNSUPPORTED_FLAGS)

nabla/induce.py

@@ -0,0 +1,176 @@
+"""T6 — goal inference + difficulty-dimension induction (the one LLM step).
+
+Runs on Gemini's free tier via its OpenAI-compatible endpoint. The client is
+injectable so tests run fully offline; real responses are cached to disk so
+re-runs are deterministic and cheap.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from .contract import Example, GoalBlock, PromptTemplate
+
+# Defaults target Gemini's free tier; every piece is env-overridable so the
+# induction provider can be swapped without code changes (e.g. Groq when a
+# daily quota is exhausted): NABLA_INDUCE_BASE_URL, NABLA_INDUCE_API_KEY_ENV
+# (name of the env var holding the key, read from env or cli/.env — never
+# hardcode keys here), NABLA_INDUCE_MODEL. Re-align the default with Phase 0's
+# frontier choice when it lands.
+INDUCE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
+INDUCE_API_KEY_ENV = "GEMINI_API_KEY"
+DEFAULT_INDUCE_MODEL = "gemini-3.5-flash"
+
+_SYSTEM = """You analyze a harvested LLM call site and produce a task profile for a downstream
+training-data generator. You are given the call site's prompt template, its output
+JSON schema, and real recorded (prompt, completion) examples.
+
+Respond with ONLY a JSON object, no markdown fences, shaped exactly:
+{
+  "description": "<what this call site's task is, one or two sentences>",
+  "constraints": ["<implicit rule mined from the system prompt or examples>", ...],
+  "difficulty_dimensions": [
+    {"name": "<snake_case dimension>", "levels": ["<easy>", "...", "<hard>"],
+     "evidence": "<which examples/template features show this dimension varies>"},
+    ...
+  ]
+}
+
+Rules:
+- description states the task, not the mechanism ("extracts X from Y into schema Z").
+- constraints are rules the completion must follow that the schema alone does not
+  enforce (null-discipline, normalization, language, tone).
+- difficulty_dimensions are the axes along which REAL inputs to this site vary in
+  ways that plausibly make the task harder. 3-6 dimensions, each with 2+ ordered
+  levels (easiest first) and evidence citing the examples or template. Do not invent
+  dimensions the examples give no evidence for."""
+
+
+class InduceError(RuntimeError):
+    """The induction model returned something unusable."""
+
+
+def induce(template: PromptTemplate, schema: dict[str, Any] | None,
+           examples: list[Example], *, model: str | None = None,
+           cache_dir: str | None = None, client: Any = None) -> GoalBlock:
+    """Frontier-model pass producing goal description, constraints, and
+    difficulty dimensions with cited evidence.
+
+    `client` is any OpenAI-compatible client (injectable for tests); when
+    None, one is built for Gemini from GEMINI_API_KEY. Responses are cached
+    in `cache_dir` (default .nabla_cache next to cwd) keyed by inputs+model,
+    so repeat runs are offline and deterministic.
+    """
+    model = model or os.environ.get("NABLA_INDUCE_MODEL", DEFAULT_INDUCE_MODEL)
+    user_payload = _build_user_payload(template, schema, examples)
+
+    cache_path = None
+    if cache_dir is not False:  # allow cache_dir=False to disable entirely
+        cache_root = Path(cache_dir or ".nabla_cache")
+        key = hashlib.sha256(json.dumps([model, _SYSTEM, user_payload],
+                                        sort_keys=True).encode()).hexdigest()
+        cache_path = cache_root / f"induce-{key}.json"
+        if cache_path.exists():
+            return _parse_goal(cache_path.read_text(encoding="utf-8"))
+
+    if client is None:
+        client = _default_client()
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": _SYSTEM},
+            {"role": "user", "content": user_payload},
+        ],
+        temperature=0.2,
+    )
+    raw = response.choices[0].message.content or ""
+    goal = _parse_goal(raw)  # validate before caching — never cache garbage
+
+    if cache_path is not None:
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        cache_path.write_text(raw, encoding="utf-8")
+    return goal
+
+
+def _default_client():
+    from openai import OpenAI
+
+    base_url = os.environ.get("NABLA_INDUCE_BASE_URL", INDUCE_BASE_URL)
+    key_env = os.environ.get("NABLA_INDUCE_API_KEY_ENV", INDUCE_API_KEY_ENV)
+    api_key = os.environ.get(key_env) or _read_dotenv_key(key_env)
+    if not api_key:
+        raise InduceError(
+            f"{key_env} not set (env or cli/.env). "
+            "T6 induction needs an API key for its OpenAI-compatible provider."
+        )
+    return OpenAI(base_url=base_url, api_key=api_key)
+
+
+def _read_dotenv_key(name: str) -> str | None:
+    # Minimal .env reader: look next to this package's parent (cli/.env), then cwd.
+    for candidate in (Path(__file__).resolve().parents[1] / ".env", Path(".env")):
+        try:
+            for line in candidate.read_text(encoding="utf-8").splitlines():
+                line = line.strip()
+                if line.startswith(f"{name}=") and not line.startswith("#"):
+                    return line.split("=", 1)[1].strip().strip('"').strip("'")
+        except OSError:
+            continue
+    return None
+
+
+def _build_user_payload(template: PromptTemplate, schema: dict[str, Any] | None,
+                        examples: list[Example]) -> str:
+    parts = ["## Prompt template\n" + template.template]
+    if template.system_prompt:
+        parts.append("## System prompt\n" + template.system_prompt)
+    parts.append("## Slots\n" + json.dumps(
+        [{"name": s.name, "inferred_type": s.inferred_type} for s in template.slots]))
+    parts.append("## Output schema\n" +
+                 (json.dumps(schema, indent=2) if schema else "(none)"))
+    shown = examples[:20]
+    ex_lines = [
+        f"### Example {i + 1}\nINPUT SLOTS: {json.dumps(e.input_slots)}\n"
+        f"PROMPT: {e.prompt}\nCOMPLETION: {e.completion}"
+        for i, e in enumerate(shown)
+    ]
+    parts.append(f"## Recorded examples ({len(shown)} of {len(examples)})\n" +
+                 "\n".join(ex_lines))
+    return "\n\n".join(parts)
+
+
+def _parse_goal(raw: str) -> GoalBlock:
+    text = raw.strip()
+    fence = re.match(r"^```(?:json)?\s*(.*?)\s*```$", text, re.DOTALL)
+    if fence:
+        text = fence.group(1)
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise InduceError(f"induction model did not return JSON: {exc}\n{raw[:500]}") from exc
+
+    description = data.get("description")
+    constraints = data.get("constraints", [])
+    dims = data.get("difficulty_dimensions", [])
+    if not isinstance(description, str) or not description.strip():
+        raise InduceError("induction result missing 'description'")
+    if not isinstance(constraints, list) or not all(isinstance(c, str) for c in constraints):
+        raise InduceError("induction result 'constraints' must be a list of strings")
+    clean_dims: list[dict[str, Any]] = []
+    for d in dims if isinstance(dims, list) else []:
+        if (isinstance(d, dict) and isinstance(d.get("name"), str)
+                and isinstance(d.get("levels"), list) and len(d["levels"]) >= 2
+                and all(isinstance(level, str) for level in d["levels"])
+                and isinstance(d.get("evidence"), str)):
+            clean_dims.append({"name": d["name"], "levels": d["levels"],
+                               "evidence": d["evidence"]})
+    if not clean_dims:
+        raise InduceError("induction result has no well-formed difficulty_dimensions")
+    return GoalBlock(description=description.strip(), constraints=constraints,
+                     difficulty_dimensions=clean_dims)

nabla/profile.py

@@ -0,0 +1,138 @@
+"""T7 — site profile assembly: compose T1–T6 outputs into a validated
+site_profile.json (the Phase 0 handoff artifact).
+
+Also owns sampling-parameter extraction (model/temperature/max_tokens/...),
+which is plain AST kwarg reading and belongs to assembly rather than any
+single workstream.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from .contract import (
+    PROFILE_VERSION,
+    Example,
+    GoalBlock,
+    PromptTemplate,
+    RawSite,
+    Verifiability,
+    compute_site_id,
+    is_supported,
+)
+from .prompt_recon import reconstruct
+from .recorder import baseline_stats
+from .schema_resolver import resolve_schema
+
+# Packaged with the module (nabla/schemas/) so installed wheels find it too.
+_SCHEMA_RESOURCE = resources.files("nabla").joinpath("schemas/site_profile.schema.json")
+
+
+def extract_sampling(site: RawSite, repo_path: str) -> dict[str, Any]:
+    """Read literal sampling kwargs off the SDK call (wrapper's call for
+    wrapper-indirect sites). Non-literal values are reported as null."""
+    ref = site.wrapper or site
+    path = Path(repo_path) / ref.file
+    sampling: dict[str, Any] = {"model": "unknown", "temperature": None,
+                                "max_tokens": None, "top_p": None, "stop": None}
+    try:
+        tree = ast.parse(path.read_text(encoding="utf-8"))
+    except (OSError, SyntaxError):
+        return sampling
+    for node in ast.walk(tree):
+        if (isinstance(node, ast.Call)
+                and node.lineno == ref.span[0]
+                and getattr(node, "end_lineno", node.lineno) == ref.span[1]):
+            for kw in node.keywords:
+                if kw.arg in sampling and isinstance(kw.value, ast.Constant):
+                    sampling[kw.arg] = kw.value.value
+            break
+    if sampling["model"] is None or sampling["model"] == "unknown":
+        sampling["model"] = "unknown"
+    return sampling
+
+
+def build_profile(site: RawSite, repo_path: str, examples: list[Example],
+                  goal: GoalBlock, recorded_from: str = "samples") -> dict[str, Any]:
+    """Assemble the complete profile dict. Raises jsonschema.ValidationError
+    if the result does not conform to the frozen schema."""
+    template: PromptTemplate = reconstruct(site, repo_path)
+    schema, verifiability = resolve_schema(site, repo_path)
+    sampling = extract_sampling(site, repo_path)
+
+    # Fold observed slot values from recorded examples into the template slots.
+    observed: dict[str, list[str]] = {}
+    for ex in examples:
+        for name, value in ex.input_slots.items():
+            vals = observed.setdefault(name, [])
+            if str(value) not in vals and len(vals) < 5:
+                vals.append(str(value))
+
+    profile = {
+        "profile_version": PROFILE_VERSION,
+        "site_id": compute_site_id(template.template, schema),
+        "source": {"file": site.file, "span": list(site.span), "symbol": site.symbol},
+        "classification": {
+            "supported": is_supported(verifiability, site.flags),
+            "verifiability": verifiability.value,
+            "flags": sorted(site.flags),
+        },
+        "prompt": {
+            "template": template.template,
+            "partial": template.partial,
+            "slots": [
+                {"name": s.name,
+                 "observed_values": observed.get(s.name, s.observed_values),
+                 "inferred_type": s.inferred_type}
+                for s in template.slots
+            ],
+            "system_prompt": template.system_prompt,
+            "embedded_few_shot": template.embedded_few_shot,
+        },
+        "output_schema": schema,
+        "sampling": sampling,
+        "examples": [e.to_dict() for e in examples],
+        "goal": {
+            "description": goal.description,
+            "constraints": goal.constraints,
+            "difficulty_dimensions": goal.difficulty_dimensions,
+        },
+        "baseline": _baseline_block(examples, sampling["model"]),
+        "provenance": {"recorded_from": recorded_from if examples else "none"},
+    }
+    validate_profile(profile)
+    return profile
+
+
+def _baseline_block(examples: list[Example], model: str) -> dict[str, Any]:
+    if not examples:
+        return {"cost_per_call_usd": 0.0, "p50_latency_ms": 0, "calls_observed": 0}
+    stats = baseline_stats(examples, model)
+    return {
+        "cost_per_call_usd": float(stats.get("cost_per_call_usd", 0.0)),
+        "p50_latency_ms": int(stats.get("p50_latency_ms", 0)),
+        "calls_observed": int(stats.get("calls_observed", len(examples))),
+    }
+
+
+def validate_profile(profile: dict[str, Any]) -> None:
+    import jsonschema
+
+    schema = json.loads(_SCHEMA_RESOURCE.read_text(encoding="utf-8"))
+    jsonschema.validate(profile, schema)
+
+
+def find_site(sites: list[RawSite], selector: str) -> RawSite:
+    """Match a site by symbol or site-id-ish prefix; error lists candidates."""
+    matches = [s for s in sites if s.symbol == selector]
+    if len(matches) == 1:
+        return matches[0]
+    if not matches:
+        available = ", ".join(sorted({s.symbol for s in sites}))
+        raise SystemExit(f"nabla: no site named {selector!r}. Sites found: {available}")
+    files = ", ".join(s.file for s in matches)
+    raise SystemExit(f"nabla: {selector!r} is ambiguous across: {files} — split not yet supported")