atomadic-forge 0.3.2__py3-none-any.whl

atomadic_forge/a1_at_functions/synergy_surface_extract.py

@@ -0,0 +1,163 @@
+"""Tier a1 — pure feature-surface extractor for the Synergy Scan.
+
+Walks ``commands/`` modules + the unified CLI to build a
+:class:`FeatureSurfaceCard` per CLI verb.  The card captures:
+
+* CLI option names
+* file-pattern args (anything ending in ``.json`` / ``.yaml`` / ``.md`` …)
+* product mentions in help text (``--json-out``, ``--save``, ``MANIFEST``)
+* schema strings (``atomadic-forge.<x>/v1``) referenced in source
+* a phase hint (``recon`` / ``ingest`` / ``materialize`` / ``emit`` / …)
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+from ..a0_qk_constants.synergy_types import FeatureSurfaceCard
+
+_PHASE_VOCAB = {
+    "recon": {"recon", "scout", "inventory", "discover"},
+    "ingest": {"ingest", "scan", "harvest", "extract"},
+    "plan": {"plan", "policy", "blueprint"},
+    "materialize": {"materialize", "rebuild", "synth", "synthesise", "synthesize",
+                    "wrap", "generate"},
+    "certify": {"certify", "verify", "audit", "smoke", "lint"},
+    "emit": {"emit", "save", "publish", "report", "manifest", "json-out"},
+    "register": {"register", "wire", "sync", "install"},
+}
+
+_FILE_RE = re.compile(r"\b([\w\-./]+\.(?:json|yaml|yml|md|toml|txt))\b")
+_SCHEMA_RE = re.compile(r"atomadic-forge\.[a-z0-9._\-]+/v\d+")
+
+
+def _docstring(node: ast.AST) -> str:
+    return (ast.get_docstring(node) or "").strip()
+
+
+def _split_words(text: str) -> set[str]:
+    return {w.lower() for w in re.findall(r"[A-Za-z][A-Za-z0-9_\-]+", text or "")
+            if len(w) >= 3}
+
+
+def _phase_hint(words: set[str], help_text: str, name: str) -> str:
+    haystack = words | {name.lower()}
+    haystack |= _split_words(help_text)
+    best, score = "misc", 0
+    for phase, terms in _PHASE_VOCAB.items():
+        hits = len(terms & haystack)
+        if hits > score:
+            best, score = phase, hits
+    return best
+
+
+def _option_args(fn: ast.FunctionDef) -> tuple[list[str], list[str]]:
+    """Return (canonical option names, file-pattern hints)."""
+    opts: list[str] = []
+    files: list[str] = []
+    for arg in fn.args.args:
+        if arg.arg in ("self", "cls", "ctx"):
+            continue
+        opts.append(arg.arg.replace("_", "-"))
+        ann = ast.unparse(arg.annotation) if arg.annotation else ""
+        if "Path" in ann or "json" in ann.lower():
+            files.append(arg.arg)
+    return opts, files
+
+
+def _typer_decorator_name(dec: ast.AST) -> str:
+    if isinstance(dec, ast.Call):
+        return _typer_decorator_name(dec.func)
+    if isinstance(dec, ast.Attribute):
+        return dec.attr
+    if isinstance(dec, ast.Name):
+        return dec.id
+    return ""
+
+
+def _is_typer_command(fn: ast.FunctionDef) -> bool:
+    for d in fn.decorator_list:
+        if _typer_decorator_name(d) == "command":
+            return True
+    return False
+
+
+def _harvest_module(py_file: Path, module: str) -> list[FeatureSurfaceCard]:
+    try:
+        text = py_file.read_text(encoding="utf-8")
+        tree = ast.parse(text)
+    except (SyntaxError, OSError):
+        return []
+    schemas = sorted(set(_SCHEMA_RE.findall(text)))
+    cards: list[FeatureSurfaceCard] = []
+    module_help = _docstring(tree).split("\n", 1)[0].strip()
+    # If module exposes its own ``app: typer.Typer`` (whole sub-CLI), make it
+    # one feature whose 'verbs' are the @app.command()-decorated functions.
+    has_typer_app = any(
+        isinstance(node, ast.Assign)
+        and any(isinstance(t, ast.Name) and t.id == "app" for t in node.targets)
+        for node in tree.body
+    )
+    inputs: list[str] = []
+    input_files: list[str] = []
+    outputs: list[str] = []
+    output_files: list[str] = []
+    vocab: set[str] = _split_words(module_help)
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef):
+            if not _is_typer_command(node) and node.name != "register":
+                continue
+            opts, files = _option_args(node)
+            inputs.extend(opts)
+            input_files.extend(files)
+            doc = _docstring(node)
+            vocab |= _split_words(doc)
+            for hit in _FILE_RE.findall(doc):
+                if any(t in opts for t in ("save", "out", "json")):
+                    output_files.append(hit)
+                else:
+                    input_files.append(hit)
+            for opt in opts:
+                if "out" in opt or "save" in opt or "report" in opt or "json" in opt:
+                    outputs.append(opt)
+    if has_typer_app or any(node for node in tree.body
+                            if isinstance(node, ast.FunctionDef)
+                            and _is_typer_command(node)):
+        name = py_file.stem
+        cards.append(FeatureSurfaceCard(
+            name=name.replace("_", "-").rstrip("-cli"),
+            module=module,
+            help_text=module_help,
+            inputs=sorted(set(inputs)),
+            input_files=sorted(set(input_files)),
+            outputs=sorted(set(outputs)),
+            output_files=sorted(set(output_files)),
+            schemas=schemas,
+            vocabulary=sorted(vocab),
+            phase_hint=_phase_hint(vocab, module_help, name),
+        ))
+    return cards
+
+
+def harvest_feature_surfaces(
+    package_root: Path,
+    package: str = "atomadic_forge",
+) -> list[FeatureSurfaceCard]:
+    """Walk ``<package_root>/<package>/commands/`` and harvest one card per file."""
+    base = package_root / package / "commands"
+    if not base.exists():
+        return []
+    out: list[FeatureSurfaceCard] = []
+    for py in sorted(base.rglob("*.py")):
+        if py.name.startswith("_"):
+            continue
+        rel = py.relative_to(package_root).with_suffix("")
+        module = ".".join(rel.parts)
+        out.extend(_harvest_module(py, module))
+    # Also include the unified CLI's hand-wired top-level commands.
+    unified = package_root / package / "a4_sy_orchestration" / "unified_cli.py"
+    if unified.exists():
+        out.extend(_harvest_module(unified, f"{package}.a4_sy_orchestration.unified_cli"))
+    return out

atomadic_forge/a1_at_functions/test_runner.py

@@ -0,0 +1,196 @@
+"""Tier a1 — pure pytest runner for the generated package.
+
+After every iterate turn, Forge runs ``pytest tests/`` against the
+emitted code and credits the certify score by the pass-ratio.  This is
+the *behavioral* check — wire/import says the package loads, but only a
+running test says it actually does what was asked.
+
+The runner is subprocess-based so it isolates from the parent process
+state.  Returns a structured report with passed/failed/error counts plus
+the trimmed pytest stdout/stderr for LLM feedback.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import TypedDict
+
+
+class TestRunReport(TypedDict):
+    schema_version: str        # "atomadic-forge.test_run/v1"
+    tests_dir: str
+    ran: bool                  # were any tests collected?
+    passed: int
+    failed: int
+    errors: int
+    skipped: int
+    total: int
+    pass_ratio: float          # 0.0 .. 1.0
+    duration_ms: int
+    pytest_summary: str        # last ~30 lines of stdout
+    failure_excerpts: list[str]  # top-N failed test names + first lines of traceback
+
+
+_SUMMARY_RE = re.compile(
+    r"(?:(\d+)\s+failed)?,?\s*(?:(\d+)\s+passed)?,?\s*"
+    r"(?:(\d+)\s+error)?,?\s*(?:(\d+)\s+skipped)?",
+    re.IGNORECASE,
+)
+# Pytest's terminal summary line — matches both decorated and -q forms:
+#   "===== 3 passed, 1 failed in 0.42s ====="
+#   "1 passed in 0.03s"
+#   "2 failed, 3 passed in 1.12s"
+_FINAL_LINE_RE = re.compile(
+    r"=*\s*"
+    r"(?:(\d+)\s+failed)?,?\s*"
+    r"(?:(\d+)\s+passed)?,?\s*"
+    r"(?:(\d+)\s+error[s]?)?,?\s*"
+    r"(?:(\d+)\s+skipped)?\s*"
+    r"in\s+[\d.]+s",
+    re.IGNORECASE,
+)
+
+
+def _parse_pytest_summary(stdout: str) -> tuple[int, int, int, int]:
+    """Return ``(passed, failed, errors, skipped)`` parsed from pytest stdout."""
+    passed = failed = errors = skipped = 0
+    for line in reversed(stdout.splitlines()):
+        m = _FINAL_LINE_RE.search(line)
+        if m:
+            failed = int(m.group(1) or 0)
+            passed = int(m.group(2) or 0)
+            errors = int(m.group(3) or 0)
+            skipped = int(m.group(4) or 0)
+            break
+    return passed, failed, errors, skipped
+
+
+def _extract_failure_excerpts(stdout: str, max_failures: int = 5) -> list[str]:
+    """Pull the FAILED test ids + a short excerpt of each traceback."""
+    out: list[str] = []
+    failed_ids: list[str] = []
+    for line in stdout.splitlines():
+        if line.startswith("FAILED "):
+            # format: "FAILED tests/test_x.py::test_y - AssertionError: ..."
+            failed_ids.append(line.replace("FAILED ", "").strip())
+    for fid in failed_ids[:max_failures]:
+        out.append(fid)
+    return out
+
+
+def _filter_tests_to_package(tests_dir: Path, package: str) -> list[str]:
+    """Return list of test file paths whose imports reference ``package``.
+
+    Closes the wrong-package gameability hole: an LLM that emits files into
+    `forge_greeter` but tests for `forge_greeter` shouldn't credit the
+    behavioural score against a request for `mdconv`.  This requires every
+    counted test file to import the requested package.
+    """
+    out: list[str] = []
+    for f in sorted(tests_dir.rglob("test_*.py")):
+        if "__pycache__" in f.parts:
+            continue
+        try:
+            text = f.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        # Match `from <pkg>.…` or `import <pkg>` (word-boundary).
+        if re.search(rf"\b(?:from|import)\s+{re.escape(package)}\b", text):
+            out.append(str(f))
+    return out
+
+
+def run_pytest(*, output_root: Path, package: str | None = None,
+               tests_subdir: str = "tests",
+               timeout_s: float = 60.0) -> TestRunReport:
+    """Run ``pytest <tests_subdir>`` inside ``output_root``.
+
+    When ``package`` is given, ONLY tests that import that package count
+    toward the pass-ratio.  This prevents wrong-package gaming where the
+    LLM emits code into ``forge_greeter`` but the request was ``mdconv`` —
+    forge_greeter's tests would otherwise credit the score.
+
+    ``PYTHONPATH`` is set so ``src/`` is importable without a pip install.
+    Skips cleanly when the tests dir doesn't exist.
+    """
+    import time
+    output_root = Path(output_root).resolve()
+    tests_dir = output_root / tests_subdir
+    src_root = output_root / "src"
+    if not tests_dir.exists():
+        return TestRunReport(
+            schema_version="atomadic-forge.test_run/v1",
+            tests_dir=str(tests_dir), ran=False, passed=0, failed=0,
+            errors=0, skipped=0, total=0, pass_ratio=0.0, duration_ms=0,
+            pytest_summary="(no tests/ directory)",
+            failure_excerpts=[],
+        )
+
+    # Filter to tests that actually target the requested package.
+    targeted: list[str] | None = None
+    if package:
+        targeted = _filter_tests_to_package(tests_dir, package)
+        if not targeted:
+            return TestRunReport(
+                schema_version="atomadic-forge.test_run/v1",
+                tests_dir=str(tests_dir), ran=True, passed=0, failed=0,
+                errors=0, skipped=0, total=0, pass_ratio=0.0, duration_ms=0,
+                pytest_summary=(
+                    f"(no test files import package {package!r} — refused to "
+                    "credit unrelated tests)"
+                ),
+                failure_excerpts=[],
+            )
+    env = {**os.environ}
+    pp = env.get("PYTHONPATH", "")
+    sep = ";" if sys.platform == "win32" else ":"
+    new_pp = str(src_root) if not pp else f"{src_root}{sep}{pp}"
+    env["PYTHONPATH"] = new_pp
+    env["PYTHONIOENCODING"] = "utf-8"
+    env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1"
+
+    pytest_targets = targeted if targeted else [str(tests_dir)]
+    start = time.perf_counter()
+    try:
+        proc = subprocess.run(
+            [sys.executable, "-m", "pytest", *pytest_targets,
+             "-o", "addopts=", "--tb=line", "-q", "--no-header",
+             "-p", "no:cacheprovider"],
+            cwd=str(output_root), env=env,
+            capture_output=True, text=True, timeout=timeout_s,
+            encoding="utf-8", errors="replace",
+        )
+        duration_ms = int((time.perf_counter() - start) * 1000)
+    except subprocess.TimeoutExpired:
+        return TestRunReport(
+            schema_version="atomadic-forge.test_run/v1",
+            tests_dir=str(tests_dir), ran=True, passed=0, failed=0,
+            errors=1, skipped=0, total=1, pass_ratio=0.0,
+            duration_ms=int(timeout_s * 1000),
+            pytest_summary=f"timeout after {timeout_s}s",
+            failure_excerpts=[],
+        )
+
+    stdout = proc.stdout or ""
+    stderr = proc.stderr or ""
+    passed, failed, errors, skipped = _parse_pytest_summary(stdout)
+    total = passed + failed + errors  # skipped doesn't count toward ratio
+    ratio = (passed / total) if total > 0 else 0.0
+
+    # The combined output is what we pass back to the LLM.
+    summary_lines = (stdout + ("\n" + stderr if stderr else "")).splitlines()
+    summary = "\n".join(summary_lines[-40:]).strip()
+
+    return TestRunReport(
+        schema_version="atomadic-forge.test_run/v1",
+        tests_dir=str(tests_dir),
+        ran=total > 0 or errors > 0,
+        passed=passed, failed=failed, errors=errors, skipped=skipped,
+        total=total, pass_ratio=ratio, duration_ms=duration_ms,
+        pytest_summary=summary,
+        failure_excerpts=_extract_failure_excerpts(stdout),
+    )

atomadic_forge/a1_at_functions/test_selector.py

@@ -0,0 +1,122 @@
+"""Tier a1 — select_tests: per-intent test selection (Codex #7).
+
+Codex's prescription:
+
+  > Agents over-run tests or under-run them. Add select_tests({
+  >   changed_files, intent }). Returns minimum test set, full-
+  >   confidence test set, commands, why those tests matter.
+
+Pure: takes paths + intent, walks tests/ to match by name + path
+heuristics. The 'minimum' set is the mirror-style direct match
+(tests/test_<stem>.py); the 'full' set adds tier-mate tests + any
+tests under the proposed file's tier.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TypedDict
+
+SCHEMA_VERSION_TEST_SELECT_V1 = "atomadic-forge.test_select/v1"
+
+_TIER_NAMES = (
+    "a0_qk_constants", "a1_at_functions", "a2_mo_composites",
+    "a3_og_features", "a4_sy_orchestration",
+)
+
+
+class TestSelection(TypedDict, total=False):
+    schema_version: str
+    intent: str
+    changed_files: list[str]
+    minimum_tests: list[str]
+    full_tests: list[str]
+    minimum_command: str
+    full_command: str
+    rationale: list[str]
+
+
+def _detect_tier(path: str) -> str | None:
+    for part in path.split("/"):
+        if part in _TIER_NAMES:
+            return part
+    return None
+
+
+def _list_tests(project_root: Path) -> list[str]:
+    candidates: list[str] = []
+    for sub in ("tests", "test"):
+        d = project_root / sub
+        if not d.is_dir():
+            continue
+        for f in sorted(d.rglob("test_*.py")):
+            candidates.append(str(f.relative_to(project_root).as_posix()))
+        for f in sorted(d.rglob("*_test.py")):
+            candidates.append(str(f.relative_to(project_root).as_posix()))
+    return sorted(set(candidates))
+
+
+def select_tests(
+    *,
+    intent: str,
+    changed_files: list[str],
+    project_root: Path,
+) -> TestSelection:
+    """Compute minimum + full test sets for a proposed change."""
+    project_root = Path(project_root).resolve()
+    all_tests = _list_tests(project_root)
+    minimum: set[str] = set()
+    full: set[str] = set()
+    rationale: list[str] = []
+
+    for path in changed_files:
+        stem = Path(path).stem
+        tier = _detect_tier(path)
+        # Direct mirror match
+        for t in all_tests:
+            tname = Path(t).stem
+            if tname in (f"test_{stem}", f"{stem}_test"):
+                minimum.add(t)
+                full.add(t)
+        # Tier-mate matches
+        if tier:
+            for t in all_tests:
+                if f"/{tier}/" in t or t.endswith(f"/{tier}"):
+                    full.add(t)
+        # Any test mentioning the stem in its filename gets full.
+        for t in all_tests:
+            if stem and stem in Path(t).stem and stem != Path(t).stem:
+                full.add(t)
+    if not minimum and all_tests:
+        # No mirror match — the safe minimum is full + a clear note.
+        rationale.append(
+            "no mirror-name test match for any changed file; "
+            "minimum_tests promoted to the full set so coverage is "
+            "preserved."
+        )
+        minimum = set(full or all_tests)
+    if not full:
+        full = set(all_tests)
+        rationale.append(
+            "no tier-mate matches found; full_tests defaulted to the "
+            "complete tests/ tree."
+        )
+    rationale.append(
+        f"intent: {intent[:200]}" if intent
+        else "no intent provided — selection is path-based only."
+    )
+
+    minimum_list = sorted(minimum)
+    full_list = sorted(full)
+    return TestSelection(
+        schema_version=SCHEMA_VERSION_TEST_SELECT_V1,
+        intent=intent,
+        changed_files=list(changed_files),
+        minimum_tests=minimum_list,
+        full_tests=full_list,
+        minimum_command=(
+            "python -m pytest " + " ".join(minimum_list)
+            if minimum_list else "python -m pytest"
+        ),
+        full_command="python -m pytest",
+        rationale=rationale,
+    )

atomadic_forge/a1_at_functions/tier_init_rebuild.py

@@ -0,0 +1,122 @@
+"""Tier a1 — pure rebuilder for each tier's ``__init__.py`` re-exports.
+
+After every iterate turn, Forge regenerates each ``aN_*/__init__.py`` so
+every public name in the tier becomes importable via the tier path:
+
+    from pkg.a1_at_functions import generate_slug, normalize_url
+
+That style is what most LLMs (and humans) reach for, but it doesn't work
+unless the tier's ``__init__.py`` re-exports.  Forge handles this
+deterministically so the LLM's emitted code 'just works'.
+
+Idempotent: running multiple times produces the same output.  The
+generated banner ``# Auto-managed by atomadic-forge`` lets future runs
+detect and refresh; hand-written init files without that banner are left
+alone.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+_BANNER = "# Auto-managed by atomadic-forge — re-exports every public name in this tier."
+_INIT_TEMPLATE = (
+    '"""Tier {tier} — re-exports."""\n'
+    "{banner}\n"
+    "\n"
+    "{imports}"
+    "\n"
+    '__all__ = [{all_list}]\n'
+)
+
+
+def _public_names_in_module(py_file: Path) -> list[str]:
+    """Return public function/class/constant names defined at module top level."""
+    try:
+        tree = ast.parse(py_file.read_text(encoding="utf-8"))
+    except (SyntaxError, OSError):
+        return []
+    names: list[str] = []
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
+            if not node.name.startswith("_"):
+                names.append(node.name)
+        elif isinstance(node, ast.Assign):
+            for t in node.targets:
+                if isinstance(t, ast.Name) and not t.id.startswith("_"):
+                    names.append(t.id)
+        elif isinstance(node, ast.AnnAssign):
+            if isinstance(node.target, ast.Name) and not node.target.id.startswith("_"):
+                names.append(node.target.id)
+    # de-dupe preserving order
+    seen: set[str] = set()
+    out: list[str] = []
+    for n in names:
+        if n not in seen:
+            seen.add(n)
+            out.append(n)
+    return out
+
+
+def render_tier_init(tier: str, *, modules: dict[str, list[str]]) -> str:
+    """Render an ``__init__.py`` for one tier.
+
+    ``modules`` is ``{module_stem: [public_name, ...]}`` for every ``.py``
+    file in that tier dir (excluding ``__init__``).
+    """
+    if not modules:
+        return f'"""Tier {tier} — empty."""\n'
+    import_lines: list[str] = []
+    flat_names: list[str] = []
+    for module_stem in sorted(modules):
+        names = sorted(modules[module_stem])
+        if not names:
+            continue
+        joined = ", ".join(names)
+        import_lines.append(f"from .{module_stem} import {joined}")
+        flat_names.extend(names)
+    if not import_lines:
+        return f'"""Tier {tier} — empty."""\n'
+    all_list = ", ".join(repr(n) for n in sorted(set(flat_names)))
+    return _INIT_TEMPLATE.format(
+        tier=tier,
+        banner=_BANNER,
+        imports="\n".join(import_lines) + "\n",
+        all_list=all_list,
+    )
+
+
+def rebuild_tier_inits(package_root: Path) -> dict[str, str]:
+    """Rebuild every ``aN_*/__init__.py`` under ``package_root``.
+
+    Returns a map of ``relative_path -> new_content`` for the files that
+    Forge re-generated (skips files without the banner — those are
+    user-edited and we leave them alone).
+    """
+    package_root = Path(package_root)
+    written: dict[str, str] = {}
+    for tier_dir in sorted(package_root.iterdir()):
+        if not tier_dir.is_dir():
+            continue
+        if not tier_dir.name.startswith(("a0_", "a1_", "a2_", "a3_", "a4_")):
+            continue
+        modules: dict[str, list[str]] = {}
+        for py in sorted(tier_dir.glob("*.py")):
+            if py.name == "__init__.py":
+                continue
+            names = _public_names_in_module(py)
+            if names:
+                modules[py.stem] = names
+        new_text = render_tier_init(tier_dir.name, modules=modules)
+        init = tier_dir / "__init__.py"
+        # Don't clobber a hand-written init (no banner ⇒ assume user-managed).
+        if init.exists():
+            current = init.read_text(encoding="utf-8")
+            if _BANNER not in current and current.strip() not in (
+                "", f'"""{tier_dir.name}."""',
+            ):
+                continue
+        init.write_text(new_text, encoding="utf-8")
+        written[init.relative_to(package_root).as_posix()] = new_text
+    return written