agentforge-framework 0.2.0__py3-none-any.whl

agentforge_framework/context/extractors/sql.py

@@ -0,0 +1,121 @@
+"""What tables a statement reads from or writes to, and what columns it names.
+
+There is no SQL parser here and there is deliberately not going to be one. A
+Context Pack is a head start, so an extractor that is right about ordinary
+statements and silent about exotic ones is worth more than a dependency and a
+dialect argument. What it must never do is claim something that is not there,
+which is why it strips comments and string literals before it looks: a table
+name inside a quoted string is prose, not a reference.
+
+Tables are references — what the query reads from and writes to. Columns are
+symbols — the names inside it a change is likely to be about.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .base import Extraction, ordered
+
+#: An identifier as the dialects AgentForge cares about write one:
+#: `orders`, `analytics.orders`, `catalog.schema.table`, and the bracketed or
+#: backticked forms of each.
+_NAME = r'[A-Za-z_][\w$]*|"[^"]+"|`[^`]+`|\[[^\]]+\]'
+_QUALIFIED = rf"(?:{_NAME})(?:\s*\.\s*(?:{_NAME}))*"
+
+#: Where a table name follows a keyword. `INTO` covers both `INSERT INTO` and
+#: `MERGE INTO`; `UPDATE` and `TABLE` cover the statements that name their
+#: target without one.
+_TABLE = re.compile(
+    rf"\b(?:FROM|JOIN|INTO|UPDATE|TABLE|USING)\s+(?!SELECT\b)({_QUALIFIED})",
+    re.IGNORECASE,
+)
+
+#: `alias.column` anywhere in the statement. The right-hand side is the column;
+#: the left is an alias this extractor does not try to resolve back to a table.
+_QUALIFIED_COLUMN = re.compile(rf"\b({_NAME})\s*\.\s*({_NAME})\b")
+
+#: The projection of a `SELECT`, which is where a bare column name is a column
+#: rather than a keyword. Bounded by the first `FROM` at any depth, because a
+#: subquery in the projection is not what this is trying to read.
+_SELECT = re.compile(r"\bSELECT\b(?:\s+DISTINCT\b)?(.*?)(?:\bFROM\b|$)", re.IGNORECASE | re.DOTALL)
+
+_COMMENTS = re.compile(r"--[^\n]*|/\*.*?\*/", re.DOTALL)
+_STRINGS = re.compile(r"'(?:[^']|'')*'")
+
+#: Words that appear where a column name would and are not one.
+_NOT_A_COLUMN = frozenset(
+    {"as", "case", "when", "then", "else", "end", "null", "distinct", "all", "and", "or", "not"}
+)
+
+
+def extract(text: str) -> Extraction:
+    """The tables a statement touches and the columns it names."""
+    statement = _STRINGS.sub("''", _COMMENTS.sub(" ", text))
+
+    tables = [_unquote(match.group(1)) for match in _TABLE.finditer(statement)]
+    columns = [_unquote(match.group(2)) for match in _QUALIFIED_COLUMN.finditer(statement)]
+
+    for projection in _SELECT.findall(statement):
+        columns.extend(_projected(projection))
+
+    # A qualified name is `schema.table`, so its right-hand side reached the
+    # column list as well. Dropping anything that is also a table keeps the pack
+    # from telling a Role that `orders` is a column of itself.
+    named = {_unquote(part) for table in tables for part in table.split(".")}
+    return Extraction(
+        symbols=tuple(column for column in ordered(columns) if column not in named),
+        references=ordered(tables),
+    )
+
+
+def _projected(projection: str) -> list[str]:
+    """The columns a `SELECT` list names, where it names them plainly.
+
+    One bare or qualified identifier per item is read as a column. An expression
+    or a function call is skipped rather than guessed at: `SUM(o.total)` has
+    already given up `total` through the qualified-column pass, and reading
+    `SUM` as a column would put a keyword in the pack.
+    """
+    columns = []
+    for item in _split_top_level(projection):
+        candidate = item.strip().rstrip(",").strip()
+        if not candidate or "(" in candidate or candidate.endswith("*"):
+            continue
+        # `o.total AS revenue` is about `total`; the alias is a name the query
+        # invents, and nothing downstream of the query is in the pack.
+        head = candidate.split()[0]
+        name = _unquote(head.rsplit(".", 1)[-1])
+        if name and name.lower() not in _NOT_A_COLUMN and re.fullmatch(r"[\w$]+", name):
+            columns.append(name)
+    return columns
+
+
+def _split_top_level(projection: str) -> list[str]:
+    """Split a `SELECT` list on the commas that separate its items."""
+    items, depth, start = [], 0, 0
+    for index, char in enumerate(projection):
+        if char == "(":
+            depth += 1
+        elif char == ")":
+            depth = max(0, depth - 1)
+        elif char == "," and depth == 0:
+            items.append(projection[start:index])
+            start = index + 1
+    items.append(projection[start:])
+    return items
+
+
+def _unquote(name: str) -> str:
+    """`"orders"`, `` `orders` ``, and `[orders]` are all `orders`."""
+    name = " ".join(name.split()).replace(" . ", ".").replace(". ", ".").replace(" .", ".")
+    for opening, closing in (('"', '"'), ("`", "`"), ("[", "]")):
+        parts = [
+            part[1:-1] if part.startswith(opening) and part.endswith(closing) else part
+            for part in name.split(".")
+        ]
+        name = ".".join(parts)
+    return name
+
+
+__all__ = ["extract"]

agentforge_framework/context/extractors/yaml.py

@@ -0,0 +1,59 @@
+"""What keys a YAML file sets, as dotted paths.
+
+A config file's shape is the useful thing about it — `gates.tests.suite` tells
+a Role where a setting lives without opening the file. The values are not
+carried: they are what the Role is about to change, and a pack that quoted them
+would be a copy of the file with extra steps.
+
+A file that will not parse yields nothing rather than failing. It is still
+carried in the pack by path, and a Role that needs it reads it.
+"""
+
+from __future__ import annotations
+
+import yaml as pyyaml
+
+from .base import Extraction, ordered
+
+#: How deep a dotted path is followed. Deep enough for a config file's shape,
+#: shallow enough that a document nesting twenty levels does not become the
+#: whole pack. The cap belongs here rather than in the resolver because it is
+#: about what a key path means, not about how large a pack may grow.
+MAX_DEPTH = 4
+
+#: How many entries of a list are walked. A list of two hundred jobs has the
+#: same shape as a list of two, and the pack only needs the shape.
+MAX_ITEMS = 3
+
+
+def extract(text: str) -> Extraction:
+    """Every key the document sets, in the order the file writes them."""
+    document = pyyaml.safe_load(text)
+    return Extraction(symbols=ordered(_keys(document, prefix="", depth=1)))
+
+
+def _keys(node, prefix: str, depth: int) -> list[str]:
+    if depth > MAX_DEPTH:
+        return []
+
+    if isinstance(node, dict):
+        found = []
+        for key, value in node.items():
+            path = f"{prefix}.{key}" if prefix else str(key)
+            found.append(path)
+            found.extend(_keys(value, path, depth + 1))
+        return found
+
+    if isinstance(node, list):
+        # A list index is not a key, so the prefix does not grow: two mappings
+        # in a list of steps contribute the same paths, which is the shape.
+        return [
+            path
+            for item in node[:MAX_ITEMS]
+            for path in _keys(item, prefix, depth)
+        ]
+
+    return []
+
+
+__all__ = ["MAX_DEPTH", "MAX_ITEMS", "extract"]

agentforge_framework/context/prompt.py

@@ -0,0 +1,104 @@
+"""The Context Pack as a Role is handed it.
+
+One renderer for every Role, because the pack is the same head start whatever
+the Role does with it, and five copies of this would drift the first time one of
+them was improved.
+
+Symbols are grouped under the file they came from rather than listed as forty
+`path::name` strings. The pack is stored qualified — that is what survives an
+Issue body and what a resolver can be deterministic about — but a prompt that
+repeated one path twenty-five times would spend on punctuation the tokens this
+whole milestone exists to save.
+
+The two sentences at the top of the block matter as much as the lists. A Role
+that reads the pack as an exhaustive account of the repository stops looking,
+and a resolver mistake then costs correctness rather than tokens — so the block
+says outright what the pack is and what it is not.
+"""
+
+from __future__ import annotations
+
+from ..core.contracts import ContextPack
+
+PREAMBLE = """\
+AgentForge resolved this from the frozen Plan before you were invoked, so you \
+do not have to rediscover it. It is a head start and not a boundary: if you \
+need a file it does not name, read that file.\
+"""
+
+#: How a symbol names the file it was read out of. The resolver writes it and
+#: this module is the only thing that takes it apart again.
+QUALIFIER = "::"
+
+
+def render_context_block(context: ContextPack) -> str:
+    """The `## Context Pack` section of a Role's prompt, or nothing.
+
+    An empty pack renders as an empty string rather than as a heading with
+    nothing under it. A Run started with no pack should not spend its prompt
+    telling a Role that AgentForge has nothing for it.
+    """
+    if not context:
+        return ""
+
+    parts = [PREAMBLE]
+
+    if context.files:
+        parts.append("**Read these files:**\n\n" + "\n".join(_file_lines(context)))
+
+    loose = _unqualified(context)
+    if loose:
+        parts.append("**The work is in these symbols:** " + ", ".join(loose))
+
+    if context.references:
+        parts.append("**Those files reach for:** " + ", ".join(context.references))
+
+    if context.conventions:
+        parts.append("**Follow these conventions:** " + ", ".join(context.conventions))
+
+    # Last in the block, and under its own heading. The Orchestrator's
+    # conventions above are about this Task; these are about the technology, and
+    # a Role reading one run-on list of both cannot tell which of them the Plan
+    # actually asked for.
+    if context.fragments:
+        parts.append(
+            "**This repository's technology is held to these conventions:**\n\n"
+            + "\n\n".join(context.fragments)
+        )
+
+    return "\n## Context Pack\n\n" + "\n\n".join(parts) + "\n"
+
+
+def _file_lines(context: ContextPack) -> list[str]:
+    """One line per file, carrying the symbols read out of it.
+
+    A file with no symbols is still a line. It is either a file the Run is about
+    to create or one of a type no Extractor claims, and in both cases the path
+    is the whole of what AgentForge knows.
+    """
+    lines = []
+    for path in context.files:
+        prefix = f"{path}{QUALIFIER}"
+        names = [
+            symbol.removeprefix(prefix) for symbol in context.symbols
+            if symbol.startswith(prefix)
+        ]
+        listed = ", ".join(names)
+        lines.append(f"- `{path}` — {listed}" if listed else f"- `{path}`")
+    return lines
+
+
+def _unqualified(context: ContextPack) -> list[str]:
+    """Symbols that name no file in the pack — the Orchestrator's own.
+
+    It declares symbols while it plans and does not always say where they live.
+    Dropping those would throw away the one part of the pack a human wrote.
+    """
+    return [
+        symbol
+        for symbol in context.symbols
+        if not any(symbol.startswith(f"{path}{QUALIFIER}") for path in context.files)
+    ]
+
+
+__all__ = ["PREAMBLE", "QUALIFIER", "render_context_block"]

agentforge_framework/context/resolver.py

@@ -0,0 +1,185 @@
+"""Turning a frozen Plan into the Context Pack its Agents are handed.
+
+Six Roles run against one Issue, and before this existed each of them opened the
+repository and rediscovered the same files. The frozen Plan already names what
+the work touches (ADR-0003), so the reading can be done once, by AgentForge,
+and handed to every Role. That is the whole idea; ADR-0010 records why it is
+resolved here rather than declared by the Orchestrator or scanned per Step.
+
+Three properties this module owes its callers:
+
+- **Deterministic.** The same Plan against the same repository resolves to the
+  same pack, so two Runs of one Issue can be compared to each other. Nothing
+  here iterates a set or walks a directory in filesystem order.
+- **Bounded.** A Plan naming forty files must not produce a pack larger than
+  the repository, so every list has a cap and the caps are constants up here
+  where a project can argue with them.
+- **Inside the repository.** A Plan naming `../../.ssh/id_rsa` resolves to
+  nothing. An Issue body is editable by anyone who can comment on it, and a
+  resolver that read whatever it was pointed at would be the way in.
+
+What it is not is a search. Nothing here guesses at files the Plan did not
+name — a pack assembled from a fresh scan would drift between Steps, and the
+frozen Plan exists so that it does not.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from pathlib import Path
+
+from ..core.contracts import ContextPack, Plan
+from .extractors import extract
+from .extractors.base import Extraction
+
+#: How large a pack may get. These bound the pack rather than the repository:
+#: the point of a Context Pack is that it is smaller than looking, and a cap of
+#: a thousand symbols would be a pack nobody saved anything by reading.
+MAX_FILES = 40
+MAX_SYMBOLS = 120
+MAX_SYMBOLS_PER_FILE = 25
+MAX_REFERENCES = 60
+
+#: Files past this size are carried by path and not read. A generated migration
+#: or a checked-in dataset yields hundreds of symbols and no understanding, and
+#: reading one is slower than every other thing this module does.
+MAX_BYTES = 200_000
+
+
+def resolve_pack(
+    plan: Plan,
+    root: Path | str,
+    declared: ContextPack | None = None,
+    extractors: Mapping[str, Callable[[str], Extraction]] | None = None,
+) -> ContextPack:
+    """The pack for this Plan: what it names, and what those files contain.
+
+    `declared` is the Orchestrator's own pack, carried in the Issue body. It is
+    kept rather than replaced — the Orchestrator read the repository while it
+    planned, and its conventions are a judgement no extractor produces — and
+    what the Plan's own steps name comes first, because that is the work.
+
+    A file the Plan names and the repository does not have is still carried. It
+    is a file the Run is about to create, and dropping it would leave a Role
+    reading a pack that disagrees with the plan it was handed.
+
+    `extractors` is the table to read with, and `None` means the built-in three.
+    A Run with active Plugins passes the wider table `core.registry` assembles,
+    which is the whole of how a Plugin's reader reaches a pack: nothing else in
+    this module knows a Plugin exists, and the caps below fall on a Plugin's
+    output exactly as they fall on a built-in extractor's.
+    """
+    root = Path(root)
+    declared = declared or ContextPack()
+
+    paths = _paths(plan, declared, root)
+    symbols = list(declared.symbols)
+    references: list[str] = list(declared.references)
+
+    for path in paths:
+        extraction = _read(root / path, extractors)
+        symbols.extend(
+            f"{path}::{name}" for name in extraction.symbols[:MAX_SYMBOLS_PER_FILE]
+        )
+        references.extend(extraction.references)
+
+    return ContextPack(
+        files=tuple(paths),
+        symbols=_capped(symbols, MAX_SYMBOLS),
+        references=_capped(references, MAX_REFERENCES),
+        conventions=declared.conventions,
+    )
+
+
+def _paths(plan: Plan, declared: ContextPack, root: Path) -> list[str]:
+    """Every file the Plan and the declared pack name, in that order.
+
+    Plan first because a step's files are the work itself, and the cap should
+    fall on the Orchestrator's supporting reading rather than on the thing the
+    Run was filed to change.
+    """
+    named = [path for step in plan.steps for path in step.files]
+    named += list(declared.files)
+
+    resolved: dict[str, None] = {}
+    for raw in named:
+        path = inside(raw, root)
+        if path is not None:
+            resolved.setdefault(path, None)
+    return list(resolved)[:MAX_FILES]
+
+
+def inside(raw: str, root: Path) -> str | None:
+    """The path as a repository-relative posix string, or `None` if it escapes.
+
+    Absolute paths and `..` are refused rather than clamped. A Plan that names
+    one is wrong about the repository, and a resolver that quietly reinterpreted
+    it would hand a Role a file nobody asked for.
+    """
+    text = str(raw).strip().replace("\\", "/")
+    if not text:
+        return None
+
+    candidate = Path(text)
+    if candidate.is_absolute() or ".." in candidate.parts:
+        return None
+
+    # Resolving both sides catches a symlink pointing out of the tree, which the
+    # `..` check above does not see.
+    try:
+        (root / candidate).resolve().relative_to(root.resolve())
+    except (OSError, ValueError):
+        return None
+
+    return candidate.as_posix().removeprefix("./")
+
+
+def _read(
+    path: Path, extractors: Mapping[str, Callable[[str], Extraction]] | None = None
+) -> Extraction:
+    """What one file contains, or an empty extraction if it cannot be read.
+
+    A missing file, a directory, an unreadable one, and a file too large to be
+    worth reading all land here, and all of them mean the same thing: the pack
+    carries the path and claims nothing about the contents.
+    """
+    text = file_text(path)
+    return extract(path, text, extractors) if text else Extraction()
+
+
+def file_text(path: Path) -> str:
+    """One file's text, or empty where reading it is not worth it or not possible.
+
+    Public because `core.registry` reads the same files when it detects a Plugin
+    by what the blast radius imports, and the two must agree about which files
+    are readable. A detection that read a two-hundred-megabyte file the pack
+    skips would be paying for an answer the pack never uses.
+    """
+    try:
+        if not path.is_file() or path.stat().st_size > MAX_BYTES:
+            return ""
+        return path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+
+
+def _capped(values, limit: int) -> tuple[str, ...]:
+    """Deduplicated in first-seen order, then truncated to the cap."""
+    seen: dict[str, None] = {}
+    for value in values:
+        text = str(value).strip()
+        if text:
+            seen.setdefault(text, None)
+    return tuple(seen)[:limit]
+
+
+__all__ = [
+    "MAX_BYTES",
+    "MAX_FILES",
+    "MAX_REFERENCES",
+    "MAX_SYMBOLS",
+    "MAX_SYMBOLS_PER_FILE",
+    "file_text",
+    "inside",
+    "resolve_pack",
+]

agentforge_framework/core/__init__.py

@@ -0,0 +1 @@
+"""Core AgentForge abstractions."""

agentforge_framework/core/commands.py

@@ -0,0 +1,170 @@
+"""Running a Command: a repeated chore, with no inference anywhere in it.
+
+`agentforge run scaffold-dbt-model orders` writes the files and exits. No Issue
+is filed, no Run starts, no Provider is invoked, and nothing here reads the
+repository to decide what to write — a Command is a template and an argument
+vector, and the whole of what it will do is readable in the Plugin that declares
+it. That is what makes its output reviewable as an ordinary diff rather than as
+a thing somebody has to check for hallucination.
+
+Three rules this module owes whoever types one:
+
+- **It never overwrites.** A Command that clobbered a file would be a Command
+  nobody dares run twice, and the failure would be silent in the one place — a
+  working tree — where the tool has already promised the diff is the review.
+- **It never writes outside the repository.** The same containment rule the
+  Context Pack resolver applies, for the same reason: a template path is data,
+  and data that renders to `../../.ssh/authorized_keys` is refused rather than
+  clamped.
+- **It runs processes through the Command Runner, under ADR-0007.** A Command
+  that starts something is subject to the same default-deny as everything else,
+  and the human typing `agentforge run` is the grant. Nothing else here starts a
+  process, and nothing here imports `subprocess`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from string import Template
+
+from ..context.resolver import inside
+from .contracts import Command
+from .process import CommandResult, CommandRunner, MissingBinary
+
+
+@dataclass(frozen=True)
+class CommandOutcome:
+    """What running one Command did, or why it did nothing.
+
+    `written` is what a human is about to read in `git status`, so it is carried
+    even when the process that followed the writing failed: files that reached
+    the tree are the tree's now, and a report that hid them would send somebody
+    looking for a mess they were not told about.
+    """
+
+    command: str
+    written: tuple[str, ...] = ()
+    result: CommandResult | None = None
+    error: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return not self.error and (self.result is None or self.result.ok)
+
+
+def run_command(
+    command: Command,
+    arguments: list[str] | tuple[str, ...],
+    root: Path | str,
+    runner: CommandRunner,
+    allow_commands: bool = False,
+) -> CommandOutcome:
+    """Run one Command in one repository, and say what it did.
+
+    `allow_commands` is ADR-0007's posture and matters only to a Command that
+    declares an `argv`: writing a file is what an Agent does anyway, and starting
+    a process is the thing the gate is about. `agentforge run` passes it because
+    a human typed the name — an explicit, attended grant is exactly what that
+    ADR asks for — and a Run passes its own flag, so a Plugin cannot become the
+    route by which an unattended Agent executes arbitrary code.
+
+    Every failure is a returned outcome rather than an exception. The caller is
+    a CLI that has to print something and pick an exit status, and a traceback
+    is neither.
+    """
+    root = Path(root)
+
+    if len(arguments) != len(command.arguments):
+        return CommandOutcome(command=command.name, error=_usage(command))
+
+    values = dict(zip(command.arguments, arguments, strict=True))
+
+    # Before anything is written, rather than after. A Command refused halfway
+    # leaves files in a tree whose author was told the Command did not run.
+    if command.argv and not allow_commands:
+        return CommandOutcome(
+            command=command.name,
+            error=(
+                f"{command.name} runs `{' '.join(command.argv)}`, and command execution "
+                "is denied here (ADR-0007). Run it yourself, or start the Run with "
+                "--allow-commands."
+            ),
+        )
+
+    try:
+        planned = [_render(template, values, root) for template in command.templates]
+    except KeyError as exc:
+        # A placeholder no argument answers for. A declaration fault rather than
+        # a typing one, and it names the placeholder so whoever wrote the Plugin
+        # can find it.
+        return CommandOutcome(
+            command=command.name,
+            error=f"{command.name} names a placeholder its arguments do not define: {exc}",
+        )
+    except ValueError as exc:
+        return CommandOutcome(command=command.name, error=str(exc))
+
+    for path, _ in planned:
+        if (root / path).exists():
+            return CommandOutcome(
+                command=command.name,
+                error=(
+                    f"{path} already exists. {command.name} writes files and never "
+                    "replaces one; move it aside, or name something else."
+                ),
+            )
+
+    written: list[str] = []
+    for path, text in planned:
+        target = root / path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(text, encoding="utf-8")
+        written.append(path)
+
+    if not command.argv:
+        return CommandOutcome(command=command.name, written=tuple(written))
+
+    argv = tuple(_substitute(part, values) for part in command.argv)
+    try:
+        result = runner.run(argv, cwd=root)
+    except MissingBinary as exc:
+        return CommandOutcome(command=command.name, written=tuple(written), error=str(exc))
+
+    return CommandOutcome(command=command.name, written=tuple(written), result=result)
+
+
+def _render(template, values: dict[str, str], root: Path) -> tuple[str, str]:
+    """One template as the path it writes and the text it writes there."""
+    rendered = _substitute(template.path, values)
+    path = inside(rendered, root)
+    if path is None:
+        raise ValueError(
+            f"{rendered!r} is outside the repository. A Command writes into the tree "
+            "it was run in and nowhere else."
+        )
+    return path, _substitute(template.text, values)
+
+
+def _substitute(source: str, values: dict[str, str]) -> str:
+    """`$name` and `${name}`, and `$$` for a literal dollar.
+
+    `string.Template` rather than `str.format`, because a dbt model is Jinja and
+    a template full of `{{ ref(...) }}` would have to double every brace it
+    already carries.
+    """
+    return Template(source).substitute(values)
+
+
+def _usage(command: Command) -> str:
+    """What to type instead, in the shape the CLI's own help uses."""
+    named = " ".join(f"<{name}>" for name in command.arguments)
+    takes = (
+        f"takes {len(command.arguments)} argument(s)"
+        if command.arguments
+        else "takes no arguments"
+    )
+    return f"{command.name} {takes}: agentforge run {command.name} {named}".rstrip()
+
+
+__all__ = ["CommandOutcome", "run_command"]