ferrum-cli 0.1.0__py3-none-any.whl

ferrum/context.py

@@ -0,0 +1,379 @@
+"""Project discovery: find the root, walk it, summarise what's there."""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path, PurePosixPath
+
+from ferrum.config import Config
+from ferrum.safety import DENIED_DIRS, is_denied, is_within
+
+log = logging.getLogger(__name__)
+
+
+class Language(str, Enum):
+    C = "c"
+    CPP = "cpp"
+    RUST = "rust"
+    ASM = "asm"
+    PYTHON = "python"
+
+    @property
+    def label(self) -> str:
+        return {
+            Language.C: "C",
+            Language.CPP: "C++",
+            Language.RUST: "Rust",
+            Language.ASM: "Assembly",
+            Language.PYTHON: "Python",
+        }[self]
+
+
+class BuildSystem(str, Enum):
+    MAKE = "make"
+    CMAKE = "cmake"
+    CARGO = "cargo"
+    PYTEST = "pytest"
+
+    @property
+    def label(self) -> str:
+        return {
+            BuildSystem.MAKE: "Make",
+            BuildSystem.CMAKE: "CMake",
+            BuildSystem.CARGO: "Cargo",
+            BuildSystem.PYTEST: "pytest",
+        }[self]
+
+
+SOURCE_EXTENSIONS: dict[str, Language] = {
+    ".c": Language.C,
+    ".h": Language.C,
+    ".cc": Language.CPP,
+    ".cpp": Language.CPP,
+    ".cxx": Language.CPP,
+    ".c++": Language.CPP,
+    ".hh": Language.CPP,
+    ".hpp": Language.CPP,
+    ".hxx": Language.CPP,
+    ".rs": Language.RUST,
+    ".s": Language.ASM,
+    ".asm": Language.ASM,
+    ".py": Language.PYTHON,
+}
+
+MAKE_NAMES = frozenset({"makefile", "gnumakefile"})
+CMAKE_NAMES = frozenset({"cmakelists.txt"})
+CARGO_NAMES = frozenset({"cargo.toml"})
+PYTEST_NAMES = frozenset({"pyproject.toml", "setup.py", "pytest.ini", "tox.ini"})
+BUILD_FILE_NAMES = MAKE_NAMES | CMAKE_NAMES | CARGO_NAMES | PYTEST_NAMES
+
+PROJECT_MARKERS = (
+    ".git",
+    "cargo.toml",
+    "cmakelists.txt",
+    "makefile",
+    "gnumakefile",
+    "pyproject.toml",
+    "setup.py",
+)
+
+
+@dataclass(frozen=True)
+class FileEntry:
+    """A file we might show the model, path relative to the project root."""
+
+    path: str
+    size: int = 0
+    language: Language | None = None
+
+
+@dataclass
+class ProjectContext:
+    """What we know about the project after a quick look."""
+
+    root: Path
+    languages: list[Language] = field(default_factory=list)
+    build_systems: list[BuildSystem] = field(default_factory=list)
+    files: list[FileEntry] = field(default_factory=list)
+    name: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.name:
+            self.name = self.root.name or str(self.root)
+
+    def display_path(self) -> str:
+        """Project path relative to where the user is standing, usually."""
+        try:
+            rel = os.path.relpath(self.root, Path.cwd())
+        except ValueError:
+            return str(self.root)
+        rel = rel.replace("\\", "/")
+        if rel == ".":
+            return "."
+        if not rel.startswith("."):
+            rel = "./" + rel
+        return rel
+
+    def summary(self) -> str:
+        """The block the CLI prints after `ferrum .`."""
+        languages = ", ".join(lang.label for lang in self.languages) or "unknown"
+        builds = ", ".join(b.label for b in self.build_systems) or "unknown"
+
+        lines = [
+            "Ferrum",
+            "─" * 6,
+            "",
+            f"Project: {self.display_path()}",
+            f"Languages: {languages}",
+            f"Build system: {builds}",
+            "",
+            "Files:",
+        ]
+        if self.files:
+            lines.extend(f"  {entry.path}" for entry in self.files)
+        else:
+            lines.append("  (none)")
+        return "\n".join(lines)
+
+
+def find_project_root(start: Path) -> Path:
+    """Nearest directory at or above start holding a project marker."""
+    start = Path(start).resolve()
+    for candidate in (start, *start.parents):
+        if any((candidate / marker).exists() for marker in PROJECT_MARKERS):
+            return candidate
+    return start
+
+
+@dataclass(frozen=True)
+class _Rule:
+    prefix: str  # directory the .gitignore lived in, "" for the root one
+    regex: re.Pattern[str]
+    negated: bool
+    dir_only: bool
+    anchored: bool
+
+    def matches(self, rel: str, is_dir: bool) -> bool:
+        if self.dir_only and not is_dir:
+            return False
+        if self.prefix:
+            if not rel.startswith(self.prefix):
+                return False
+            rel = rel[len(self.prefix) :]
+        if self.anchored:
+            return self.regex.match(rel) is not None
+        # Patterns without a slash match any name at any depth.
+        return self.regex.match(PurePosixPath(rel).name) is not None
+
+
+def _translate(pattern: str) -> str:
+    """Turn a gitignore glob into a regex. Good enough for real-world files."""
+    out: list[str] = []
+    i = 0
+    while i < len(pattern):
+        char = pattern[i]
+        if char == "*" and pattern[i : i + 2] == "**":
+            if pattern[i : i + 3] == "**/":
+                out.append("(?:.*/)?")  # any number of directories
+                i += 3
+            else:
+                out.append(".*")
+                i += 2
+        elif char == "*":
+            out.append("[^/]*")
+            i += 1
+        elif char == "?":
+            out.append("[^/]")
+            i += 1
+        elif char == "[":
+            end = pattern.find("]", i)
+            if end == -1:
+                out.append(re.escape(char))
+                i += 1
+            else:
+                body = pattern[i + 1 : end]
+                if body.startswith("!"):  # git uses ! where regex uses ^
+                    body = "^" + body[1:]
+                out.append("[" + body + "]")
+                i = end + 1
+        else:
+            out.append(re.escape(char))
+            i += 1
+    return "".join(out)
+
+
+def _compile_rule(line: str, base_rel: str) -> _Rule | None:
+    pattern = line.rstrip()
+    if not pattern or pattern.startswith("#"):
+        return None
+    negated = pattern.startswith("!")
+    if negated:
+        pattern = pattern[1:]
+    dir_only = pattern.endswith("/")
+    if dir_only:
+        pattern = pattern[:-1]
+    if not pattern:
+        return None
+    anchored = pattern.startswith("/") or "/" in pattern
+    pattern = pattern.removeprefix("/")
+    prefix = "" if base_rel == "." else base_rel + "/"
+    return _Rule(
+        prefix=prefix,
+        regex=re.compile("^" + _translate(pattern) + "$"),
+        negated=negated,
+        dir_only=dir_only,
+        anchored=anchored,
+    )
+
+
+def _load_gitignore(directory: Path, base_rel: str) -> list[_Rule]:
+    path = directory / ".gitignore"
+    if not path.is_file():
+        return []
+    try:
+        if path.stat().st_size > 64 * 1024:
+            log.debug("skipping oversized .gitignore: %s", path)
+            return []
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        log.debug("cannot read %s: %s", path, exc)
+        return []
+    rules = []
+    for line in text.splitlines():
+        rule = _compile_rule(line, base_rel)
+        if rule is not None:
+            rules.append(rule)
+    return rules
+
+
+def _is_ignored(rel: str, rules: list[_Rule], is_dir: bool) -> bool:
+    ignored = False
+    for rule in rules:  # last matching rule wins, git-style
+        if rule.matches(rel, is_dir):
+            ignored = not rule.negated
+    return ignored
+
+
+def _ancestor_rules(root: Path, start: Path) -> list[_Rule]:
+    """Gitignore rules from root down to (but not including) start."""
+    if start == root:
+        return []
+    rel = start.relative_to(root)
+    rules: list[_Rule] = []
+    for i in range(len(rel.parts)):
+        base = "/".join(rel.parts[:i]) or "."
+        rules.extend(_load_gitignore(root.joinpath(*rel.parts[:i]), base))
+    return rules
+
+
+def _language_of(name: str) -> Language | None:
+    return SOURCE_EXTENSIONS.get(PurePosixPath(name).suffix.lower())
+
+
+def _walk(
+    root: Path,
+    current: Path,
+    base_rel: str,
+    inherited: list[_Rule],
+    out: list[FileEntry],
+    config: Config,
+) -> None:
+    rules = list(inherited) + _load_gitignore(current, base_rel)
+    try:
+        entries = sorted(current.iterdir())
+    except OSError as exc:
+        log.debug("cannot list %s: %s", current, exc)
+        return
+    for entry in entries:
+        try:
+            rel = entry.relative_to(root).as_posix()
+            if entry.is_dir():
+                if entry.is_symlink() or entry.name in DENIED_DIRS or is_denied(rel):
+                    continue
+                if _is_ignored(rel, rules, is_dir=True):
+                    continue
+                _walk(root, entry, rel, rules, out, config)
+            elif entry.is_file():
+                if is_denied(rel) or _is_ignored(rel, rules, is_dir=False):
+                    continue
+                size = entry.stat().st_size
+                if size > config.max_file_bytes:
+                    continue
+                language = _language_of(entry.name)
+                if language is None and not _is_build_file(entry.name):
+                    continue
+                out.append(FileEntry(path=rel, size=size, language=language))
+        except OSError as exc:
+            log.debug("skipping %s: %s", entry, exc)
+
+
+def _is_build_file(name: str) -> bool:
+    return name.lower() in BUILD_FILE_NAMES
+
+
+def collect_files(
+    root: Path, config: Config, *, start: Path | None = None
+) -> list[FileEntry]:
+    """Source and build files under start (default: root), sorted by path.
+
+    Honours every .gitignore from the root down to start, plus the always-on
+    deny rules and the per-file size cap.
+    """
+    root = Path(root).resolve()
+    start = Path(start).resolve() if start else root
+    if not is_within(root, start):
+        raise ValueError(f"start is outside the project root: {start}")
+    rules = _ancestor_rules(root, start)
+    out: list[FileEntry] = []
+    base = "." if start == root else start.relative_to(root).as_posix()
+    _walk(root, start, base, rules, out, config)
+    out.sort(key=lambda entry: entry.path)
+    return out
+
+
+def limit_entries(entries: list[FileEntry], config: Config) -> list[FileEntry]:
+    """Cut a file list down to the context budget and the file cap."""
+    limited: list[FileEntry] = []
+    total = 0
+    for entry in entries:
+        if len(limited) >= config.max_files or total + entry.size > config.max_context_bytes:
+            break
+        limited.append(entry)
+        total += entry.size
+    return limited
+
+
+def _detect_languages(entries: list[FileEntry]) -> list[Language]:
+    found = {entry.language for entry in entries if entry.language}
+    return [lang for lang in Language if lang in found]
+
+
+def _detect_build_systems(entries: list[FileEntry]) -> list[BuildSystem]:
+    found: set[BuildSystem] = set()
+    for entry in entries:
+        name = PurePosixPath(entry.path).name.lower()
+        if name in MAKE_NAMES:
+            found.add(BuildSystem.MAKE)
+        elif name in CMAKE_NAMES:
+            found.add(BuildSystem.CMAKE)
+        elif name in CARGO_NAMES:
+            found.add(BuildSystem.CARGO)
+        elif name in PYTEST_NAMES:
+            found.add(BuildSystem.PYTEST)
+    return [build for build in BuildSystem if build in found]
+
+
+def discover(root: Path, config: Config | None = None) -> ProjectContext:
+    config = config or Config()
+    root = Path(root).resolve()
+    candidates = collect_files(root, config)
+    return ProjectContext(
+        root=root,
+        languages=_detect_languages(candidates),
+        build_systems=_detect_build_systems(candidates),
+        files=limit_entries(candidates, config),
+    )

ferrum/model.py

@@ -0,0 +1,302 @@
+"""Model access through an OpenAI-compatible chat endpoint.
+
+Ollama exposes one at http://localhost:11434/v1; vLLM and friends do too.
+Only the endpoints actually needed here are implemented: chat completion
+with tool calls, and a best-effort model listing for error hints.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+import urllib.error
+import urllib.request
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from ferrum import __version__
+
+log = logging.getLogger(__name__)
+
+MAX_ATTEMPTS = 3
+RETRYABLE_STATUS = {500, 502, 503, 504}
+# Free tiers meter per minute; waiting out the window beats failing the run.
+RATE_LIMIT_BACKOFF = (15, 30)
+
+MALFORMED_JSON = "malformed_json"
+
+
+class ProviderError(Exception):
+    """The model could not be reached or answered unusably."""
+
+
+class AuthenticationError(ProviderError):
+    pass
+
+
+class RateLimitError(ProviderError):
+    pass
+
+
+@dataclass(frozen=True)
+class ToolCall:
+    id: str
+    name: str
+    arguments: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ModelResponse:
+    content: str
+    tool_calls: list[ToolCall] = field(default_factory=list)
+
+
+class ModelProvider(ABC):
+    @abstractmethod
+    def complete(
+        self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]
+    ) -> ModelResponse:
+        """One round trip: messages in, content and tool calls out."""
+
+
+class OpenAICompatibleProvider(ModelProvider):
+    def __init__(
+        self,
+        base_url: str,
+        model: str,
+        api_key: str = "",
+        timeout: int = 60,
+        max_tokens: int = 2048,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_tokens = max_tokens
+
+    def complete(
+        self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]
+    ) -> ModelResponse:
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "messages": messages,
+            "stream": False,
+            # Without an explicit cap some servers truncate mid-tool-call.
+            "max_tokens": self.max_tokens,
+            "temperature": 0.2,
+        }
+        if tools:
+            payload["tools"] = tools
+        body = json.dumps(payload).encode("utf-8")
+        headers = {
+            "Content-Type": "application/json",
+            # Some edges (Cloudflare) reject urllib's default signature.
+            "User-Agent": f"ferrum/{__version__}",
+        }
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        request = urllib.request.Request(
+            f"{self.base_url}/chat/completions",
+            data=body,
+            headers=headers,
+            method="POST",
+        )
+        raw = self._send(request)
+        return self._parse(raw)
+
+    def _send(self, request: urllib.request.Request) -> bytes:
+        # Gateways flake; retry transient failures before giving up.
+        for attempt in range(1, MAX_ATTEMPTS + 1):
+            try:
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    return response.read()
+            except urllib.error.HTTPError as exc:
+                if exc.code == 429 and attempt < MAX_ATTEMPTS:
+                    wait = RATE_LIMIT_BACKOFF[attempt - 1]
+                    log.warning(
+                        "model server HTTP 429 (rate limited),"
+                        " waiting %ds (attempt %d/%d)",
+                        wait, attempt, MAX_ATTEMPTS,
+                    )
+                    exc.close()
+                    time.sleep(wait)
+                    continue
+                if exc.code in RETRYABLE_STATUS and attempt < MAX_ATTEMPTS:
+                    log.warning(
+                        "model server HTTP %s (attempt %d/%d), retrying",
+                        exc.code, attempt, MAX_ATTEMPTS,
+                    )
+                    exc.close()
+                    time.sleep(attempt)
+                    continue
+                raise self._http_error(exc) from exc
+            except TimeoutError as exc:
+                raise ProviderError(
+                    f"model at {self.base_url} did not answer within "
+                    f"{self.timeout}s"
+                ) from exc
+            except urllib.error.URLError as exc:
+                if attempt < MAX_ATTEMPTS:
+                    log.warning(
+                        "cannot reach model (attempt %d/%d), retrying: %s",
+                        attempt, MAX_ATTEMPTS, exc.reason,
+                    )
+                    time.sleep(attempt)
+                    continue
+                raise ProviderError(
+                    f"cannot reach model at {self.base_url} "
+                    f"(is the server running?): {exc.reason}"
+                ) from exc
+            except OSError as exc:
+                raise ProviderError(
+                    f"cannot reach model at {self.base_url}: {exc}"
+                ) from exc
+        raise ProviderError(f"cannot reach model at {self.base_url}")  # pragma: no cover
+
+    def _http_error(self, exc: urllib.error.HTTPError) -> ProviderError:
+        try:
+            detail = exc.read().decode("utf-8", "replace")[:300]
+        except OSError:
+            detail = ""
+        if exc.code in (401, 403):
+            return AuthenticationError(
+                f"the model server refused the request (HTTP {exc.code}): {detail}"
+            )
+        if exc.code == 429:
+            return RateLimitError(
+                f"rate limited by the model server (HTTP 429): {detail}"
+            )
+        return ProviderError(f"model server returned HTTP {exc.code}: {detail}")
+
+    def _parse(self, raw: bytes) -> ModelResponse:
+        try:
+            message = json.loads(raw)["choices"][0]["message"]
+        except (ValueError, KeyError, IndexError, TypeError) as exc:
+            raise ProviderError(f"malformed response from model: {exc}") from exc
+        content = message.get("content") or ""
+        if not isinstance(content, str):
+            content = str(content)
+        calls: list[ToolCall] = []
+        for entry in message.get("tool_calls") or []:
+            if not isinstance(entry, dict):
+                continue
+            function = entry.get("function") or {}
+            name = function.get("name") or ""
+            raw_args = function.get("arguments") or "{}"
+            if isinstance(raw_args, dict):
+                arguments: dict[str, Any] = raw_args
+            else:
+                try:
+                    parsed = json.loads(raw_args)
+                    arguments = parsed if isinstance(parsed, dict) else {"_": parsed}
+                except ValueError:
+                    # Small models emit unescaped newlines; let the agent
+                    # feed the error back instead of killing the session.
+                    arguments = {MALFORMED_JSON: str(raw_args)}
+            arguments = _normalize_arguments(arguments)
+            calls.append(
+                ToolCall(
+                    id=str(entry.get("id") or name or len(calls)),
+                    name=str(name),
+                    arguments=arguments,
+                )
+            )
+        return ModelResponse(content=content, tool_calls=calls)
+
+
+def _normalize_arguments(args: dict[str, Any]) -> dict[str, Any]:
+    """Small models often echo the tools schema instead of real values.
+
+    Unwrap the shapes they produce; anything schema-shaped becomes a
+    MALFORMED marker so the harness can ask the model to retry with values.
+    """
+    single = args.get("arguments")
+    if set(args) == {"arguments"} and isinstance(single, dict):
+        return _normalize_arguments(single)
+    parameters = args.get("parameters")
+    if isinstance(parameters, dict):
+        if "properties" in parameters:
+            return {MALFORMED_JSON: json.dumps(args)}
+        return parameters
+    if "properties" in args or "required" in args:
+        return {MALFORMED_JSON: json.dumps(args)}
+    return args
+
+
+def extract_tool_calls(content: str) -> list[ToolCall]:
+    """Fallback for models that write tool calls as plain JSON in their text.
+
+    Scans for JSON objects that name a tool; anything else in the prose is
+    ignored. The harness reports unknown names back, so invented tools get
+    corrected on the next turn.
+    """
+    if not content:
+        return []
+    decoder = json.JSONDecoder()
+    calls: list[ToolCall] = []
+    position = 0
+    while True:
+        start = content.find("{", position)
+        if start == -1:
+            break
+        try:
+            obj, end = decoder.raw_decode(content, start)
+        except ValueError:
+            position = start + 1
+            continue
+        position = end
+        if not isinstance(obj, dict):
+            continue
+        name = obj.get("name") or obj.get("tool")
+        if not name and isinstance(obj.get("function"), str):
+            name = obj["function"]
+        if not isinstance(name, str) or not name:
+            continue
+        arguments = obj.get("arguments", obj.get("parameters", obj.get("args", {})))
+        if isinstance(arguments, str):
+            try:
+                parsed = json.loads(arguments)
+            except ValueError:
+                arguments = {MALFORMED_JSON: arguments}
+            else:
+                arguments = parsed if isinstance(parsed, dict) else {"_": parsed}
+        elif not isinstance(arguments, dict):
+            arguments = {"_": arguments}
+        arguments = _normalize_arguments(arguments)
+        calls.append(ToolCall(id=f"text-{len(calls)}", name=name, arguments=arguments))
+    return calls
+
+
+def probe_endpoint(
+    base_url: str, timeout: int = 5, api_key: str = ""
+) -> tuple[list[str], str | None]:
+    """(models, error) for GET /models. Unlike list_models, failures are
+    reported instead of swallowed, so `ferrum doctor` can show why."""
+    url = base_url.rstrip("/") + "/models"
+    request = urllib.request.Request(url)
+    request.add_header("User-Agent", f"ferrum/{__version__}")
+    if api_key:
+        request.add_header("Authorization", f"Bearer {api_key}")
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            data = json.loads(response.read())
+        models = [str(m["id"]) for m in data.get("data", []) if m.get("id")]
+    except urllib.error.HTTPError as exc:
+        exc.close()
+        return [], f"HTTP {exc.code}"
+    except Exception as exc:  # noqa: BLE001 - report any failure, raise none
+        log.debug("model listing failed: %s", exc)
+        return [], str(exc) or type(exc).__name__
+    return models, None
+
+
+def list_models(base_url: str, timeout: int = 5, api_key: str = "") -> list[str]:
+    """Best-effort model listing for hints; never raises.
+
+    Auth is optional: some endpoints (Cloudflare-fronted clouds) refuse
+    anonymous GET /models, while local Ollama ignores the header.
+    """
+    models, _error = probe_endpoint(base_url, timeout=timeout, api_key=api_key)
+    return models