apsimo 1.3.0__py3-none-any.whl

apsimo/toolsmith/miner.py

@@ -0,0 +1,145 @@
+"""Journal miner: turns repeated procedures into tool candidates (Mind M1).
+
+Greenfield per the audit: nothing else aggregates the action journal for
+recurrence. A candidate is a cluster of journal entries whose descriptions
+normalize to the same shape and recur at least `min_occurrences` times in a
+domain, and for which no live tool already exists.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_NUM_RE = re.compile(r"\b\d+\b")
+_HEX_RE = re.compile(r"\b[0-9a-f]{6,}\b")
+_WS_RE = re.compile(r"\s+")
+_STOP = {"the", "a", "an", "to", "for", "of", "and", "on", "in", "with",
+         "is", "was", "at", "by", "this", "that", "it", "from"}
+
+
+def _normalize(text: str) -> str:
+    """Collapse a description to its recurring shape: lowercase, digits and
+    hashes masked, stopwords dropped, whitespace squeezed."""
+    t = (text or "").lower()
+    t = _HEX_RE.sub("#", t)
+    t = _NUM_RE.sub("#", t)
+    t = _WS_RE.sub(" ", t).strip()
+    toks = [w for w in re.findall(r"[a-z#_]+", t) if w not in _STOP]
+    return " ".join(toks[:12])
+
+
+@dataclass
+class ToolCandidate:
+    signature: str
+    domain: str
+    description: str            # a representative journal description
+    occurrences: int
+    evidence: List[str] = field(default_factory=list)   # journal refs/ids
+    sample_descriptions: List[str] = field(default_factory=list)
+
+
+class ToolsmithMiner:
+    def __init__(self, journal: Any = None, registry: Any = None) -> None:
+        self._journal = journal
+        self._registry = registry
+
+    def _j(self) -> Any:
+        if self._journal is not None:
+            return self._journal
+        try:
+            from apsimo.api.routers import host
+            sm = getattr(host, "_self_model", None)
+            return getattr(sm, "journal", None) if sm is not None else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _min_occurrences() -> int:
+        try:
+            return int(os.environ.get("COLONY_TOOLSMITH_MIN_OCCURRENCES", "4"))
+        except ValueError:
+            return 4
+
+    @staticmethod
+    def _excluded_domains() -> set:
+        raw = os.environ.get(
+            "COLONY_TOOLSMITH_EXCLUDE_DOMAINS",
+            "meta_learning,sandbox,toolsmith")
+        return {d.strip() for d in raw.split(",") if d.strip()}
+
+    def mine(self, *, limit: int = 1000) -> List[ToolCandidate]:
+        """Return tool candidates ranked by occurrence count."""
+        journal = self._j()
+        if journal is None:
+            return []
+        try:
+            entries = journal.recent(limit=limit)
+        except Exception as exc:
+            logger.warning("toolsmith miner: journal read failed: %s", exc)
+            return []
+        excluded = self._excluded_domains()
+        clusters: Dict[str, Dict[str, Any]] = defaultdict(
+            lambda: {"count": 0, "domain": "", "descs": [], "refs": []})
+        for e in entries:
+            domain = (e.get("domain") or "").strip().lower()
+            if domain in excluded:
+                continue
+            # only recurring *actions*, not held/blocked deliberations
+            if e.get("decision") not in ("acted", "noted", None):
+                continue
+            desc = e.get("description") or ""
+            sig = _normalize(desc)
+            if len(sig) < 8:
+                continue
+            key = f"{domain}::{sig}"
+            c = clusters[key]
+            c["count"] += 1
+            c["domain"] = domain
+            if len(c["descs"]) < 5:
+                c["descs"].append(desc)
+            ref = e.get("ref") or str(e.get("id") or "")
+            if ref and len(c["refs"]) < 20:
+                c["refs"].append(ref)
+
+        existing = self._existing_names()
+        out: List[ToolCandidate] = []
+        floor = self._min_occurrences()
+        for key, c in clusters.items():
+            if c["count"] < floor:
+                continue
+            sig = key.split("::", 1)[1]
+            if self._signature_covered(sig, existing):
+                continue
+            out.append(ToolCandidate(
+                signature=sig, domain=c["domain"],
+                description=c["descs"][0] if c["descs"] else sig,
+                occurrences=c["count"], evidence=c["refs"],
+                sample_descriptions=c["descs"]))
+        out.sort(key=lambda x: x.occurrences, reverse=True)
+        return out
+
+    def _existing_names(self) -> List[str]:
+        if self._registry is None:
+            return []
+        try:
+            return [t.name for t in self._registry.list()]
+        except Exception:
+            return []
+
+    @staticmethod
+    def _signature_covered(sig: str, tool_names: List[str]) -> bool:
+        """Cheap guard against re-mining a procedure a tool already covers:
+        a tool whose name tokens are a subset of the signature tokens."""
+        sig_tokens = set(sig.replace("#", " ").split())
+        for name in tool_names:
+            name_tokens = set(name.split("_"))
+            if name_tokens and name_tokens <= sig_tokens:
+                return True
+        return False

apsimo/toolsmith/policy.py

@@ -0,0 +1,110 @@
+"""Small static policy for Toolsmith's pure-function capability lane."""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+
+
+SAFE_STDLIB_MODULES = frozenset({
+    "base64", "binascii", "bisect", "calendar", "collections", "csv",
+    "datetime", "decimal", "difflib", "enum", "fractions", "functools",
+    "hashlib", "heapq", "html", "io", "itertools", "json", "math",
+    "operator", "re", "statistics", "string", "textwrap", "unicodedata",
+    "uuid",
+})
+DENIED_CALLS = frozenset({
+    "__import__", "breakpoint", "compile", "delattr", "eval", "exec",
+    "getattr", "globals", "hash", "id", "input", "locals", "open",
+    "setattr", "vars",
+})
+NONDETERMINISTIC_ATTRIBUTES = frozenset({
+    "now", "random", "randint", "randrange", "today", "uniform",
+    "urandom", "utcnow", "uuid1", "uuid4",
+})
+DENIED_NODES = (
+    ast.AsyncFunctionDef,
+    ast.Await,
+    ast.ClassDef,
+    ast.Delete,
+    ast.Global,
+    ast.Nonlocal,
+    ast.Yield,
+    ast.YieldFrom,
+)
+
+
+@dataclass(frozen=True)
+class StaticPolicyVerdict:
+    allowed: bool
+    reasons: tuple[str, ...]
+
+
+class _PureToolVisitor(ast.NodeVisitor):
+    def __init__(self) -> None:
+        self.reasons: list[str] = []
+
+    def _deny(self, reason: str) -> None:
+        if reason not in self.reasons:
+            self.reasons.append(reason)
+
+    def generic_visit(self, node: ast.AST) -> None:
+        if isinstance(node, DENIED_NODES):
+            self._deny(f"unsupported syntax: {type(node).__name__}")
+        super().generic_visit(node)
+
+    def visit_Import(self, node: ast.Import) -> None:
+        for alias in node.names:
+            root = alias.name.split(".", 1)[0]
+            if root not in SAFE_STDLIB_MODULES:
+                self._deny(f"module is not in the pure allowlist: {root}")
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+        root = (node.module or "").split(".", 1)[0]
+        if node.level or root not in SAFE_STDLIB_MODULES:
+            self._deny(f"module is not in the pure allowlist: {root or 'relative'}")
+        self.generic_visit(node)
+
+    def visit_Call(self, node: ast.Call) -> None:
+        if isinstance(node.func, ast.Name) and node.func.id in DENIED_CALLS:
+            self._deny(f"call is not allowed: {node.func.id}")
+        self.generic_visit(node)
+
+    def visit_Attribute(self, node: ast.Attribute) -> None:
+        if node.attr.startswith("__"):
+            self._deny("dunder attribute access is not allowed")
+        if node.attr in NONDETERMINISTIC_ATTRIBUTES:
+            self._deny(f"nondeterministic attribute is not allowed: {node.attr}")
+        self.generic_visit(node)
+
+    def visit_Name(self, node: ast.Name) -> None:
+        if node.id.startswith("__"):
+            self._deny("dunder name access is not allowed")
+
+
+def validate_pure_tool(source_code: str, test_source: str) -> StaticPolicyVerdict:
+    """Validate source and its self-test before either reaches the sandbox."""
+
+    reasons: list[str] = []
+    trees: list[tuple[str, ast.Module]] = []
+    for label, source in (("source", source_code), ("test", test_source)):
+        try:
+            trees.append((label, ast.parse(source or "", filename=f"<{label}>")))
+        except SyntaxError:
+            reasons.append(f"{label} is not valid Python")
+    if reasons:
+        return StaticPolicyVerdict(False, tuple(reasons))
+
+    source_tree = trees[0][1]
+    run_defs = [
+        node for node in source_tree.body
+        if isinstance(node, ast.FunctionDef) and node.name == "run"
+    ]
+    if len(run_defs) != 1:
+        reasons.append("source must define exactly one top-level run function")
+    visitor = _PureToolVisitor()
+    for _, tree in trees:
+        visitor.visit(tree)
+    reasons.extend(visitor.reasons)
+    return StaticPolicyVerdict(not reasons, tuple(dict.fromkeys(reasons)))