askfaro-progressive-context 0.1.0__py3-none-any.whl

askfaro_progressive_context/__init__.py

@@ -0,0 +1,57 @@
+"""askfaro-progressive-context: compile any content into a tiered, budget-aware,
+agent-navigable progressive-disclosure manifest, plus an expansion protocol."""
+
+from .eval import CaseResult, EvalReport, NavCase, run_case, run_eval
+from .llm import LLMClient, OpenAICompatibleClient
+from .navigator import KeywordNavigator, LLMNavigator, Navigator
+from .runtime import (
+    VIEW_LEVELS,
+    BudgetExceeded,
+    FrontierEntry,
+    LeafResolver,
+    Runtime,
+    SearchBackend,
+    dict_resolver,
+    render_descriptor,
+)
+from .session import LOCAL, REMOTE, ModeConfig, NavSession
+from .tokenizer import make_tokenizer
+from .types import PROTOCOL_USAGE, Manifest, Node, Payload, Variant, estimate_tokens
+from .validate import schema_errors, structural_errors, validate
+
+__version__ = "0.0.7"
+
+__all__ = [
+    "BudgetExceeded",
+    "CaseResult",
+    "EvalReport",
+    "FrontierEntry",
+    "KeywordNavigator",
+    "LLMClient",
+    "LLMNavigator",
+    "LOCAL",
+    "REMOTE",
+    "LeafResolver",
+    "Manifest",
+    "ModeConfig",
+    "NavCase",
+    "NavSession",
+    "Navigator",
+    "Node",
+    "OpenAICompatibleClient",
+    "PROTOCOL_USAGE",
+    "Payload",
+    "Runtime",
+    "SearchBackend",
+    "VIEW_LEVELS",
+    "Variant",
+    "dict_resolver",
+    "estimate_tokens",
+    "make_tokenizer",
+    "render_descriptor",
+    "run_case",
+    "run_eval",
+    "schema_errors",
+    "structural_errors",
+    "validate",
+]

askfaro_progressive_context/build/__init__.py

@@ -0,0 +1,33 @@
+"""The compiler: source content -> annotated tree -> pcx manifest variants.
+
+Pipeline: an Adapter yields a SourceTree (native structure, verbatim leaves),
+the descriptor engine generates what/when/keywords, cost annotation tokenizes
+and rolls up subtree costs, and emit writes one manifest per budget variant
+plus an llms.txt export.
+"""
+
+from .compiler import BuildResult, compile_source
+from .descriptors import (
+    Descriptor,
+    DescriptorModel,
+    FakeDescriptorModel,
+    Grade,
+    LLMDescriptorModel,
+    cache_from_manifest,
+    generate_descriptors,
+)
+from .ir import SourceNode, SourceTree
+
+__all__ = [
+    "BuildResult",
+    "Descriptor",
+    "DescriptorModel",
+    "FakeDescriptorModel",
+    "Grade",
+    "LLMDescriptorModel",
+    "SourceNode",
+    "SourceTree",
+    "cache_from_manifest",
+    "compile_source",
+    "generate_descriptors",
+]

askfaro_progressive_context/build/_frontmatter.py

@@ -0,0 +1,62 @@
+"""Tiny frontmatter reader for markdown sources.
+
+Uses PyYAML if available; otherwise a minimal parser that handles the shapes
+our adapters need: top-level `key: value`, inline lists `[a, b]`, and a single
+nested mapping block (e.g. a one-fact memory store's `metadata:`). Keeps the core
+dependency-free.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
+    if not text.startswith("---"):
+        return {}, text
+    end = text.find("\n---", 3)
+    if end == -1:
+        return {}, text
+    block = text[3:end].strip("\n")
+    body = text[end + 4 :].lstrip("\n")
+    return _parse_yaml(block), body
+
+
+def _parse_yaml(block: str) -> dict[str, Any]:
+    try:
+        import yaml  # type: ignore
+
+        data = yaml.safe_load(block)
+        return data if isinstance(data, dict) else {}
+    except ImportError:
+        return _minimal_parse(block)
+
+
+def _scalar(v: str) -> Any:
+    v = v.strip()
+    if v.startswith("[") and v.endswith("]"):
+        inner = v[1:-1].strip()
+        return [x.strip().strip("\"'") for x in inner.split(",")] if inner else []
+    return v.strip("\"'")
+
+
+def _minimal_parse(block: str) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    parent: str | None = None
+    for line in block.splitlines():
+        if not line.strip() or line.strip().startswith("#"):
+            continue
+        indented = line[0] in " \t"
+        key, _, val = line.strip().partition(":")
+        key = key.strip()
+        if indented and parent is not None:
+            if not isinstance(out.get(parent), dict):
+                out[parent] = {}
+            out[parent][key] = _scalar(val)
+        elif val.strip() == "":
+            out[key] = {}
+            parent = key
+        else:
+            out[key] = _scalar(val)
+            parent = None
+    return out

askfaro_progressive_context/build/adapters/__init__.py

@@ -0,0 +1,21 @@
+"""Adapters turn a source on disk into a SourceTree.
+
+Phase 1 ships the four already-hierarchical kinds (no structure inference):
+docs, skills, tools, memory.
+"""
+
+from .base import Adapter, get_adapter, register_adapter
+from .docs import DocsAdapter
+from .memory import MemoryAdapter
+from .skills import SkillsAdapter
+from .tools import ToolsAdapter
+
+__all__ = [
+    "Adapter",
+    "DocsAdapter",
+    "MemoryAdapter",
+    "SkillsAdapter",
+    "ToolsAdapter",
+    "get_adapter",
+    "register_adapter",
+]

askfaro_progressive_context/build/adapters/base.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from ..ir import SourceTree
+
+_REGISTRY: dict[str, "Adapter"] = {}
+
+
+class Adapter(Protocol):
+    kind: str
+
+    def load(self, path: Path, *, source_id: str | None = None) -> SourceTree:
+        ...
+
+
+def register_adapter(adapter: "Adapter") -> "Adapter":
+    _REGISTRY[adapter.kind] = adapter
+    return adapter
+
+
+def get_adapter(kind: str) -> "Adapter":
+    if kind not in _REGISTRY:
+        raise KeyError(f"unknown adapter kind {kind!r}; known: {sorted(_REGISTRY)}")
+    return _REGISTRY[kind]
+
+
+def slugify(text: str) -> str:
+    out = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
+    while "--" in out:
+        out = out.replace("--", "-")
+    return out or "node"

askfaro_progressive_context/build/adapters/docs.py

@@ -0,0 +1,61 @@
+"""docs adapter — a directory tree of markdown.
+
+Mirrors the filesystem: directories become branches, `.md`/`.mdx` files become
+leaves. Titles come from the first H1 if present, else the filename. The native
+hierarchy is used as-is (no clustering).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from .._frontmatter import split_frontmatter
+from ..ir import SourceNode, SourceTree
+from .base import register_adapter, slugify
+
+_H1 = re.compile(r"^#\s+(.+)$", re.MULTILINE)
+_DOC_EXT = {".md", ".mdx", ".markdown"}
+
+
+def _title_of(body: str, fallback: str) -> str:
+    m = _H1.search(body)
+    return m.group(1).strip() if m else fallback
+
+
+def _humanize(name: str) -> str:
+    return name.replace("-", " ").replace("_", " ").strip().title()
+
+
+class _DocsAdapter:
+    kind = "docs"
+
+    def _file_node(self, file: Path) -> SourceNode:
+        fm, body = split_frontmatter(file.read_text())
+        title = fm.get("title") or _title_of(body, _humanize(file.stem))
+        return SourceNode(
+            id=slugify(file.stem),
+            title=title,
+            content=body.strip(),
+            hint=fm.get("description"),
+        )
+
+    def _dir_node(self, directory: Path, node_id: str, title: str) -> SourceNode:
+        node = SourceNode(id=node_id, title=title, hint=f"Docs under {directory.name}/.")
+        for child in sorted(directory.iterdir(), key=lambda p: (p.is_file(), p.name)):
+            if child.is_dir():
+                sub = self._dir_node(child, slugify(child.name), _humanize(child.name))
+                if sub.children:
+                    node.children.append(sub)
+            elif child.suffix.lower() in _DOC_EXT:
+                node.children.append(self._file_node(child))
+        return node
+
+    def load(self, path: Path, *, source_id: str | None = None) -> SourceTree:
+        root_dir = Path(path)
+        sid = source_id or root_dir.name
+        root = self._dir_node(root_dir, "r", sid)
+        return SourceTree(source_id=sid, kind=self.kind, root=root)
+
+
+DocsAdapter = register_adapter(_DocsAdapter())

askfaro_progressive_context/build/adapters/memory.py

@@ -0,0 +1,56 @@
+"""memory adapter — a directory of one-fact markdown files with frontmatter.
+
+Each file is a leaf; `name`/`description` frontmatter become the title/hint and
+the body is the verbatim content. Files are grouped into tier-1 branches by
+`metadata.type` (e.g. user / feedback / project / reference) when present —
+matching a one-fact-per-file memory layout; generic to any such collection.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .._frontmatter import split_frontmatter
+from ..ir import SourceNode, SourceTree
+from .base import register_adapter, slugify
+
+
+class _MemoryAdapter:
+    kind = "memory"
+
+    def load(self, path: Path, *, source_id: str | None = None) -> SourceTree:
+        root_dir = Path(path)
+        sid = source_id or root_dir.name
+
+        groups: dict[str, list[SourceNode]] = {}
+        for md in sorted(root_dir.glob("*.md")):
+            if md.name.upper() == "MEMORY.md".upper():
+                continue  # the index, not a fact
+            fm, body = split_frontmatter(md.read_text())
+            meta = fm.get("metadata") if isinstance(fm.get("metadata"), dict) else {}
+            mtype = (meta or {}).get("type", "")
+            name = fm.get("name") or md.stem
+            leaf = SourceNode(
+                id=slugify(name),
+                title=name,
+                content=body.strip(),
+                hint=fm.get("description"),
+            )
+            groups.setdefault(mtype, []).append(leaf)
+
+        root = SourceNode(id="r", title=sid, hint="A memory store of discrete facts.")
+        if len(groups) <= 1:
+            for leaves in groups.values():
+                root.children.extend(leaves)
+        else:
+            for mtype, leaves in sorted(groups.items()):
+                if not mtype:
+                    root.children.extend(leaves)
+                    continue
+                root.children.append(
+                    SourceNode(id=slugify(mtype), title=mtype, hint=f"{mtype} memories.", children=leaves)
+                )
+        return SourceTree(source_id=sid, kind=self.kind, root=root)
+
+
+MemoryAdapter = register_adapter(_MemoryAdapter())

askfaro_progressive_context/build/adapters/skills.py

@@ -0,0 +1,62 @@
+"""skills adapter — a directory of skills, one markdown file per skill.
+
+Each skill file carries frontmatter (`name`, `description`, and optionally
+`when`/`when_to_use` and `category`); the body is the verbatim skill content.
+Skills are grouped into tier-1 branches by `category` when present. This is a
+generic skills layout — a host-side shim maps your skill source
+into it without this package importing the host app.
+
+Skill *selection* is exactly the agent-navigated case: a name+purpose+when
+manifest up front, the full skill body fetched on expand.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .._frontmatter import split_frontmatter
+from ..ir import SourceNode, SourceTree
+from .base import register_adapter, slugify
+
+
+class _SkillsAdapter:
+    kind = "skills"
+
+    def load(self, path: Path, *, source_id: str | None = None) -> SourceTree:
+        root_dir = Path(path)
+        sid = source_id or root_dir.name
+
+        groups: dict[str, list[SourceNode]] = {}
+        for md in sorted(root_dir.rglob("*.md")):
+            fm, body = split_frontmatter(md.read_text())
+            name = fm.get("name") or md.stem
+            when_hint = fm.get("when") or fm.get("when_to_use")
+            hint = fm.get("description")
+            if when_hint:
+                hint = f"{hint or ''}\nWhen to use: {when_hint}".strip()
+            category = fm.get("category", "")
+            leaf = SourceNode(
+                id=slugify(name),
+                title=name,
+                content=body.strip(),
+                hint=hint,
+                keywords=fm.get("keywords", []) if isinstance(fm.get("keywords"), list) else [],
+            )
+            groups.setdefault(category, []).append(leaf)
+
+        root = SourceNode(id="r", title=f"{sid} skills", hint="Reusable skills for performing tasks.")
+        if len(groups) <= 1:
+            for leaves in groups.values():
+                root.children.extend(leaves)
+        else:
+            for category, leaves in sorted(groups.items()):
+                if not category:
+                    root.children.extend(leaves)
+                    continue
+                root.children.append(
+                    SourceNode(id=slugify(category), title=category, hint=f"{category} skills.", children=leaves)
+                )
+        return SourceTree(source_id=sid, kind=self.kind, root=root)
+
+
+SkillsAdapter = register_adapter(_SkillsAdapter())

askfaro_progressive_context/build/adapters/tools.py

@@ -0,0 +1,82 @@
+"""tools adapter — a JSON file of tool/function schemas.
+
+Accepts an OpenAI-style list (`[{name, description, parameters}, ...]`), a
+`{"tools": [...]}` wrapper, or a `{name: schema}` mapping. Each tool becomes a
+leaf whose verbatim content is its full schema; the existing `description`
+becomes a descriptor hint. Tools are grouped into tier-1 branches by namespace
+(the part before the first `.`/`/`/`:`/`__` in the name) when present.
+
+This is the proven progressive-tool-disclosure pattern: a tiny name+purpose
+manifest up front, the full schema fetched on expand.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from ..ir import SourceNode, SourceTree
+from .base import register_adapter, slugify
+
+_NS = re.compile(r"[./:]|__")
+
+
+def _normalize(raw) -> list[dict]:
+    if isinstance(raw, dict) and "tools" in raw:
+        raw = raw["tools"]
+    if isinstance(raw, dict):
+        out = []
+        for name, schema in raw.items():
+            entry = dict(schema) if isinstance(schema, dict) else {"schema": schema}
+            entry.setdefault("name", name)
+            out.append(entry)
+        return out
+    return list(raw)
+
+
+def _name_of(tool: dict) -> str:
+    return tool.get("name") or tool.get("function", {}).get("name") or "tool"
+
+
+def _description_of(tool: dict) -> str | None:
+    return tool.get("description") or tool.get("function", {}).get("description")
+
+
+class _ToolsAdapter:
+    kind = "tools"
+
+    def load(self, path: Path, *, source_id: str | None = None) -> SourceTree:
+        tools = _normalize(json.loads(Path(path).read_text()))
+        sid = source_id or Path(path).stem
+
+        groups: dict[str, list[SourceNode]] = {}
+        for tool in tools:
+            name = _name_of(tool)
+            ns_match = _NS.split(name, 1)
+            ns = ns_match[0] if len(ns_match) > 1 else ""
+            leaf = SourceNode(
+                id=slugify(name),
+                title=name,
+                content=json.dumps(tool, indent=2, sort_keys=True),
+                format="json",
+                hint=_description_of(tool),
+                keywords=[t for t in _NS.split(name) if t],
+            )
+            groups.setdefault(ns, []).append(leaf)
+
+        root = SourceNode(id="r", title=f"{sid} tools", hint="Callable tools and their schemas.")
+        if len(groups) == 1 and "" in groups:
+            root.children = groups[""]
+        else:
+            for ns, leaves in sorted(groups.items()):
+                if ns == "":
+                    root.children.extend(leaves)
+                    continue
+                root.children.append(
+                    SourceNode(id=slugify(ns), title=ns, hint=f"Tools in the {ns} namespace.", children=leaves)
+                )
+        return SourceTree(source_id=sid, kind=self.kind, root=root)
+
+
+ToolsAdapter = register_adapter(_ToolsAdapter())

askfaro_progressive_context/build/compiler.py

@@ -0,0 +1,71 @@
+"""Compiler orchestration: SourceTree -> descriptors -> costs -> manifests."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..tokenizer import Tokenizer, heuristic_tokenizer
+from .cost import annotate
+from .descriptors import DescriptorModel, cache_from_manifest, generate_descriptors
+from .emit import build_manifest, to_llms_txt
+from .ir import SourceTree
+
+
+@dataclass
+class BuildResult:
+    source_id: str
+    manifests: dict[int, dict[str, Any]]  # budget -> manifest dict
+    llms_txt: str
+    stats: dict[str, Any] = field(default_factory=dict)
+
+
+def compile_source(
+    tree: SourceTree,
+    model: DescriptorModel,
+    budgets: list[int],
+    *,
+    tokenizer: Tokenizer | None = None,
+    contrastive: bool = True,
+    contrast_chunk: int = 8,
+    grade_threshold: float = 0.7,
+    max_repairs: int = 1,
+    max_workers: int = 1,
+    prior_manifest: dict | None = None,
+    generated_at: str | None = None,
+) -> BuildResult:
+    tokenizer = tokenizer or heuristic_tokenizer()
+    budgets = sorted(set(budgets))
+
+    cache = cache_from_manifest(prior_manifest) if prior_manifest else None
+    gen_stats: dict = {}
+    descriptors = generate_descriptors(
+        tree,
+        model,
+        contrastive=contrastive,
+        contrast_chunk=contrast_chunk,
+        grade_threshold=grade_threshold,
+        max_repairs=max_repairs,
+        max_workers=max_workers,
+        cache=cache,
+        _stats=gen_stats,
+    )
+    costs = annotate(tree.root, descriptors, tokenizer)
+
+    manifests = {
+        b: build_manifest(tree, descriptors, costs, b, siblings=budgets, generated_at=generated_at)
+        for b in budgets
+    }
+
+    nodes = list(tree.root.walk())
+    leaves = [n for n in nodes if n.is_leaf]
+    stats = {
+        "nodes": len(nodes),
+        "leaves": len(leaves),
+        "branches": len(nodes) - len(leaves),
+        "full_tokens": costs[tree.root.id].subtree_tokens,
+        "manifest_tokens": manifests[budgets[0]]["variant"]["manifest_tokens"],
+        "regenerated": gen_stats.get("regenerated", len(nodes)),
+        "reused": gen_stats.get("reused", 0),
+    }
+    return BuildResult(source_id=tree.source_id, manifests=manifests, llms_txt=to_llms_txt(tree, descriptors), stats=stats)

askfaro_progressive_context/build/cost.py

@@ -0,0 +1,47 @@
+"""Cost annotation — tokenize leaves, roll up subtree costs, hash for caching.
+
+`tokens` is the cost to expand a leaf's full content (0 for branches).
+`desc_tokens` is the cost of showing the node's descriptor in a frontier.
+`subtree_tokens` is the cost to expand everything beneath a node, rolled up
+bottom-up. `content_hash` lets a later build reuse descriptors for unchanged
+nodes (leaf = hash of content; branch = hash of child hashes).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..tokenizer import Tokenizer
+from .descriptors import Descriptor
+from .ir import SourceNode, content_hashes
+
+
+@dataclass
+class Cost:
+    tokens: int
+    desc_tokens: int
+    subtree_tokens: int
+    content_hash: str
+
+
+def annotate(root: SourceNode, descriptors: dict[str, Descriptor], tokenizer: Tokenizer) -> dict[str, Cost]:
+    hashes = content_hashes(root)
+    costs: dict[str, Cost] = {}
+
+    for node in root.post_order():
+        d = descriptors[node.id]
+        desc_text = " ".join(filter(None, [node.title, d.what, d.when, " ".join(d.keywords)]))
+        desc_tokens = tokenizer(desc_text)
+
+        if node.is_leaf:
+            tokens = tokenizer(node.content or "")
+            costs[node.id] = Cost(tokens, desc_tokens, tokens, hashes[node.id])
+        else:
+            child_costs = [costs[c.id] for c in node.children]
+            costs[node.id] = Cost(
+                tokens=0,
+                desc_tokens=desc_tokens,
+                subtree_tokens=sum(c.subtree_tokens for c in child_costs),
+                content_hash=hashes[node.id],
+            )
+    return costs