folio-cli 0.3.0__py3-none-any.whl

folio/config.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import copy
+import os
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Standalone tool checkout (…/Desktop/folio) — used only as a fallback for
+# editable installs; the shared index lives under ~/.folio by default.
+TOOL_ROOT = Path(__file__).resolve().parents[2]
+
+
+def default_home() -> Path:
+    """Shared index root (~/.folio). Computed at call time (never cache Path.home())."""
+    return Path.home() / ".folio"
+
+
+def package_dir() -> Path:
+    return Path(__file__).resolve().parent
+
+
+def default_config_text() -> str:
+    """Load bundled config.yaml (works for editable + installed wheels)."""
+    bundled = package_dir() / "assets" / "config.yaml"
+    if bundled.exists():
+        return bundled.read_text(encoding="utf-8")
+    legacy = TOOL_ROOT / "config.yaml"
+    if legacy.exists():
+        return legacy.read_text(encoding="utf-8")
+    try:
+        return resources.files("folio.assets").joinpath("config.yaml").read_text(
+            encoding="utf-8"
+        )
+    except Exception as e:  # noqa: BLE001
+        raise FileNotFoundError("Bundled folio config.yaml not found") from e
+
+
+def resolve_out_dir(explicit: Path | None = None) -> Path:
+    """Resolve shared knowledge index root.
+
+    Priority:
+    1. explicit --out
+    2. FOLIO_HOME env
+    3. ~/.folio
+    """
+    if explicit is not None:
+        return explicit.expanduser().resolve()
+    env = os.environ.get("FOLIO_HOME")
+    if env:
+        return Path(env).expanduser().resolve()
+    return default_home().resolve()
+
+
+def load_config(
+    *,
+    out_dir: Path | None = None,
+    config_path: Path | None = None,
+) -> dict[str, Any]:
+    if config_path is not None:
+        raw = config_path.read_text(encoding="utf-8")
+    else:
+        root = resolve_out_dir(out_dir)
+        local_cfg = root / "config.yaml"
+        raw = local_cfg.read_text(encoding="utf-8") if local_cfg.exists() else default_config_text()
+
+    cfg = yaml.safe_load(raw)
+    cfg = copy.deepcopy(cfg)
+    root = resolve_out_dir(out_dir)
+    paths = cfg.setdefault("paths", {})
+    paths["sources_dir"] = root / "sources"
+    paths["chroma_dir"] = root / "chroma"
+    paths["bm25_path"] = root / "bm25" / "index.pkl"
+    paths["manifest_path"] = root / "manifest.json"
+    paths["model_cache_dir"] = root / "model-cache"
+    paths["out_dir"] = root
+    cfg["_out_dir"] = root
+    return cfg
+
+
+def ensure_data_dirs(cfg: dict[str, Any]) -> None:
+    paths = cfg["paths"]
+    for key in ("sources_dir", "chroma_dir", "model_cache_dir"):
+        Path(paths[key]).mkdir(parents=True, exist_ok=True)
+    Path(paths["bm25_path"]).parent.mkdir(parents=True, exist_ok=True)
+    Path(paths["manifest_path"]).parent.mkdir(parents=True, exist_ok=True)
+
+
+def write_default_config(out_dir: Path | None = None) -> Path:
+    root = resolve_out_dir(out_dir)
+    root.mkdir(parents=True, exist_ok=True)
+    dest = root / "config.yaml"
+    if not dest.exists():
+        dest.write_text(default_config_text(), encoding="utf-8")
+    return dest

folio/distribute.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import tarfile
+import tempfile
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from .config import ensure_data_dirs, resolve_out_dir, write_default_config
+from .ingest import index_version_now, ingest
+from .sync import sync_sources
+
+# Files/dirs included in a portable index pack (query-ready without sources).
+_PACK_MEMBERS = ("chroma", "bm25", "manifest.json", "config.yaml")
+
+
+def _release_url(cfg: dict[str, Any]) -> str:
+    release = cfg.get("release") or {}
+    return (release.get("index_url") or "").strip()
+
+
+def pack_index(cfg: dict[str, Any], *, dest: Path | None = None) -> Path:
+    """Create a tarball of the shared index for GitHub Release upload."""
+    out = Path(cfg["paths"]["out_dir"])
+    manifest_path = Path(cfg["paths"]["manifest_path"])
+    if not manifest_path.exists():
+        raise FileNotFoundError(f"No index to pack. Run `folio build` first ({manifest_path})")
+
+    data = json.loads(manifest_path.read_text(encoding="utf-8"))
+    version = data.get("index_version") or index_version_now()
+    if dest is None:
+        dest = Path.cwd() / "dist" / f"folio-index-{version}.tar.gz"
+    dest = dest.expanduser().resolve()
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    with tarfile.open(dest, "w:gz") as tar:
+        for name in _PACK_MEMBERS:
+            path = out / name
+            if not path.exists():
+                raise FileNotFoundError(f"Missing pack member: {path}")
+            tar.add(path, arcname=name)
+
+    print(f"[pack] wrote {dest} (index_version={version})")
+    return dest
+
+
+def _extract_pack(archive: Path, dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(archive, "r:gz") as tar:
+        # Python 3.12+ supports filter=; keep compatible extract.
+        tar.extractall(dest)  # noqa: S202 — local/trusted release asset
+
+
+def bootstrap_index(
+    cfg: dict[str, Any],
+    *,
+    url: str | None = None,
+    archive: Path | None = None,
+    force: bool = False,
+) -> Path:
+    """Install a prebuilt index into FOLIO_HOME (~/.folio)."""
+    out = Path(cfg["paths"]["out_dir"])
+    manifest = Path(cfg["paths"]["manifest_path"])
+    if manifest.exists() and not force:
+        print(f"[bootstrap] index already present at {out} (use --force to replace)")
+        return out
+
+    ensure_data_dirs(cfg)
+    write_default_config(out)
+
+    if archive is not None:
+        src = archive.expanduser().resolve()
+        if not src.exists():
+            raise FileNotFoundError(src)
+        print(f"[bootstrap] extracting {src} → {out}")
+        _extract_into_out(src, out)
+        return out
+
+    resolved = (url or _release_url(cfg)).strip()
+    if not resolved:
+        raise RuntimeError(
+            "No release URL configured. Set release.index_url in config, "
+            "pass --url, or run `folio build` / `folio update --from-source`."
+        )
+
+    with tempfile.TemporaryDirectory(prefix="folio-bootstrap-") as tmp:
+        tmp_path = Path(tmp) / "folio-index.tar.gz"
+        print(f"[bootstrap] downloading {resolved}")
+        try:
+            urllib.request.urlretrieve(resolved, tmp_path)  # noqa: S310
+        except urllib.error.URLError as e:
+            raise RuntimeError(f"Failed to download index: {e}") from e
+        _extract_into_out(tmp_path, out)
+
+    print(f"[bootstrap] ready at {out}")
+    return out
+
+
+def _extract_into_out(archive: Path, out: Path) -> None:
+    with tempfile.TemporaryDirectory(prefix="folio-extract-") as tmp:
+        staging = Path(tmp) / "staging"
+        staging.mkdir()
+        _extract_pack(archive, staging)
+        for name in _PACK_MEMBERS:
+            src = staging / name
+            if not src.exists():
+                raise RuntimeError(f"Invalid index pack: missing {name}")
+            dest = out / name
+            if dest.exists():
+                if dest.is_dir():
+                    shutil.rmtree(dest)
+                else:
+                    dest.unlink()
+            if src.is_dir():
+                shutil.copytree(src, dest)
+            else:
+                shutil.copy2(src, dest)
+
+
+def update_index(
+    cfg: dict[str, Any],
+    *,
+    from_source: bool = False,
+    url: str | None = None,
+    force: bool = True,
+) -> int:
+    """Refresh the shared index: Release pack by default, or rebuild from sources."""
+    ensure_data_dirs(cfg)
+    write_default_config(cfg["paths"]["out_dir"])
+
+    if from_source:
+        print("[update] rebuilding from curated sources…")
+        sync_sources(cfg, force=False)
+        ingest(cfg, rebuild=True)
+        return 0
+
+    try:
+        bootstrap_index(cfg, url=url, force=force)
+        return 0
+    except RuntimeError as e:
+        print(f"[update] bootstrap unavailable: {e}", file=sys.stderr)
+        print("[update] falling back to --from-source build…", file=sys.stderr)
+        sync_sources(cfg, force=False)
+        ingest(cfg, rebuild=True)
+        return 0
+
+
+def migrate_legacy_index(legacy: Path | None = None) -> Path | None:
+    """One-shot: move Desktop/aieng/aieng-out → ~/.folio if needed."""
+    dest = resolve_out_dir()
+    if (dest / "manifest.json").exists():
+        return dest
+
+    candidates = []
+    if legacy:
+        candidates.append(legacy.expanduser())
+    candidates.extend(
+        [
+            Path.home() / "Desktop" / "aieng" / "aieng-out",
+            Path.home() / "Desktop" / "folio" / "aieng-out",
+            Path.home() / "Desktop" / "folio" / "folio-out",
+            Path(__file__).resolve().parents[2] / "aieng-out",
+        ]
+    )
+    for src in candidates:
+        if (src / "manifest.json").exists():
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            if dest.exists():
+                shutil.rmtree(dest)
+            shutil.move(str(src), str(dest))
+            print(f"[migrate] moved {src} → {dest}")
+            return dest
+    return None

folio/ingest.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+from tqdm import tqdm
+
+from . import __version__ as cli_version
+from .chunk import Chunk, chunk_file
+from .config import ensure_data_dirs
+from .store import build_bm25, reset_collection, save_bm25, upsert_chunks
+from .sync import iter_markdown
+
+
+def index_version_now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y.%m.%d")
+
+
+def ingest(cfg: dict[str, Any], *, rebuild: bool = True) -> dict[str, Any]:
+    ensure_data_dirs(cfg)
+    files = iter_markdown(cfg)
+    if not files:
+        raise RuntimeError("No markdown files found. Run sync first.")
+
+    chunk_cfg = cfg["chunking"]
+    all_chunks: list[Chunk] = []
+    sources_dir = cfg["paths"]["sources_dir"]
+
+    for source_id, path in tqdm(files, desc="chunking"):
+        chunks = chunk_file(
+            path,
+            source_id=source_id,
+            sources_root=sources_dir / source_id,
+            max_chars=int(chunk_cfg["max_chars"]),
+            overlap_chars=int(chunk_cfg["overlap_chars"]),
+            min_chars=int(chunk_cfg["min_chars"]),
+        )
+        all_chunks.extend(chunks)
+
+    collection = reset_collection(cfg) if rebuild else reset_collection(cfg)
+    n = upsert_chunks(collection, all_chunks, batch_size=int(cfg["embedding"]["batch_size"]))
+    bm25, ids, chunks = build_bm25(all_chunks)
+    save_bm25(cfg["paths"]["bm25_path"], bm25, ids, chunks)
+
+    index_version = index_version_now()
+    manifest = {
+        "index_version": index_version,
+        "cli_version": cli_version,
+        "ingested_at": datetime.now(timezone.utc).isoformat(),
+        "files": len(files),
+        "chunks": n,
+        "embedding_model": cfg["embedding"]["model"],
+        "collection": cfg["collection"]["name"],
+        "sources": [
+            {
+                "id": s["id"],
+                "repo": s["repo"],
+                "sparse_path": s["sparse_path"],
+            }
+            for s in cfg["sources"]
+        ],
+        "file_list": [
+            {"source_id": sid, "path": str(p.relative_to(sources_dir))}
+            for sid, p in files
+        ],
+    }
+    cfg["paths"]["manifest_path"].write_text(
+        json.dumps(manifest, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    print(
+        f"[ingest] index_version={index_version} files={len(files)} "
+        f"chunks={n} chroma={cfg['paths']['chroma_dir']}"
+    )
+    return manifest

folio/install.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+from .config import package_dir
+
+TOOL = "folio"
+_CURSOR_RULE_PATH = Path(".cursor") / "rules" / "folio.mdc"
+_AGENTS_MD_MARKER = "# folio"
+
+_CURSOR_RULE = """\
+---
+description: folio curated AI / MLOps / infra knowledge RAG
+alwaysApply: true
+---
+
+This project uses the standalone **folio** tool (shared curricula RAG on the machine).
+
+**MANDATORY: Before deciding or implementing AI engineering / MLOps / RAG / agent / GPU / AI infra / related AI architecture, you MUST run folio first:**
+- `folio query "<question>"` — hybrid retrieve grounded citations
+- `folio query "<question>" --json` — machine-readable hits for agents
+- `folio status` — check whether the shared index exists (`~/.folio` or `$FOLIO_HOME`)
+
+This applies to YOU and to every subagent you spawn for those topics. Do not skip folio because the topic “feels familiar” — the index surfaces curriculum trade-offs (hybrid RAG vs RAGless, CRAG, RRF, etc.) that training memory alone will miss.
+
+**Token rule (strict):** Do **not** WebFetch, clone, or paste the curated curriculum GitHub repos into chat context. Those sources are already indexed. Retrieve with `folio query` only. If the index is missing, tell the user to run `folio bootstrap` or `folio build` / `folio update` — do not dump raw markdown into the conversation.
+
+Only skip folio when:
+1. The task is ordinary product/codebase work with no AI / MLOps / AI-infra architecture decision
+2. The shared index is missing — then instruct `folio bootstrap` (or `folio build`) once
+
+After `folio query`, ground choices in returned `source_id` + `rel_path` + `heading_path` citations.
+"""
+
+_CLAUDE_REGISTRATION = """\
+# folio
+
+Standalone tool: local hybrid RAG for AI engineering / MLOps / AI infra decisions.
+Shared index: `~/.folio/` (override with `FOLIO_HOME`).
+
+Before AI-eng / MLOps / RAG / agent / AI-infra work, run:
+`folio query "<question>" --json`
+
+Do **not** fetch the curriculum GitHub repos into chat — query the local index only.
+
+Skill: `.claude/skills/folio/SKILL.md` / `.agents/skills/folio/SKILL.md`.
+If index missing: `folio bootstrap` or `folio build`. Refresh: `folio update`.
+"""
+
+_PLATFORM_SKILL_DST = {
+    "claude": Path(".claude") / "skills" / "folio" / "SKILL.md",
+    "codex": Path(".codex") / "skills" / "folio" / "SKILL.md",
+    "agents": Path(".agents") / "skills" / "folio" / "SKILL.md",
+    "cursor": None,
+}
+
+
+def _skill_src() -> Path:
+    src = package_dir() / "assets" / "SKILL.md"
+    if not src.exists():
+        raise FileNotFoundError(f"Missing packaged skill: {src}")
+    return src
+
+
+def _write_skill(dst: Path) -> None:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    text = _skill_src().read_text(encoding="utf-8")
+    if dst.exists() and dst.read_text(encoding="utf-8") == text:
+        print(f"folio skill already configured at {dst.resolve()} (no change)")
+        return
+    action = "updated" if dst.exists() else "written"
+    dst.write_text(text, encoding="utf-8")
+    print(f"folio skill {action} at {dst.resolve()}")
+
+
+def _cursor_install(project_dir: Path) -> None:
+    rule_path = project_dir / _CURSOR_RULE_PATH
+    rule_path.parent.mkdir(parents=True, exist_ok=True)
+    if rule_path.exists() and rule_path.read_text(encoding="utf-8") == _CURSOR_RULE:
+        print(f"folio Cursor rule at {rule_path.resolve()} already configured (no change)")
+        return
+    action = "updated" if rule_path.exists() else "written"
+    rule_path.write_text(_CURSOR_RULE, encoding="utf-8")
+    print(f"folio Cursor rule {action} at {rule_path.resolve()}")
+    print("Cursor will retrieve-first via folio for AI / MLOps / infra tasks.")
+    print("Shared index: ~/.folio — run `folio bootstrap` or `folio build` if missing.")
+
+
+def _cursor_uninstall(project_dir: Path) -> None:
+    rule_path = project_dir / _CURSOR_RULE_PATH
+    # Also remove legacy aieng rule if present
+    legacy = project_dir / ".cursor" / "rules" / "aieng.mdc"
+    removed_any = False
+    for path in (rule_path, legacy):
+        if path.exists():
+            path.unlink()
+            print(f"removed {path.resolve()}")
+            removed_any = True
+    if not removed_any:
+        print("No folio Cursor rule found — nothing to do")
+
+
+def _replace_or_append_section(content: str, marker: str, section: str) -> str:
+    lines = content.splitlines()
+    start = None
+    for i, line in enumerate(lines):
+        if line.strip() == marker:
+            start = i
+            break
+    if start is None:
+        # Drop legacy aieng section if present
+        content = _strip_marker_section(content, "# aieng")
+        body = content.rstrip()
+        if body:
+            return body + "\n\n" + section.strip() + "\n"
+        return section.strip() + "\n"
+
+    end = len(lines)
+    for j in range(start + 1, len(lines)):
+        if lines[j].startswith("# "):
+            end = j
+            break
+    new_lines = lines[:start] + section.strip().splitlines() + ([""] if end < len(lines) else []) + lines[end:]
+    return "\n".join(new_lines).rstrip() + "\n"
+
+
+def _strip_marker_section(content: str, marker: str) -> str:
+    lines = content.splitlines()
+    start = None
+    for i, line in enumerate(lines):
+        if line.strip() == marker:
+            start = i
+            break
+    if start is None:
+        return content
+    end = len(lines)
+    for j in range(start + 1, len(lines)):
+        if lines[j].startswith("# "):
+            end = j
+            break
+    new_lines = lines[:start] + lines[end:]
+    return "\n".join(new_lines).rstrip() + ("\n" if new_lines else "")
+
+
+def _claude_md_register(project_dir: Path | None, user_global: bool) -> None:
+    if user_global:
+        target = Path.home() / ".claude" / "CLAUDE.md"
+    else:
+        assert project_dir is not None
+        target = project_dir / "CLAUDE.md"
+    try:
+        existed = target.exists()
+        content = target.read_text(encoding="utf-8") if existed else ""
+        content = _strip_marker_section(content, "# aieng")
+        new_content = _replace_or_append_section(content, _AGENTS_MD_MARKER, _CLAUDE_REGISTRATION)
+        if existed and new_content == content:
+            print(f"folio already registered in {target} (no change)")
+            return
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(new_content, encoding="utf-8")
+        print(f"folio registered in {target.resolve()}")
+    except OSError as exc:
+        print(f"skipped CLAUDE.md registration: {exc}", file=sys.stderr)
+
+
+def _agents_md_register(project_dir: Path) -> None:
+    target = project_dir / "AGENTS.md"
+    try:
+        existed = target.exists()
+        content = target.read_text(encoding="utf-8") if existed else ""
+        content = _strip_marker_section(content, "# aieng")
+        new_content = _replace_or_append_section(content, _AGENTS_MD_MARKER, _CLAUDE_REGISTRATION)
+        if existed and new_content == content:
+            print(f"folio already registered in {target} (no change)")
+            return
+        target.write_text(new_content, encoding="utf-8")
+        print(f"folio registered in {target.resolve()}")
+    except OSError as exc:
+        print(f"skipped AGENTS.md registration: {exc}", file=sys.stderr)
+
+
+def _skill_dst_for(platform: str, *, project: bool, project_dir: Path) -> Path | None:
+    rel = _PLATFORM_SKILL_DST.get(platform)
+    if rel is None:
+        return None
+    if project:
+        return project_dir / rel
+    return Path.home() / rel
+
+
+def install(
+    *,
+    platform: str = "cursor",
+    project: bool = False,
+    project_dir: Path | None = None,
+) -> int:
+    project_dir = (project_dir or Path(".")).resolve()
+    platform = platform.lower().strip()
+    if platform == "skills":
+        platform = "agents"
+
+    known = {"cursor", "claude", "codex", "agents"}
+    if platform not in known:
+        print(
+            f"error: unknown platform '{platform}'. Choose from: {', '.join(sorted(known))}",
+            file=sys.stderr,
+        )
+        return 2
+
+    if platform == "cursor":
+        _cursor_install(project_dir)
+        if project:
+            _write_skill(project_dir / ".agents" / "skills" / "folio" / "SKILL.md")
+            # remove legacy skill dir
+            legacy_skill = project_dir / ".agents" / "skills" / "aieng"
+            if legacy_skill.exists():
+                shutil.rmtree(legacy_skill)
+                print(f"removed legacy {legacy_skill}")
+            _agents_md_register(project_dir)
+        return 0
+
+    dst = _skill_dst_for(platform, project=project, project_dir=project_dir)
+    assert dst is not None
+    _write_skill(dst)
+
+    if platform == "claude":
+        _claude_md_register(project_dir if project else None, user_global=not project)
+    if platform == "agents" and project:
+        _agents_md_register(project_dir)
+
+    print()
+    print("Next: `folio bootstrap` (or `folio build`), then `folio query \"...\"`.")
+    return 0
+
+
+def uninstall(
+    *,
+    platform: str | None = None,
+    project: bool = False,
+    project_dir: Path | None = None,
+    purge: bool = False,
+) -> int:
+    project_dir = (project_dir or Path(".")).resolve()
+    platforms = (
+        [platform.lower()]
+        if platform
+        else ["cursor", "claude", "codex", "agents"]
+    )
+    for p in platforms:
+        if p == "skills":
+            p = "agents"
+        if p == "cursor":
+            _cursor_uninstall(project_dir)
+            for name in ("folio", "aieng"):
+                skill = project_dir / ".agents" / "skills" / name
+                if skill.exists():
+                    shutil.rmtree(skill)
+                    print(f"removed {skill}")
+            continue
+        dst = _skill_dst_for(p, project=project, project_dir=project_dir)
+        if dst and dst.exists():
+            skill_dir = dst.parent
+            shutil.rmtree(skill_dir)
+            print(f"removed {skill_dir}")
+        elif not project and p in _PLATFORM_SKILL_DST and _PLATFORM_SKILL_DST[p]:
+            user_dst = Path.home() / _PLATFORM_SKILL_DST[p]  # type: ignore[operator]
+            if user_dst.exists():
+                shutil.rmtree(user_dst.parent)
+                print(f"removed {user_dst.parent}")
+
+    if purge:
+        for name in ("folio-out", "aieng-out"):
+            out = project_dir / name
+            if out.exists():
+                shutil.rmtree(out)
+                print(f"purged {out}")
+    return 0