folio-cli 0.3.0__py3-none-any.whl

folio/__init__.py

@@ -0,0 +1,3 @@
+"""folio — local hybrid RAG over curated AI / MLOps / data curricula (graphify-style)."""
+
+__version__ = "0.3.0"

folio/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

folio/assets/SKILL.md

@@ -0,0 +1,75 @@
+---
+name: folio
+description: "Use for any AI engineering / MLOps / RAG / agent / GPU / AI infra / related AI architecture decision. Queries the local hybrid RAG over curated curricula (~/.folio). Treat those design questions as a folio query first."
+---
+
+# /folio
+
+Local hybrid RAG over curated AI-engineer / MLOps / infra markdown curricula.
+Persistent vector + BM25 index in `~/.folio/` (or `$FOLIO_HOME`). No API key required.
+
+## Usage
+
+```
+/folio                                              # orient + ensure index
+/folio bootstrap                                    # download prebuilt index (preferred)
+/folio build                                        # sync curated sources + ingest
+/folio update                                       # refresh from maintainer Release
+/folio update --from-source                         # rebuild from curated git sources
+/folio query "<question>"                           # hybrid retrieve (BM25 + dense + RRF + CRAG)
+/folio query "<question>" --json                    # machine-readable hits for agents
+/folio query "<question>" -k 10                     # more chunks
+/folio query "<question>" --rerank                  # optional cross-encoder
+/folio status                                       # CLI + index health
+/folio install --project --platform cursor          # wire this repo's assistant
+```
+
+## What folio is for
+
+Ground AI / MLOps / AI-infra decisions in retrieved citations from curated curricula
+(currently senior AI revision guide + AI infra lessons; more domains later).
+
+**When to query:** RAG, embeddings, vector DBs, agents, training/GPU, model optimization,
+feature stores, GitOps, SRE/SLO for ML, AI infra, or related senior AI trade-offs.
+
+**When not to:** ordinary product code exploration — use graphify / Read / Grep for that.
+
+## Token rule (strict)
+
+Do **not** WebFetch, clone, or paste the curriculum GitHub repos into chat.
+They are already indexed. Only `folio query`. If the index is missing, tell the user
+to run `folio bootstrap` or `folio build` — never dump raw curriculum markdown into context.
+
+## What You Must Do When Invoked
+
+### Fast path — index exists
+
+If `~/.folio/manifest.json` exists (or `$FOLIO_HOME/manifest.json`):
+
+```bash
+folio query "<question>" --json
+```
+
+Ground your answer **only** in returned citations (`source_id`, `rel_path`, `heading_path`, `text`).
+If scores are weak / empty, broaden terms and re-query once before inventing architecture.
+
+### No index yet
+
+```bash
+folio bootstrap   # or: folio build
+folio query "<question>" --json
+```
+
+### Install into a project
+
+```bash
+folio install --project --platform cursor
+folio bootstrap   # once per machine
+```
+
+## Rules for answers
+
+1. Prefer patterns that appear in retrieved chunks; cite `source_id` + file + heading.
+2. When sources conflict (e.g. hybrid RAG vs RAGless), state the trade-off and pick for *this* constraint.
+3. Do not dump entire markdown files when retrieval exists.
+4. Never skip retrieval for “quick” AI architecture guesses when the index is present.

folio/assets/__init__.py

@@ -0,0 +1 @@
+"""Packaged static assets (config, skill templates)."""

folio/assets/config.yaml

@@ -0,0 +1,77 @@
+# Hybrid RAG — curated AI eng / MLOps / AI infra curricula
+# BM25 + dense → RRF → CRAG widen → optional cross-encoder rerank
+
+sources:
+  # --- already trusted (user seeds) ---
+  - id: senior-ai-revision
+    repo: https://github.com/RavSinghChandan/02-senior-ai-engineer-study-path.git
+    sparse_path: revision-1
+    branch: main
+  - id: ai-infra-lessons
+    repo: https://github.com/ai-infra-curriculum/ai-infra-senior-engineer-learning.git
+    sparse_path: lessons
+    branch: main
+
+  # --- same curriculum factory (MD lessons, senior depth) ---
+  - id: agentic-ai-developer
+    repo: https://github.com/ai-engineering-curriculum/agentic-ai-developer-learning.git
+    sparse_path: lessons
+    branch: main
+  - id: agentic-ai-engineer
+    repo: https://github.com/ai-engineering-curriculum/agentic-ai-engineer-learning.git
+    sparse_path: lessons
+    branch: main
+  - id: senior-agentic-ai-engineer
+    repo: https://github.com/ai-engineering-curriculum/senior-agentic-ai-engineer-learning.git
+    sparse_path: lessons
+    branch: main
+  - id: applied-ai-engineer
+    repo: https://github.com/ai-engineering-curriculum/applied-ai-engineer-learning.git
+    sparse_path: lessons
+    branch: main
+  - id: ai-eval-engineer
+    repo: https://github.com/ai-engineering-curriculum/ai-eval-engineer-learning.git
+    sparse_path: lessons
+    branch: main
+  - id: agentic-systems-architect
+    repo: https://github.com/ai-engineering-curriculum/agentic-systems-architect-learning.git
+    sparse_path: lessons
+    branch: main
+
+  # --- classic MLOps course (markdown-heavy modules) ---
+  - id: mlops-zoomcamp
+    repo: https://github.com/DataTalksClub/mlops-zoomcamp.git
+    sparse_path: .
+    branch: main
+
+release:
+  # Filled after first GitHub Release upload of folio-index.tar.gz
+  index_url: "https://github.com/Iman-Sharei/folio/releases/latest/download/folio-index.tar.gz"
+
+paths:
+  sources_dir: data/sources
+  chroma_dir: data/chroma
+  bm25_path: data/bm25/index.pkl
+  manifest_path: data/manifest.json
+
+chunking:
+  max_chars: 1200
+  overlap_chars: 150
+  min_chars: 80
+
+embedding:
+  model: onnx-minilm-l6-v2
+  batch_size: 64
+
+collection:
+  name: folio_knowledge
+
+retrieval:
+  dense_k: 20
+  bm25_k: 20
+  final_k: 8
+  rrf_k: 60
+  crag_min_best_score: 0.012
+  crag_widen_factor: 2
+  reranker_model: cross-encoder/ms-marco-MiniLM-L-6-v2
+  confidence_floor: 0.008

folio/chunk.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
+
+
+@dataclass
+class Chunk:
+    chunk_id: str
+    text: str
+    source_id: str
+    rel_path: str
+    heading_path: str
+    chunk_index: int
+    content_hash: str
+    char_count: int
+
+    def to_metadata(self) -> dict:
+        meta = asdict(self)
+        # Chroma metadata values must be str/int/float/bool
+        meta.pop("text")
+        return meta
+
+
+def _file_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
+
+
+def _split_oversized(text: str, max_chars: int, overlap: int) -> list[str]:
+    if len(text) <= max_chars:
+        return [text]
+    parts: list[str] = []
+    paragraphs = re.split(r"\n\s*\n", text)
+    buf = ""
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+        candidate = f"{buf}\n\n{para}".strip() if buf else para
+        if len(candidate) <= max_chars:
+            buf = candidate
+            continue
+        if buf:
+            parts.append(buf)
+        if len(para) <= max_chars:
+            buf = para
+        else:
+            start = 0
+            while start < len(para):
+                end = min(start + max_chars, len(para))
+                parts.append(para[start:end])
+                if end >= len(para):
+                    break
+                start = max(0, end - overlap)
+            buf = ""
+    if buf:
+        parts.append(buf)
+
+    if overlap <= 0 or len(parts) <= 1:
+        return parts
+
+    # Soft overlap between adjacent oversized splits already handled for long paras;
+    # stitch short tails if needed by overlapping previous suffix.
+    overlapped: list[str] = [parts[0]]
+    for i in range(1, len(parts)):
+        prev = parts[i - 1]
+        prefix = prev[-overlap:] if len(prev) > overlap else prev
+        nxt = parts[i]
+        if not nxt.startswith(prefix[:40]):
+            overlapped.append(f"{prefix}\n{nxt}".strip())
+        else:
+            overlapped.append(nxt)
+    return overlapped
+
+
+def chunk_markdown(
+    text: str,
+    *,
+    source_id: str,
+    rel_path: str,
+    max_chars: int = 1200,
+    overlap_chars: int = 150,
+    min_chars: int = 80,
+) -> list[Chunk]:
+    """Structure-aware chunking: split on markdown headings, then size-guard."""
+    content_hash = _file_hash(text)
+    sections: list[tuple[str, str]] = []
+    matches = list(HEADING_RE.finditer(text))
+
+    if not matches:
+        sections.append(("", text.strip()))
+    else:
+        # Preface before first heading
+        preface = text[: matches[0].start()].strip()
+        if preface:
+            sections.append(("", preface))
+
+        stack: list[tuple[int, str]] = []
+        for i, m in enumerate(matches):
+            level = len(m.group(1))
+            title = m.group(2).strip()
+            while stack and stack[-1][0] >= level:
+                stack.pop()
+            stack.append((level, title))
+            heading_path = " > ".join(t for _, t in stack)
+            start = m.start()
+            end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+            body = text[start:end].strip()
+            sections.append((heading_path, body))
+
+    raw_chunks: list[tuple[str, str]] = []
+    for heading_path, body in sections:
+        for piece in _split_oversized(body, max_chars, overlap_chars):
+            piece = piece.strip()
+            if len(piece) < min_chars:
+                continue
+            raw_chunks.append((heading_path, piece))
+
+    chunks: list[Chunk] = []
+    for idx, (heading_path, piece) in enumerate(raw_chunks):
+        digest = hashlib.sha256(
+            f"{source_id}:{rel_path}:{idx}:{piece}".encode("utf-8")
+        ).hexdigest()[:24]
+        chunks.append(
+            Chunk(
+                chunk_id=f"{source_id}:{digest}",
+                text=piece,
+                source_id=source_id,
+                rel_path=rel_path,
+                heading_path=heading_path or "(preamble)",
+                chunk_index=idx,
+                content_hash=content_hash,
+                char_count=len(piece),
+            )
+        )
+    return chunks
+
+
+def chunk_file(
+    path: Path,
+    *,
+    source_id: str,
+    sources_root: Path,
+    max_chars: int,
+    overlap_chars: int,
+    min_chars: int,
+) -> list[Chunk]:
+    text = path.read_text(encoding="utf-8", errors="replace")
+    try:
+        rel = str(path.relative_to(sources_root))
+    except ValueError:
+        rel = path.name
+    return chunk_markdown(
+        text,
+        source_id=source_id,
+        rel_path=rel,
+        max_chars=max_chars,
+        overlap_chars=overlap_chars,
+        min_chars=min_chars,
+    )

folio/cli.py

@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from importlib.metadata import PackageNotFoundError, version as pkg_version
+from pathlib import Path
+
+from . import __version__
+from .config import ensure_data_dirs, load_config, resolve_out_dir, write_default_config
+from .distribute import bootstrap_index, migrate_legacy_index, pack_index, update_index
+from .ingest import ingest
+from .install import install as install_skill
+from .install import uninstall as uninstall_skill
+from .retrieve import format_context, retrieve
+from .sync import sync_sources
+
+
+def _cli_version() -> str:
+    try:
+        return pkg_version("folio")
+    except PackageNotFoundError:
+        return __version__
+
+
+def _cfg(args: argparse.Namespace) -> dict:
+    out = getattr(args, "out", None)
+    return load_config(out_dir=out)
+
+
+def _add_out_arg(p: argparse.ArgumentParser) -> None:
+    p.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="Index root (default: ~/.folio or $FOLIO_HOME)",
+    )
+
+
+def cmd_sync(args: argparse.Namespace) -> int:
+    cfg = _cfg(args)
+    ensure_data_dirs(cfg)
+    write_default_config(cfg["paths"]["out_dir"])
+    sync_sources(cfg, force=args.force)
+    return 0
+
+
+def cmd_build(args: argparse.Namespace) -> int:
+    """Sync curated sources + ingest into the shared index."""
+    cfg = _cfg(args)
+    ensure_data_dirs(cfg)
+    write_default_config(cfg["paths"]["out_dir"])
+    sync_sources(cfg, force=False)
+    ingest(cfg, rebuild=True)
+    return 0
+
+
+def cmd_ingest(args: argparse.Namespace) -> int:
+    cfg = _cfg(args)
+    ensure_data_dirs(cfg)
+    if args.sync_first:
+        sync_sources(cfg, force=False)
+    ingest(cfg, rebuild=True)
+    return 0
+
+
+def cmd_bootstrap(args: argparse.Namespace) -> int:
+    migrate_legacy_index()
+    cfg = _cfg(args)
+    bootstrap_index(
+        cfg,
+        url=args.url,
+        archive=args.archive,
+        force=args.force,
+    )
+    return 0
+
+
+def cmd_update(args: argparse.Namespace) -> int:
+    """Maintainer-published index refresh (Release), or rebuild from sources."""
+    cfg = _cfg(args)
+    return update_index(
+        cfg,
+        from_source=args.from_source,
+        url=args.url,
+        force=True,
+    )
+
+
+def cmd_pack(args: argparse.Namespace) -> int:
+    cfg = _cfg(args)
+    pack_index(cfg, dest=args.dest)
+    return 0
+
+
+def cmd_query(args: argparse.Namespace) -> int:
+    migrate_legacy_index()
+    cfg = _cfg(args)
+    manifest = Path(cfg["paths"]["manifest_path"])
+    if not manifest.exists():
+        print(
+            "error: no index found. Run `folio bootstrap` or `folio build` first "
+            f"(expected {manifest})",
+            file=sys.stderr,
+        )
+        return 2
+    hits = retrieve(
+        cfg,
+        args.query,
+        final_k=args.k,
+        use_rerank=args.rerank,
+        source_filter=args.source,
+    )
+    if args.json:
+        payload = [
+            {
+                "chunk_id": h.chunk_id,
+                "score": h.score,
+                "source_id": h.source_id,
+                "rel_path": h.rel_path,
+                "heading_path": h.heading_path,
+                "dense_rank": h.dense_rank,
+                "bm25_rank": h.bm25_rank,
+                "text": h.text,
+            }
+            for h in hits
+        ]
+        print(json.dumps(payload, indent=2, ensure_ascii=False))
+    else:
+        if not hits:
+            print("No hits above confidence floor. Try a broader query or update the index.")
+            return 1
+        print(format_context(hits))
+        print("\n# decision hint: ground architecture choices in the citations above.")
+    return 0
+
+
+def cmd_status(args: argparse.Namespace) -> int:
+    migrate_legacy_index()
+    cfg = _cfg(args)
+    out = cfg["paths"]["out_dir"]
+    manifest = cfg["paths"]["manifest_path"]
+    print(f"cli:      folio {_cli_version()}")
+    print(f"out:      {out}")
+    print(f"chroma:   {cfg['paths']['chroma_dir']} exists={Path(cfg['paths']['chroma_dir']).exists()}")
+    print(f"bm25:     {cfg['paths']['bm25_path']} exists={Path(cfg['paths']['bm25_path']).exists()}")
+    print(f"manifest: {manifest} exists={manifest.exists()}")
+    if manifest.exists():
+        data = json.loads(Path(manifest).read_text(encoding="utf-8"))
+        keys = (
+            "index_version",
+            "ingested_at",
+            "files",
+            "chunks",
+            "embedding_model",
+            "cli_version",
+        )
+        print(json.dumps({k: data[k] for k in keys if k in data}, indent=2))
+    return 0
+
+
+def cmd_install(args: argparse.Namespace) -> int:
+    return install_skill(
+        platform=args.platform,
+        project=args.project,
+        project_dir=Path(args.dir).resolve() if args.dir else Path.cwd(),
+    )
+
+
+def cmd_uninstall(args: argparse.Namespace) -> int:
+    return uninstall_skill(
+        platform=args.platform,
+        project=args.project,
+        project_dir=Path(args.dir).resolve() if args.dir else Path.cwd(),
+        purge=args.purge,
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="folio",
+        description=(
+            "Local hybrid RAG over curated AI / MLOps / infra curricula "
+            "(graphify-style CLI for any assistant)"
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"folio {_cli_version()}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_sync = sub.add_parser("sync", help="Sparse-clone / update curated GitHub markdown sources")
+    _add_out_arg(p_sync)
+    p_sync.add_argument("--force", action="store_true")
+    p_sync.set_defaults(func=cmd_sync)
+
+    p_build = sub.add_parser("build", help="Sync + chunk + embed + index from curated sources")
+    _add_out_arg(p_build)
+    p_build.set_defaults(func=cmd_build)
+
+    p_ingest = sub.add_parser("ingest", help="Chunk + embed + index (no sync unless --sync-first)")
+    _add_out_arg(p_ingest)
+    p_ingest.add_argument("--sync-first", action="store_true")
+    p_ingest.set_defaults(func=cmd_ingest)
+
+    p_bootstrap = sub.add_parser(
+        "bootstrap",
+        help="Download/install a prebuilt index into ~/.folio (or $FOLIO_HOME)",
+    )
+    _add_out_arg(p_bootstrap)
+    p_bootstrap.add_argument("--url", default=None, help="Override release.index_url")
+    p_bootstrap.add_argument(
+        "--archive",
+        type=Path,
+        default=None,
+        help="Install from a local folio-index-*.tar.gz instead of URL",
+    )
+    p_bootstrap.add_argument("--force", action="store_true", help="Replace existing index")
+    p_bootstrap.set_defaults(func=cmd_bootstrap)
+
+    p_update = sub.add_parser(
+        "update",
+        help="Refresh index from maintainer Release (default) or rebuild with --from-source",
+    )
+    _add_out_arg(p_update)
+    p_update.add_argument("--url", default=None, help="Override release.index_url")
+    p_update.add_argument(
+        "--from-source",
+        action="store_true",
+        help="Rebuild by syncing curated repos + embedding (slow)",
+    )
+    p_update.set_defaults(func=cmd_update)
+
+    p_pack = sub.add_parser(
+        "pack",
+        help="Maintainer: pack ~/.folio index into dist/folio-index-*.tar.gz for a Release",
+    )
+    _add_out_arg(p_pack)
+    p_pack.add_argument(
+        "--dest",
+        type=Path,
+        default=None,
+        help="Output tarball path (default: dist/folio-index-<version>.tar.gz)",
+    )
+    p_pack.set_defaults(func=cmd_pack)
+
+    p_query = sub.add_parser(
+        "query",
+        help="Hybrid retrieve for AI-eng / MLOps / AI infra / related AI questions",
+    )
+    _add_out_arg(p_query)
+    p_query.add_argument("query", help="Natural language task or question")
+    p_query.add_argument("-k", type=int, default=None)
+    p_query.add_argument("--rerank", action="store_true")
+    p_query.add_argument("--source", default=None)
+    p_query.add_argument("--json", action="store_true")
+    p_query.set_defaults(func=cmd_query)
+
+    p_status = sub.add_parser("status", help="Show CLI + index status")
+    _add_out_arg(p_status)
+    p_status.set_defaults(func=cmd_status)
+
+    p_install = sub.add_parser(
+        "install",
+        help="Register skill/rules with an AI assistant (cursor|claude|codex|agents)",
+    )
+    p_install.add_argument(
+        "--platform",
+        default="cursor",
+        help="cursor|claude|codex|agents (default: cursor)",
+    )
+    p_install.add_argument(
+        "--project",
+        action="store_true",
+        help="Write into the current repo (commit-friendly), like graphify --project",
+    )
+    p_install.add_argument("--dir", default=".", help="Project directory (default: cwd)")
+    p_install.set_defaults(func=cmd_install)
+
+    p_uninstall = sub.add_parser("uninstall", help="Remove folio assistant registration")
+    p_uninstall.add_argument("--platform", default=None)
+    p_uninstall.add_argument("--project", action="store_true")
+    p_uninstall.add_argument("--dir", default=".")
+    p_uninstall.add_argument(
+        "--purge",
+        action="store_true",
+        help="Also delete a leftover ./folio-out/ in the project (index normally lives in ~/.folio)",
+    )
+    p_uninstall.set_defaults(func=cmd_uninstall)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    # Patch Chroma ONNX cache early (before collection open in subcommands).
+    out_hint = resolve_out_dir()
+    from .store import _ensure_local_chroma_cache
+
+    _ensure_local_chroma_cache(out_hint / "model-cache")
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return int(args.func(args))
+    except FileNotFoundError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+    except Exception as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 1