folio-cli 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio/__init__.py +3 -0
- folio/__main__.py +4 -0
- folio/assets/SKILL.md +75 -0
- folio/assets/__init__.py +1 -0
- folio/assets/config.yaml +77 -0
- folio/chunk.py +164 -0
- folio/cli.py +308 -0
- folio/config.py +97 -0
- folio/distribute.py +177 -0
- folio/ingest.py +76 -0
- folio/install.py +279 -0
- folio/retrieve.py +185 -0
- folio/store.py +120 -0
- folio/sync.py +110 -0
- folio_cli-0.3.0.dist-info/METADATA +180 -0
- folio_cli-0.3.0.dist-info/RECORD +20 -0
- folio_cli-0.3.0.dist-info/WHEEL +5 -0
- folio_cli-0.3.0.dist-info/entry_points.txt +2 -0
- folio_cli-0.3.0.dist-info/licenses/LICENSE +176 -0
- folio_cli-0.3.0.dist-info/top_level.txt +1 -0
folio/retrieve.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .store import get_collection, load_bm25
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Hit:
|
|
11
|
+
chunk_id: str
|
|
12
|
+
text: str
|
|
13
|
+
score: float
|
|
14
|
+
source_id: str
|
|
15
|
+
rel_path: str
|
|
16
|
+
heading_path: str
|
|
17
|
+
dense_rank: int | None = None
|
|
18
|
+
bm25_rank: int | None = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def rrf_fuse(
|
|
22
|
+
ranked_lists: list[list[str]],
|
|
23
|
+
*,
|
|
24
|
+
k: int = 60,
|
|
25
|
+
) -> dict[str, float]:
|
|
26
|
+
"""Reciprocal Rank Fusion — merges rankings without score normalisation."""
|
|
27
|
+
scores: dict[str, float] = {}
|
|
28
|
+
for ranked in ranked_lists:
|
|
29
|
+
for rank, doc_id in enumerate(ranked, start=1):
|
|
30
|
+
scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
|
|
31
|
+
return scores
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _dense_search(cfg: dict[str, Any], query: str, k: int) -> list[tuple[str, str, dict, float]]:
|
|
35
|
+
collection = get_collection(cfg)
|
|
36
|
+
result = collection.query(
|
|
37
|
+
query_texts=[query],
|
|
38
|
+
n_results=min(k, max(collection.count(), 1)),
|
|
39
|
+
include=["documents", "metadatas", "distances"],
|
|
40
|
+
)
|
|
41
|
+
ids = result["ids"][0] if result["ids"] else []
|
|
42
|
+
docs = result["documents"][0] if result["documents"] else []
|
|
43
|
+
metas = result["metadatas"][0] if result["metadatas"] else []
|
|
44
|
+
dists = result["distances"][0] if result["distances"] else []
|
|
45
|
+
out: list[tuple[str, str, dict, float]] = []
|
|
46
|
+
for i, doc_id in enumerate(ids):
|
|
47
|
+
# cosine distance → similarity-ish for ranking display
|
|
48
|
+
sim = 1.0 - float(dists[i]) if i < len(dists) else 0.0
|
|
49
|
+
out.append((doc_id, docs[i], metas[i] or {}, sim))
|
|
50
|
+
return out
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _bm25_search(cfg: dict[str, Any], query: str, k: int) -> list[tuple[str, str, dict, float]]:
|
|
54
|
+
payload = load_bm25(cfg["paths"]["bm25_path"])
|
|
55
|
+
bm25 = payload["bm25"]
|
|
56
|
+
ids: list[str] = payload["ids"]
|
|
57
|
+
texts: list[str] = payload["texts"]
|
|
58
|
+
metas: list[dict] = payload["metadatas"]
|
|
59
|
+
scores = bm25.get_scores(query.lower().split())
|
|
60
|
+
ranked = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:k]
|
|
61
|
+
return [(ids[i], texts[i], metas[i], float(scores[i])) for i in ranked if scores[i] > 0]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def retrieve(
|
|
65
|
+
cfg: dict[str, Any],
|
|
66
|
+
query: str,
|
|
67
|
+
*,
|
|
68
|
+
final_k: int | None = None,
|
|
69
|
+
use_rerank: bool = False,
|
|
70
|
+
source_filter: str | None = None,
|
|
71
|
+
) -> list[Hit]:
|
|
72
|
+
"""
|
|
73
|
+
Hybrid path from senior revision (Bench):
|
|
74
|
+
query → BM25 + dense → RRF → CRAG widen → optional cross-encoder → top-k
|
|
75
|
+
"""
|
|
76
|
+
rcfg = cfg["retrieval"]
|
|
77
|
+
dense_k = int(rcfg["dense_k"])
|
|
78
|
+
bm25_k = int(rcfg["bm25_k"])
|
|
79
|
+
final_k = final_k or int(rcfg["final_k"])
|
|
80
|
+
rrf_k = int(rcfg["rrf_k"])
|
|
81
|
+
|
|
82
|
+
dense_hits = _dense_search(cfg, query, dense_k)
|
|
83
|
+
bm25_hits = _bm25_search(cfg, query, bm25_k)
|
|
84
|
+
|
|
85
|
+
dense_ids = [h[0] for h in dense_hits]
|
|
86
|
+
bm25_ids = [h[0] for h in bm25_hits]
|
|
87
|
+
fused = rrf_fuse([dense_ids, bm25_ids], k=rrf_k)
|
|
88
|
+
|
|
89
|
+
# CRAG-style widen when best fusion score is weak
|
|
90
|
+
best = max(fused.values()) if fused else 0.0
|
|
91
|
+
if best < float(rcfg["crag_min_best_score"]):
|
|
92
|
+
widen = int(rcfg["crag_widen_factor"])
|
|
93
|
+
dense_hits = _dense_search(cfg, query, dense_k * widen)
|
|
94
|
+
bm25_hits = _bm25_search(cfg, query, bm25_k * widen)
|
|
95
|
+
fused = rrf_fuse([[h[0] for h in dense_hits], [h[0] for h in bm25_hits]], k=rrf_k)
|
|
96
|
+
|
|
97
|
+
by_id: dict[str, tuple[str, dict]] = {}
|
|
98
|
+
dense_rank = {h[0]: i + 1 for i, h in enumerate(dense_hits)}
|
|
99
|
+
bm25_rank = {h[0]: i + 1 for i, h in enumerate(bm25_hits)}
|
|
100
|
+
for doc_id, text, meta, _ in dense_hits + bm25_hits:
|
|
101
|
+
by_id[doc_id] = (text, meta)
|
|
102
|
+
|
|
103
|
+
ordered = sorted(fused.items(), key=lambda x: x[1], reverse=True)
|
|
104
|
+
hits: list[Hit] = []
|
|
105
|
+
for doc_id, score in ordered:
|
|
106
|
+
if doc_id not in by_id:
|
|
107
|
+
continue
|
|
108
|
+
text, meta = by_id[doc_id]
|
|
109
|
+
sid = str(meta.get("source_id", ""))
|
|
110
|
+
if source_filter and sid != source_filter:
|
|
111
|
+
continue
|
|
112
|
+
hits.append(
|
|
113
|
+
Hit(
|
|
114
|
+
chunk_id=doc_id,
|
|
115
|
+
text=text,
|
|
116
|
+
score=score,
|
|
117
|
+
source_id=sid,
|
|
118
|
+
rel_path=str(meta.get("rel_path", "")),
|
|
119
|
+
heading_path=str(meta.get("heading_path", "")),
|
|
120
|
+
dense_rank=dense_rank.get(doc_id),
|
|
121
|
+
bm25_rank=bm25_rank.get(doc_id),
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if use_rerank and hits:
|
|
126
|
+
hits = _cross_encoder_rerank(cfg, query, hits, final_k * 3)[:final_k]
|
|
127
|
+
else:
|
|
128
|
+
hits = hits[:final_k]
|
|
129
|
+
|
|
130
|
+
floor = float(rcfg.get("confidence_floor", 0.0))
|
|
131
|
+
return [h for h in hits if h.score >= floor]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _cross_encoder_rerank(
|
|
135
|
+
cfg: dict[str, Any], query: str, hits: list[Hit], pool: int
|
|
136
|
+
) -> list[Hit]:
|
|
137
|
+
try:
|
|
138
|
+
from sentence_transformers import CrossEncoder
|
|
139
|
+
except ImportError as e:
|
|
140
|
+
raise RuntimeError(
|
|
141
|
+
"Rerank requires optional deps: pip install 'folio[rerank]'"
|
|
142
|
+
) from e
|
|
143
|
+
|
|
144
|
+
model_name = cfg["retrieval"]["reranker_model"]
|
|
145
|
+
model = CrossEncoder(model_name)
|
|
146
|
+
pool_hits = hits[:pool]
|
|
147
|
+
pairs = [(query, h.text) for h in pool_hits]
|
|
148
|
+
scores = model.predict(pairs)
|
|
149
|
+
rescored = []
|
|
150
|
+
for h, s in zip(pool_hits, scores):
|
|
151
|
+
rescored.append(
|
|
152
|
+
Hit(
|
|
153
|
+
chunk_id=h.chunk_id,
|
|
154
|
+
text=h.text,
|
|
155
|
+
score=float(s),
|
|
156
|
+
source_id=h.source_id,
|
|
157
|
+
rel_path=h.rel_path,
|
|
158
|
+
heading_path=h.heading_path,
|
|
159
|
+
dense_rank=h.dense_rank,
|
|
160
|
+
bm25_rank=h.bm25_rank,
|
|
161
|
+
)
|
|
162
|
+
)
|
|
163
|
+
rescored.sort(key=lambda x: x.score, reverse=True)
|
|
164
|
+
return rescored
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def format_context(hits: list[Hit], *, max_chars: int = 12000) -> str:
|
|
168
|
+
parts: list[str] = []
|
|
169
|
+
used = 0
|
|
170
|
+
for i, h in enumerate(hits, start=1):
|
|
171
|
+
header = (
|
|
172
|
+
f"[{i}] source={h.source_id} file={h.rel_path}\n"
|
|
173
|
+
f"heading={h.heading_path} score={h.score:.4f}\n"
|
|
174
|
+
)
|
|
175
|
+
body = h.text
|
|
176
|
+
block = f"{header}{body}\n"
|
|
177
|
+
if used + len(block) > max_chars:
|
|
178
|
+
# Always emit at least one (possibly truncated) citation.
|
|
179
|
+
if not parts:
|
|
180
|
+
room = max(max_chars - used - len(header) - 1, 0)
|
|
181
|
+
parts.append(f"{header}{body[:room]}\n")
|
|
182
|
+
break
|
|
183
|
+
parts.append(block)
|
|
184
|
+
used += len(block)
|
|
185
|
+
return "\n---\n".join(parts)
|
folio/store.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import pickle
|
|
5
|
+
import shutil
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import chromadb
|
|
10
|
+
from chromadb.api.models.Collection import Collection
|
|
11
|
+
from chromadb.utils.embedding_functions import DefaultEmbeddingFunction
|
|
12
|
+
from rank_bm25 import BM25Okapi
|
|
13
|
+
|
|
14
|
+
from .chunk import Chunk
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _ensure_local_chroma_cache(model_cache_dir: Path | None = None) -> Path:
|
|
18
|
+
"""Keep ONNX model cache under the folio index home (sandbox-friendly).
|
|
19
|
+
|
|
20
|
+
Chroma binds ``ONNXMiniLM_L6_V2.DOWNLOAD_PATH`` at import time from Path.home(),
|
|
21
|
+
so we rewrite that class attribute to a folio-local directory.
|
|
22
|
+
"""
|
|
23
|
+
cache = (model_cache_dir or Path.home() / ".folio" / "model-cache").resolve()
|
|
24
|
+
cache.mkdir(parents=True, exist_ok=True)
|
|
25
|
+
# Do NOT rewrite $HOME — that breaks Path.home() for the whole process.
|
|
26
|
+
os.environ["CHROMA_CACHE_DIR"] = str(cache)
|
|
27
|
+
os.environ["XDG_CACHE_HOME"] = str(cache)
|
|
28
|
+
|
|
29
|
+
from chromadb.utils.embedding_functions.onnx_mini_lm_l6_v2 import ONNXMiniLM_L6_V2
|
|
30
|
+
|
|
31
|
+
download = cache / "onnx_models" / ONNXMiniLM_L6_V2.MODEL_NAME
|
|
32
|
+
download.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
ONNXMiniLM_L6_V2.DOWNLOAD_PATH = download
|
|
34
|
+
|
|
35
|
+
real_home = Path.home()
|
|
36
|
+
for candidate in (
|
|
37
|
+
real_home / ".cache/chroma/onnx_models" / ONNXMiniLM_L6_V2.MODEL_NAME,
|
|
38
|
+
real_home / ".folio" / "model-cache" / "onnx_models" / ONNXMiniLM_L6_V2.MODEL_NAME,
|
|
39
|
+
real_home
|
|
40
|
+
/ ".folio"
|
|
41
|
+
/ "model-cache"
|
|
42
|
+
/ "home"
|
|
43
|
+
/ ".cache"
|
|
44
|
+
/ "chroma"
|
|
45
|
+
/ "onnx_models"
|
|
46
|
+
/ ONNXMiniLM_L6_V2.MODEL_NAME,
|
|
47
|
+
Path(__file__).resolve().parents[2] / "data/model-cache/onnx_models" / ONNXMiniLM_L6_V2.MODEL_NAME,
|
|
48
|
+
):
|
|
49
|
+
if (candidate / "onnx" / "model.onnx").exists() and not (
|
|
50
|
+
download / "onnx" / "model.onnx"
|
|
51
|
+
).exists():
|
|
52
|
+
shutil.copytree(candidate, download, dirs_exist_ok=True)
|
|
53
|
+
break
|
|
54
|
+
return cache
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_embedding_fn(model_cache_dir: Path | None = None) -> DefaultEmbeddingFunction:
|
|
58
|
+
_ensure_local_chroma_cache(model_cache_dir)
|
|
59
|
+
return DefaultEmbeddingFunction()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def get_collection(cfg: dict[str, Any]) -> Collection:
|
|
63
|
+
cache = Path(cfg["paths"]["model_cache_dir"])
|
|
64
|
+
_ensure_local_chroma_cache(cache)
|
|
65
|
+
client = chromadb.PersistentClient(path=str(cfg["paths"]["chroma_dir"]))
|
|
66
|
+
emb = get_embedding_fn(cache)
|
|
67
|
+
return client.get_or_create_collection(
|
|
68
|
+
name=cfg["collection"]["name"],
|
|
69
|
+
embedding_function=emb,
|
|
70
|
+
metadata={"hnsw:space": "cosine"},
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def reset_collection(cfg: dict[str, Any]) -> Collection:
|
|
75
|
+
cache = Path(cfg["paths"]["model_cache_dir"])
|
|
76
|
+
_ensure_local_chroma_cache(cache)
|
|
77
|
+
client = chromadb.PersistentClient(path=str(cfg["paths"]["chroma_dir"]))
|
|
78
|
+
name = cfg["collection"]["name"]
|
|
79
|
+
try:
|
|
80
|
+
client.delete_collection(name)
|
|
81
|
+
except Exception:
|
|
82
|
+
pass
|
|
83
|
+
return get_collection(cfg)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def upsert_chunks(collection: Collection, chunks: list[Chunk], batch_size: int = 64) -> int:
|
|
87
|
+
if not chunks:
|
|
88
|
+
return 0
|
|
89
|
+
from tqdm import tqdm
|
|
90
|
+
|
|
91
|
+
for i in tqdm(range(0, len(chunks), batch_size), desc="embedding+upsert"):
|
|
92
|
+
batch = chunks[i : i + batch_size]
|
|
93
|
+
collection.upsert(
|
|
94
|
+
ids=[c.chunk_id for c in batch],
|
|
95
|
+
documents=[c.text for c in batch],
|
|
96
|
+
metadatas=[c.to_metadata() for c in batch],
|
|
97
|
+
)
|
|
98
|
+
return len(chunks)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def build_bm25(chunks: list[Chunk]) -> tuple[BM25Okapi, list[str], list[Chunk]]:
|
|
102
|
+
tokenized = [c.text.lower().split() for c in chunks]
|
|
103
|
+
return BM25Okapi(tokenized), [c.chunk_id for c in chunks], chunks
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def save_bm25(path: Path, bm25: BM25Okapi, ids: list[str], chunks: list[Chunk]) -> None:
|
|
107
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
payload = {
|
|
109
|
+
"bm25": bm25,
|
|
110
|
+
"ids": ids,
|
|
111
|
+
"texts": [c.text for c in chunks],
|
|
112
|
+
"metadatas": [c.to_metadata() for c in chunks],
|
|
113
|
+
}
|
|
114
|
+
with path.open("wb") as f:
|
|
115
|
+
pickle.dump(payload, f)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def load_bm25(path: Path) -> dict[str, Any]:
|
|
119
|
+
with path.open("rb") as f:
|
|
120
|
+
return pickle.load(f)
|
folio/sync.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import shutil
|
|
4
|
+
import subprocess
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _run(cmd: list[str], cwd: Path | None = None) -> None:
|
|
10
|
+
subprocess.run(cmd, cwd=cwd, check=True)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _clone_url(repo: str) -> str:
|
|
14
|
+
"""Normalize local paths to file:// so sparse/depth clones behave."""
|
|
15
|
+
path = Path(repo)
|
|
16
|
+
if path.exists():
|
|
17
|
+
return path.resolve().as_uri()
|
|
18
|
+
return repo
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _is_full_tree(sparse: str) -> bool:
|
|
22
|
+
return sparse.strip() in {".", "", "/"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _content_root(dest: Path, sparse: str) -> Path:
|
|
26
|
+
if _is_full_tree(sparse):
|
|
27
|
+
return dest
|
|
28
|
+
return dest / sparse
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sync_sources(cfg: dict[str, Any], force: bool = False) -> list[Path]:
|
|
32
|
+
"""Sparse-clone / pull each configured GitHub path into data/sources/<id>/."""
|
|
33
|
+
sources_dir: Path = cfg["paths"]["sources_dir"]
|
|
34
|
+
sources_dir.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
synced: list[Path] = []
|
|
36
|
+
|
|
37
|
+
for src in cfg["sources"]:
|
|
38
|
+
dest = sources_dir / src["id"]
|
|
39
|
+
sparse = src["sparse_path"]
|
|
40
|
+
branch = src.get("branch", "main")
|
|
41
|
+
repo = _clone_url(src["repo"])
|
|
42
|
+
full = _is_full_tree(sparse)
|
|
43
|
+
|
|
44
|
+
if dest.exists() and force:
|
|
45
|
+
shutil.rmtree(dest)
|
|
46
|
+
|
|
47
|
+
if dest.exists() and (dest / ".git").exists():
|
|
48
|
+
_run(["git", "fetch", "--depth", "1", "origin", branch], cwd=dest)
|
|
49
|
+
if full:
|
|
50
|
+
# materialize whole tree (sparse "." only keeps cone root)
|
|
51
|
+
_run(["git", "sparse-checkout", "disable"], cwd=dest)
|
|
52
|
+
_run(["git", "checkout", f"origin/{branch}", "--", "."], cwd=dest)
|
|
53
|
+
else:
|
|
54
|
+
_run(["git", "checkout", f"origin/{branch}", "--", sparse], cwd=dest)
|
|
55
|
+
_run(["git", "sparse-checkout", "set", sparse], cwd=dest)
|
|
56
|
+
else:
|
|
57
|
+
if dest.exists():
|
|
58
|
+
shutil.rmtree(dest)
|
|
59
|
+
if full:
|
|
60
|
+
_run(
|
|
61
|
+
[
|
|
62
|
+
"git",
|
|
63
|
+
"clone",
|
|
64
|
+
"--depth",
|
|
65
|
+
"1",
|
|
66
|
+
"--branch",
|
|
67
|
+
branch,
|
|
68
|
+
repo,
|
|
69
|
+
str(dest),
|
|
70
|
+
]
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
_run(
|
|
75
|
+
[
|
|
76
|
+
"git",
|
|
77
|
+
"clone",
|
|
78
|
+
"--depth",
|
|
79
|
+
"1",
|
|
80
|
+
"--filter=blob:none",
|
|
81
|
+
"--sparse",
|
|
82
|
+
"--branch",
|
|
83
|
+
branch,
|
|
84
|
+
repo,
|
|
85
|
+
str(dest),
|
|
86
|
+
]
|
|
87
|
+
)
|
|
88
|
+
_run(["git", "sparse-checkout", "set", sparse], cwd=dest)
|
|
89
|
+
|
|
90
|
+
content_root = _content_root(dest, sparse)
|
|
91
|
+
if not content_root.exists():
|
|
92
|
+
raise FileNotFoundError(f"Expected sparse path missing: {content_root}")
|
|
93
|
+
synced.append(content_root)
|
|
94
|
+
print(f"[sync] {src['id']} → {content_root}")
|
|
95
|
+
|
|
96
|
+
return synced
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def iter_markdown(cfg: dict[str, Any]) -> list[tuple[str, Path]]:
|
|
100
|
+
"""Return (source_id, absolute md path) for every markdown file under synced roots."""
|
|
101
|
+
sources_dir: Path = cfg["paths"]["sources_dir"]
|
|
102
|
+
out: list[tuple[str, Path]] = []
|
|
103
|
+
for src in cfg["sources"]:
|
|
104
|
+
root = _content_root(sources_dir / src["id"], src["sparse_path"])
|
|
105
|
+
if not root.exists():
|
|
106
|
+
continue
|
|
107
|
+
for path in sorted(root.rglob("*.md")):
|
|
108
|
+
if path.is_file() and ".git" not in path.parts:
|
|
109
|
+
out.append((src["id"], path))
|
|
110
|
+
return out
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: folio-cli
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Local hybrid RAG over curated AI / MLOps / infra curricula — graphify-style CLI for any assistant
|
|
5
|
+
Author: Iman Sharei
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/Iman-Sharei/folio
|
|
8
|
+
Project-URL: Repository, https://github.com/Iman-Sharei/folio
|
|
9
|
+
Project-URL: Issues, https://github.com/Iman-Sharei/folio/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Iman-Sharei/folio/releases
|
|
11
|
+
Keywords: rag,mlops,ai-engineering,chromadb,cli,uv
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: chromadb<1.1,>=0.5.23
|
|
23
|
+
Requires-Dist: rank-bm25>=0.2.2
|
|
24
|
+
Requires-Dist: PyYAML>=6.0.2
|
|
25
|
+
Requires-Dist: tqdm>=4.66.0
|
|
26
|
+
Requires-Dist: numpy<3,>=1.26.0
|
|
27
|
+
Provides-Extra: rerank
|
|
28
|
+
Requires-Dist: sentence-transformers<6,>=3.3.0; extra == "rerank"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
32
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
33
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# folio
|
|
37
|
+
|
|
38
|
+
Local hybrid RAG over curated **AI engineering / MLOps / AI infra** markdown curricula
|
|
39
|
+
(graphify-style CLI). Assistants query a **shared machine index** instead of stuffing
|
|
40
|
+
GitHub repos into chat — lower token use, grounded citations.
|
|
41
|
+
|
|
42
|
+
| Source | Path |
|
|
43
|
+
|--------|------|
|
|
44
|
+
| [02-senior-ai-engineer-study-path](https://github.com/RavSinghChandan/02-senior-ai-engineer-study-path) | `revision-1/` |
|
|
45
|
+
| [ai-infra-senior-engineer-learning](https://github.com/ai-infra-curriculum/ai-infra-senior-engineer-learning) | `lessons/` |
|
|
46
|
+
| [agentic-ai-developer-learning](https://github.com/ai-engineering-curriculum/agentic-ai-developer-learning) | `lessons/` |
|
|
47
|
+
| [agentic-ai-engineer-learning](https://github.com/ai-engineering-curriculum/agentic-ai-engineer-learning) | `lessons/` |
|
|
48
|
+
| [senior-agentic-ai-engineer-learning](https://github.com/ai-engineering-curriculum/senior-agentic-ai-engineer-learning) | `lessons/` |
|
|
49
|
+
| [applied-ai-engineer-learning](https://github.com/ai-engineering-curriculum/applied-ai-engineer-learning) | `lessons/` |
|
|
50
|
+
| [ai-eval-engineer-learning](https://github.com/ai-engineering-curriculum/ai-eval-engineer-learning) | `lessons/` |
|
|
51
|
+
| [agentic-systems-architect-learning](https://github.com/ai-engineering-curriculum/agentic-systems-architect-learning) | `lessons/` |
|
|
52
|
+
| [mlops-zoomcamp](https://github.com/DataTalksClub/mlops-zoomcamp) | repo MD modules |
|
|
53
|
+
|
|
54
|
+
## Install (once on this machine)
|
|
55
|
+
|
|
56
|
+
PyPI distribution name is **`folio-cli`** (plain `folio` is taken). The CLI command is still **`folio`**.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
uv tool install folio-cli
|
|
60
|
+
folio --help
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
From a checkout (editable / unreleased):
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
cd ~/Desktop/folio
|
|
67
|
+
uv tool install --editable .
|
|
68
|
+
# or: uv sync && uv run folio status
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Git install without waiting for PyPI:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
uv tool install git+https://github.com/Iman-Sharei/folio
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Get the knowledge index
|
|
78
|
+
|
|
79
|
+
Index lives in **`~/.folio/`** (override with `FOLIO_HOME`). Not inside product repos.
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
folio bootstrap # preferred: prebuilt pack from Release
|
|
83
|
+
# or:
|
|
84
|
+
folio build # sync curated sources + embed (slower)
|
|
85
|
+
folio status
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Refresh when the maintainer publishes new curricula:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
folio update # download new Release pack (falls back to build)
|
|
92
|
+
folio update --from-source # force rebuild from git sources
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Maintainer release flow:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
folio build
|
|
99
|
+
folio pack # → dist/folio-index-YYYY.MM.DD.tar.gz
|
|
100
|
+
# upload tarball to GitHub Release; set release.index_url in config
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Use from any project (like graphify)
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
cd /path/to/your-project
|
|
107
|
+
folio install --project --platform cursor
|
|
108
|
+
folio query "hybrid RAG vs RAGless" --json
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Token rule:** assistants must **not** WebFetch/clone curriculum repos into chat.
|
|
112
|
+
Only `folio query`.
|
|
113
|
+
|
|
114
|
+
## Architecture
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
curated MD → chunk → ONNX MiniLM
|
|
118
|
+
↓
|
|
119
|
+
ChromaDB + BM25 → ~/.folio/
|
|
120
|
+
↓
|
|
121
|
+
query → dense + BM25 → RRF → CRAG → citations (~1–2k tokens, not the whole corpus)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Commands
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
folio bootstrap [--url URL] [--archive FILE] [--force]
|
|
128
|
+
folio build
|
|
129
|
+
folio update [--from-source]
|
|
130
|
+
folio pack [--dest PATH]
|
|
131
|
+
folio sync
|
|
132
|
+
folio query "<q>" [--json] [-k N] [--rerank]
|
|
133
|
+
folio status
|
|
134
|
+
folio install --project --platform cursor|claude|codex|agents
|
|
135
|
+
folio uninstall --project [--purge]
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Tests
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
uv sync --extra dev
|
|
142
|
+
uv run pytest -q # all
|
|
143
|
+
uv run pytest -q -m "not integration" # fast (no Chroma embed)
|
|
144
|
+
uv run pytest -q -m integration # sync + ingest + retrieve + pack
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
CI runs both layers on Python 3.11/3.12. Tag `vX.Y.Z` to:
|
|
148
|
+
- upload the curricula index pack (`folio-index.tar.gz`) to the GitHub Release
|
|
149
|
+
- publish the `folio-cli` wheel/sdist to PyPI (Trusted Publishing; see below)
|
|
150
|
+
|
|
151
|
+
## Maintainer: publish `folio-cli` to PyPI
|
|
152
|
+
|
|
153
|
+
One-time setup (API token **or** Trusted Publishing):
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
./scripts/publish-pypi-wizard.sh
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
After Trusted Publishing is configured, a version tag is enough:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
git tag -a v0.3.0 -m v0.3.0
|
|
163
|
+
git push origin v0.3.0
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Manual local publish (token in env):
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
uv build
|
|
170
|
+
UV_PUBLISH_TOKEN=pypi-... uv publish
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## Maintainer: publish index for `folio bootstrap`
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
folio build
|
|
177
|
+
folio pack # → dist/folio-index-YYYY.MM.DD.tar.gz
|
|
178
|
+
# upload as GitHub Release asset, then set in config:
|
|
179
|
+
# release.index_url: https://github.com/<you>/folio/releases/latest/download/folio-index.tar.gz
|
|
180
|
+
```
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
folio/__init__.py,sha256=Q77p87qEG_CqU5xHsu6FYG9y56H_6jnxAzwv-N-JXrc,115
|
|
2
|
+
folio/__main__.py,sha256=MHKZ_ae3fSLGTLUUMOx15fWdeOnJSHhq-zslRP5F5Lc,79
|
|
3
|
+
folio/chunk.py,sha256=0zNJSMkMVwJmJHKc_HGN5sF4N3Cvku175wiFZ36qNCM,4752
|
|
4
|
+
folio/cli.py,sha256=XeYzVheI3CW5X3JlJ6udn95vt40YG8VCL1A2hYAgKlk,9789
|
|
5
|
+
folio/config.py,sha256=1mcPxE8dQ73l-1MsFSX2dyKoT3OuadqalYhNECohv-g,3058
|
|
6
|
+
folio/distribute.py,sha256=MRODICLDkJkqUyvV5DAN43TcMlYWrBT2JL22BA16r_0,5953
|
|
7
|
+
folio/ingest.py,sha256=kFZVyd65WFVTdA8ErJYz6e1iKnwz4ZhHtnHxvFliSP8,2465
|
|
8
|
+
folio/install.py,sha256=Dii5KAYlpMyzusBEkjKbUuu9xBKoka4BgYokNl-mTDI,10443
|
|
9
|
+
folio/retrieve.py,sha256=hFZEUXZalyvOyFm1tcaHBgSwzMiziXDQJ7LpDjW5IBU,6273
|
|
10
|
+
folio/store.py,sha256=_CFj56P_piuqKI6RjBdlAVYmVFF0SRNhw0VRJFpiY8Y,4085
|
|
11
|
+
folio/sync.py,sha256=vbEJsF_D6tLggveyXqdezRKwfx9XrceNO1LivhOYMaQ,3673
|
|
12
|
+
folio/assets/SKILL.md,sha256=iKp6s_Rj_xmIsCcotC4Ks0BJi5DId0UhOchZk1aB7Rc,3048
|
|
13
|
+
folio/assets/__init__.py,sha256=8q20BIdzAx2q3dbubqUibZCBdLz8-YXgmf_DJzs9O9Q,56
|
|
14
|
+
folio/assets/config.yaml,sha256=bvAHUds1o9NvbiSU2KFgIQDVJnioC2fLb6pV-yUTSpI,2366
|
|
15
|
+
folio_cli-0.3.0.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
|
|
16
|
+
folio_cli-0.3.0.dist-info/METADATA,sha256=XEU6gpYx0HvOMd9K6fcVmf6GSq-CsW0a3BGZpJuvPvQ,5846
|
|
17
|
+
folio_cli-0.3.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
18
|
+
folio_cli-0.3.0.dist-info/entry_points.txt,sha256=EmYULC3K327zuj7qeVnwcnV8Oj_fcd31B8Max_7Gfvo,41
|
|
19
|
+
folio_cli-0.3.0.dist-info/top_level.txt,sha256=7_AGBjZLkQ_w5N3T3ujiEwtUghuLRfDJKS2n3kjj2gM,6
|
|
20
|
+
folio_cli-0.3.0.dist-info/RECORD,,
|