folderlake 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
folderlake/home.html ADDED
@@ -0,0 +1,305 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>FolderLake</title>
7
+ <style>
8
+ :root { --orange:#ed7d31; --blue:#4a80d0; --ink:#1c1c1c; --muted:#777;
9
+ --line:#ddd; --bg:#f4f4f4; }
10
+ * { box-sizing: border-box; }
11
+ body { margin:0; font:15px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif;
12
+ background:var(--bg); color:var(--ink); }
13
+ .wrap { max-width:880px; margin:0 auto; padding:28px 18px 60px; }
14
+ header h1 { font-size:30px; margin:0 0 4px; letter-spacing:-.5px; }
15
+ header p { margin:0 0 22px; color:var(--muted); }
16
+ .card { background:#fff; border:1px solid var(--line); border-radius:12px;
17
+ padding:18px 20px; margin-bottom:18px; }
18
+ .card h2 { font-size:17px; margin:0 0 12px; }
19
+ .row { display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
20
+ input[type=text], input[type=password], select {
21
+ font:inherit; padding:9px 11px; border:1px solid var(--line);
22
+ border-radius:8px; background:#fff; }
23
+ input.grow { flex:1; min-width:220px; }
24
+ button { font:inherit; padding:9px 16px; border-radius:8px; cursor:pointer;
25
+ border:1px solid var(--line); background:#fff; }
26
+ button.primary { background:var(--orange); border-color:var(--orange);
27
+ color:#fff; font-weight:600; }
28
+ button.danger { background:#d64545; border-color:#d64545; color:#fff; }
29
+ button:disabled { opacity:.5; cursor:default; }
30
+ label.check { display:inline-flex; gap:6px; align-items:center; }
31
+ .muted { color:var(--muted); font-size:13px; }
32
+ .catalog { border-top:1px solid var(--line); padding:14px 2px; }
33
+ .catalog:first-of-type { border-top:none; }
34
+ .catalog .name { font-weight:650; font-size:16px; }
35
+ .catalog .path { color:var(--muted); font-size:13px; word-break:break-all; }
36
+ .chips { display:flex; gap:6px; flex-wrap:wrap; margin:6px 0; }
37
+ .chip { background:#eef3fb; color:#2c5aa0; border-radius:20px;
38
+ padding:1px 10px; font-size:12.5px; }
39
+ .chip.rules { background:#f0f0f0; color:#666; }
40
+ .actions { display:flex; gap:8px; margin-top:8px; flex-wrap:wrap; }
41
+ .confirm { color:#b03030; font-size:14px; }
42
+ #job { display:none; }
43
+ #jobbar { height:10px; background:#eee; border-radius:6px; overflow:hidden;
44
+ margin:10px 0 6px; }
45
+ #jobfill { height:100%; width:0%; background:var(--orange);
46
+ transition:width .3s ease; }
47
+ .error { color:#b03030; }
48
+ .ok { color:#1d7a4f; }
49
+ .grid2 { display:grid; grid-template-columns:repeat(auto-fit,minmax(240px,1fr));
50
+ gap:10px 16px; }
51
+ .field label { display:block; font-size:13px; color:var(--muted);
52
+ margin-bottom:4px; }
53
+ .field input, .field select { width:100%; }
54
+ footer { color:var(--muted); font-size:13px; text-align:center; margin-top:8px; }
55
+ </style>
56
+ </head>
57
+ <body>
58
+ <div class="wrap">
59
+ <header>
60
+ <h1>FolderLake</h1>
61
+ <p>Point it at a folder. Know what's in it. Catalogs live in each
62
+ folder as <code>folderlake.duckdb</code>; this page remembers them.</p>
63
+ </header>
64
+
65
+ <section class="card">
66
+ <h2>New catalog</h2>
67
+ <div class="row">
68
+ <input type="text" id="path" class="grow"
69
+ placeholder="Folder path, e.g. /Users/you/Documents/Projects">
70
+ <button id="browse" type="button">Browse…</button>
71
+ </div>
72
+ <div class="row" style="margin-top:10px">
73
+ <label class="check"><input type="checkbox" id="use-llm">
74
+ Use LLM for topics</label>
75
+ <button id="start" class="primary" type="button">Scan folder</button>
76
+ <span class="muted">A first scan takes a while on big folders —
77
+ progress shows here, the page never just sits frozen.</span>
78
+ </div>
79
+ <div id="job">
80
+ <div id="jobbar"><div id="jobfill"></div></div>
81
+ <div id="jobtext" class="muted">Starting…</div>
82
+ </div>
83
+ </section>
84
+
85
+ <section class="card">
86
+ <h2>Your catalogs</h2>
87
+ <div id="catalogs"><p class="muted">Loading…</p></div>
88
+ </section>
89
+
90
+ <section class="card">
91
+ <h2>Settings</h2>
92
+ <div class="grid2">
93
+ <div class="field"><label>LLM provider</label>
94
+ <select id="s-provider">
95
+ <option value="ollama">Ollama (local)</option>
96
+ <option value="openai">OpenAI-compatible</option>
97
+ <option value="anthropic">Anthropic</option>
98
+ <option value="huggingface">Hugging Face</option>
99
+ </select></div>
100
+ <div class="field"><label>Model (blank = provider default)</label>
101
+ <input type="text" id="s-model" placeholder="e.g. llama3.1"></div>
102
+ <div class="field"><label>Base URL (for OpenAI-compatible endpoints)</label>
103
+ <input type="text" id="s-base" placeholder="https://…/v1"></div>
104
+ <div class="field"><label>API key</label>
105
+ <input type="password" id="s-key" placeholder="(not set)"></div>
106
+ </div>
107
+ <div class="row" style="margin-top:12px">
108
+ <label class="check"><input type="checkbox" id="s-usellm">
109
+ Use LLM for topics by default</label>
110
+ <button id="s-save" type="button">Save settings</button>
111
+ <span id="s-note" class="muted"></span>
112
+ </div>
113
+ <p class="muted">Settings are stored only on this machine, in
114
+ <code>~/.folderlake/registry.sqlite3</code>. Each catalog's history
115
+ entry keeps a snapshot of the settings it was scanned with.</p>
116
+ </section>
117
+
118
+ <footer>FolderLake — local-first. Nothing leaves your machine.</footer>
119
+ </div>
120
+
121
+ <script>
122
+ const $ = (s) => document.querySelector(s);
123
+ let activeJob = null; // {id, isNew}
124
+ let confirmId = null; // catalog id currently asking "really delete?"
125
+ let catalogs = [];
126
+
127
+ async function api(path, opts) {
128
+ const r = await fetch(path, opts);
129
+ const data = await r.json().catch(() => ({}));
130
+ if (!r.ok) throw new Error(data.error || ("HTTP " + r.status));
131
+ return data;
132
+ }
133
+ const post = (path, body) => api(path, {
134
+ method: "POST", headers: {"Content-Type": "application/json"},
135
+ body: JSON.stringify(body || {}) });
136
+
137
+ function fmtDate(ts) {
138
+ if (!ts) return "—";
139
+ return new Date(ts * 1000).toLocaleString([], {month:"short", day:"numeric",
140
+ hour:"2-digit", minute:"2-digit"});
141
+ }
142
+ function fmtMB(b) {
143
+ if (!b) return "0 MB";
144
+ return b >= 1e9 ? (b/1e9).toFixed(1)+" GB"
145
+ : b >= 1e6 ? (b/1e6).toFixed(1)+" MB" : Math.round(b/1e3)+" KB";
146
+ }
147
+
148
+ function renderCatalogs() {
149
+ const box = $("#catalogs");
150
+ if (!catalogs.length) {
151
+ box.innerHTML = '<p class="muted">No catalogs yet — scan a folder above ' +
152
+ 'and it will appear here for one-click reopening.</p>';
153
+ return;
154
+ }
155
+ box.innerHTML = "";
156
+ for (const c of catalogs) {
157
+ const div = document.createElement("div");
158
+ div.className = "catalog";
159
+ const s = c.settings || {};
160
+ const chip = s.use_llm
161
+ ? `<span class="chip">LLM · ${escapeHtml(s.llm_provider || "ollama")}${s.llm_model ? " / " + escapeHtml(s.llm_model) : ""}</span>`
162
+ : '<span class="chip rules">Rule topics (no LLM)</span>';
163
+ let actions;
164
+ if (confirmId === c.id) {
165
+ actions = `<span class="confirm">Delete this catalog? The record and
166
+ its <code>folderlake.duckdb</code> file are removed; your files stay.</span>
167
+ <button class="danger" data-act="confirm" data-id="${c.id}">Confirm delete</button>
168
+ <button data-act="cancel">Cancel</button>`;
169
+ } else {
170
+ actions = `<button class="primary" data-act="open" data-id="${c.id}" ${c.exists ? "" : "disabled"}>Open</button>
171
+ <button data-act="refresh" data-id="${c.id}" ${c.exists ? "" : "disabled"}>Refresh</button>
172
+ <button data-act="del" data-id="${c.id}">Delete</button>`;
173
+ }
174
+ div.innerHTML = `
175
+ <div class="name">${escapeHtml(c.name)}${c.exists ? "" :
176
+ ' <span class="error">(folder not found)</span>'}</div>
177
+ <div class="path">${escapeHtml(c.path)}</div>
178
+ <div class="chips">${chip}
179
+ <span class="chip rules">${c.file_count} files · ${fmtMB(c.catalog_bytes)}</span>
180
+ <span class="chip rules">scanned ${fmtDate(c.updated_at)}</span></div>
181
+ <div class="actions">${actions}</div>`;
182
+ box.appendChild(div);
183
+ }
184
+ }
185
+ function escapeHtml(s) {
186
+ return String(s).replace(/[&<>"']/g, (m) => ({"&":"&amp;","<":"&lt;",
187
+ ">":"&gt;",'"':"&quot;","'":"&#39;"}[m]));
188
+ }
189
+
190
+ $("#catalogs").addEventListener("click", async (e) => {
191
+ const btn = e.target.closest("button");
192
+ if (!btn) { confirmId = null; renderCatalogs(); return; }
193
+ const id = btn.dataset.id, act = btn.dataset.act;
194
+ try {
195
+ if (act === "open") {
196
+ const r = await post(`/api/catalogs/${id}/open`);
197
+ location.href = r.url;
198
+ } else if (act === "refresh") {
199
+ confirmId = null; renderCatalogs();
200
+ startJob(await post(`/api/catalogs/${id}/refresh`), false);
201
+ } else if (act === "del") {
202
+ confirmId = id; renderCatalogs();
203
+ } else if (act === "cancel") {
204
+ confirmId = null; renderCatalogs();
205
+ } else if (act === "confirm") {
206
+ await api(`/api/catalogs/${id}`, {method: "DELETE"});
207
+ confirmId = null;
208
+ await loadCatalogs();
209
+ }
210
+ } catch (err) { alert(err.message); }
211
+ });
212
+ document.addEventListener("click", (e) => {
213
+ if (confirmId && !e.target.closest(".catalog")) {
214
+ confirmId = null; renderCatalogs();
215
+ }
216
+ });
217
+
218
+ async function loadCatalogs() {
219
+ catalogs = await api("/api/catalogs");
220
+ renderCatalogs();
221
+ }
222
+
223
+ function startJob(job, isNew) {
224
+ activeJob = {...job, isNew};
225
+ $("#job").style.display = "block";
226
+ $("#start").disabled = true;
227
+ pollJob();
228
+ }
229
+ async function pollJob() {
230
+ if (!activeJob) return;
231
+ try {
232
+ const j = await api("/api/jobs/" + activeJob.id);
233
+ const pct = j.total ? Math.round(100 * j.done / j.total) : 0;
234
+ $("#jobfill").style.width = (j.phase === "scanning" ? pct : 100) + "%";
235
+ $("#jobtext").textContent =
236
+ j.status === "error" ? "" :
237
+ `${j.phase} — ${j.done}/${j.total} files${j.current ? " — " + j.current : ""}`;
238
+ if (j.status === "done") {
239
+ const wasNew = activeJob.isNew, cid = j.catalog_id;
240
+ activeJob = null;
241
+ $("#start").disabled = false;
242
+ $("#jobtext").innerHTML = '<span class="ok">Done.</span>';
243
+ await loadCatalogs();
244
+ if (wasNew && cid) {
245
+ const r = await post(`/api/catalogs/${cid}/open`);
246
+ location.href = r.url;
247
+ }
248
+ return;
249
+ }
250
+ if (j.status === "error") {
251
+ $("#jobtext").innerHTML =
252
+ `<span class="error">Scan failed: ${escapeHtml(j.message || "")}</span>`;
253
+ activeJob = null;
254
+ $("#start").disabled = false;
255
+ await loadCatalogs();
256
+ return;
257
+ }
258
+ } catch (e) { /* transient; keep polling */ }
259
+ setTimeout(pollJob, 600);
260
+ }
261
+
262
+ $("#browse").addEventListener("click", async () => {
263
+ try {
264
+ const r = await post("/api/browse");
265
+ if (r.path) $("#path").value = r.path; // cancelled = just nothing
266
+ } catch (e) { /* no desktop dialog available; typing still works */ }
267
+ });
268
+ $("#start").addEventListener("click", async () => {
269
+ const path = $("#path").value.trim();
270
+ if (!path) { alert("Enter a folder path (or use Browse…)."); return; }
271
+ try {
272
+ startJob(await post("/api/scan", {
273
+ path, use_llm: $("#use-llm").checked }), true);
274
+ } catch (err) { alert(err.message); }
275
+ });
276
+
277
+ async function loadSettings() {
278
+ const s = await api("/api/settings");
279
+ $("#s-provider").value = s.llm_provider || "ollama";
280
+ $("#s-model").value = s.llm_model || "";
281
+ $("#s-base").value = s.llm_base_url || "";
282
+ $("#s-key").placeholder = s.llm_api_key_set ? "•••••••• (saved)" : "(not set)";
283
+ $("#s-usellm").checked = !!s.use_llm;
284
+ $("#use-llm").checked = !!s.use_llm;
285
+ }
286
+ $("#s-save").addEventListener("click", async () => {
287
+ try {
288
+ const s = await post("/api/settings", {
289
+ llm_provider: $("#s-provider").value,
290
+ llm_model: $("#s-model").value.trim(),
291
+ llm_base_url: $("#s-base").value.trim(),
292
+ llm_api_key: $("#s-key").value,
293
+ use_llm: $("#s-usellm").checked,
294
+ });
295
+ $("#s-key").value = "";
296
+ $("#s-key").placeholder = s.llm_api_key_set ? "•••••••• (saved)" : "(not set)";
297
+ $("#s-note").textContent = "Saved.";
298
+ setTimeout(() => $("#s-note").textContent = "", 2500);
299
+ } catch (err) { alert(err.message); }
300
+ });
301
+
302
+ (async () => { await loadSettings(); await loadCatalogs(); })();
303
+ </script>
304
+ </body>
305
+ </html>
folderlake/llm.py ADDED
@@ -0,0 +1,210 @@
1
+ """LLM provider layer — one prompt in, one reply out.
2
+
3
+ Transport: LiteLLM when it is installed (`pip install folderlake[llm]`)
4
+ — provider quirks, retries and response shapes are a solved problem
5
+ and we do not reinvent them. Without LiteLLM, a small built-in client
6
+ below speaks to the same providers with the same env configuration,
7
+ so the base install stays dependency-light. Both paths return None on
8
+ any failure; callers fall back to rule-based topics.
9
+
10
+ Providers (FOLDERLAKE_LLM_PROVIDER):
11
+ ollama Local Ollama server (default). OLLAMA_HOST or
12
+ FOLDERLAKE_LLM_BASE_URL, model default "qwen3".
13
+ openai Any OpenAI-compatible chat endpoint: OpenAI itself,
14
+ DeepSeek, Qwen/DashScope compatible mode, Groq, Together,
15
+ LM Studio, vLLM, text-generation-webui — set
16
+ FOLDERLAKE_LLM_BASE_URL + FOLDERLAKE_LLM_API_KEY.
17
+ Default base https://api.openai.com/v1, model gpt-4o-mini.
18
+ anthropic Anthropic Messages API. Key from FOLDERLAKE_LLM_API_KEY
19
+ or ANTHROPIC_API_KEY; model default claude-haiku-4-5.
20
+ huggingface Hugging Face's OpenAI-compatible router
21
+ (https://router.huggingface.co/v1) — hosted inference for
22
+ open models without running them locally. Key from
23
+ FOLDERLAKE_LLM_API_KEY or HF_TOKEN. (Running transformers
24
+ locally is deliberately not bundled: torch alone would
25
+ dwarf the rest of the app; a local HF model is reachable
26
+ anyway through the openai provider + a local server.)
27
+
28
+ Every failure returns None — callers fall back to rule-based topics.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import os
35
+ import urllib.request
36
+
37
+ TIMEOUT = 30
38
+
39
+
40
+ def _env(*names: str, default: str = "") -> str:
41
+ for n in names:
42
+ v = os.environ.get(n)
43
+ if v:
44
+ return v
45
+ return default
46
+
47
+
48
+ def provider_name() -> str:
49
+ p = _env("FOLDERLAKE_LLM_PROVIDER", default="ollama").lower()
50
+ return {"hf": "huggingface"}.get(p, p)
51
+
52
+
53
+ def _model(default: str) -> str:
54
+ return _env("FOLDERLAKE_LLM_MODEL", default=default)
55
+
56
+
57
+ def _post(url: str, payload: dict, headers: dict | None = None) -> dict | None:
58
+ req = urllib.request.Request(
59
+ url,
60
+ data=json.dumps(payload).encode(),
61
+ headers={"Content-Type": "application/json", **(headers or {})},
62
+ )
63
+ try:
64
+ with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
65
+ return json.loads(resp.read())
66
+ except Exception:
67
+ return None
68
+
69
+
70
+ def _chat_completions(base: str, key: str, model: str, prompt: str) -> str | None:
71
+ headers = {"Authorization": f"Bearer {key}"} if key else {}
72
+ payload = _post(
73
+ f"{base.rstrip('/')}/chat/completions",
74
+ {"model": model, "messages": [{"role": "user", "content": prompt}]},
75
+ headers,
76
+ )
77
+ if not payload:
78
+ return None
79
+ try:
80
+ return payload["choices"][0]["message"]["content"]
81
+ except (KeyError, IndexError, TypeError):
82
+ return None
83
+
84
+
85
+ def _litellm_target(provider: str) -> tuple[str, str | None, str | None]:
86
+ """(model, api_base, api_key) for LiteLLM, per provider."""
87
+ if provider == "ollama":
88
+ host = _env("FOLDERLAKE_LLM_BASE_URL", "OLLAMA_HOST",
89
+ default="http://localhost:11434")
90
+ return f"ollama/{_model('qwen3')}", host, None
91
+ if provider == "openai":
92
+ return (
93
+ f"openai/{_model('gpt-4o-mini')}",
94
+ _env("FOLDERLAKE_LLM_BASE_URL", default="https://api.openai.com/v1"),
95
+ _env("FOLDERLAKE_LLM_API_KEY", "OPENAI_API_KEY") or None,
96
+ )
97
+ if provider == "anthropic":
98
+ return (
99
+ f"anthropic/{_model('claude-haiku-4-5')}",
100
+ None,
101
+ _env("FOLDERLAKE_LLM_API_KEY", "ANTHROPIC_API_KEY") or None,
102
+ )
103
+ # Hugging Face's router is OpenAI-compatible: reach it through
104
+ # LiteLLM's openai route with the router as api_base.
105
+ return (
106
+ f"openai/{_model('meta-llama/Llama-3.1-70B-Instruct')}",
107
+ _env("FOLDERLAKE_LLM_BASE_URL",
108
+ default="https://router.huggingface.co/v1"),
109
+ _env("FOLDERLAKE_LLM_API_KEY", "HF_TOKEN", "HUGGINGFACE_API_KEY") or None,
110
+ )
111
+
112
+
113
+ def _chat_litellm(prompt: str, provider: str) -> str | None:
114
+ """Via LiteLLM; None when unavailable or on any error."""
115
+ try:
116
+ import litellm
117
+ except Exception:
118
+ return None
119
+ litellm.suppress_debug_info = True
120
+ model, api_base, api_key = _litellm_target(provider)
121
+ try:
122
+ resp = litellm.completion(
123
+ model=model,
124
+ messages=[{"role": "user", "content": prompt}],
125
+ api_base=api_base,
126
+ api_key=api_key,
127
+ timeout=TIMEOUT,
128
+ )
129
+ return resp.choices[0].message.content
130
+ except Exception:
131
+ return None
132
+
133
+
134
+ def chat(prompt: str) -> str | None:
135
+ """Send one prompt to the configured provider; raw reply text or None.
136
+
137
+ LiteLLM first when installed; the built-in client below otherwise
138
+ (and as a fallback if the LiteLLM call itself fails).
139
+ """
140
+ provider = provider_name()
141
+ via_lib = _chat_litellm(prompt, provider)
142
+ if via_lib is not None:
143
+ return via_lib
144
+ if provider == "ollama":
145
+ host = _env("FOLDERLAKE_LLM_BASE_URL", "OLLAMA_HOST",
146
+ default="http://localhost:11434")
147
+ payload = _post(
148
+ f"{host.rstrip('/')}/api/generate",
149
+ {"model": _model("qwen3"), "prompt": prompt, "stream": False},
150
+ )
151
+ return payload.get("response") if payload else None
152
+ if provider == "openai":
153
+ return _chat_completions(
154
+ _env("FOLDERLAKE_LLM_BASE_URL", default="https://api.openai.com/v1"),
155
+ _env("FOLDERLAKE_LLM_API_KEY", "OPENAI_API_KEY"),
156
+ _model("gpt-4o-mini"),
157
+ prompt,
158
+ )
159
+ if provider == "huggingface":
160
+ return _chat_completions(
161
+ _env("FOLDERLAKE_LLM_BASE_URL",
162
+ default="https://router.huggingface.co/v1"),
163
+ _env("FOLDERLAKE_LLM_API_KEY", "HF_TOKEN", "HUGGINGFACE_API_KEY"),
164
+ _model("meta-llama/Llama-3.1-70B-Instruct"),
165
+ prompt,
166
+ )
167
+ if provider == "anthropic":
168
+ base = _env("FOLDERLAKE_LLM_BASE_URL",
169
+ default="https://api.anthropic.com")
170
+ payload = _post(
171
+ f"{base.rstrip('/')}/v1/messages",
172
+ {
173
+ "model": _model("claude-haiku-4-5"),
174
+ "max_tokens": 1024,
175
+ "messages": [{"role": "user", "content": prompt}],
176
+ },
177
+ {
178
+ "x-api-key": _env("FOLDERLAKE_LLM_API_KEY", "ANTHROPIC_API_KEY"),
179
+ "anthropic-version": "2023-06-01",
180
+ },
181
+ )
182
+ if not payload:
183
+ return None
184
+ try:
185
+ return "".join(
186
+ block.get("text", "")
187
+ for block in payload["content"]
188
+ if block.get("type") == "text"
189
+ ) or None
190
+ except (KeyError, TypeError):
191
+ return None
192
+ return None
193
+
194
+
195
+ def parse_json_response(text: str | None) -> dict | None:
196
+ """Tolerant JSON extraction: strips code fences, finds the first object."""
197
+ if not text:
198
+ return None
199
+ start = text.find("{")
200
+ if start < 0:
201
+ return None
202
+ try:
203
+ obj, _ = json.JSONDecoder().raw_decode(text[start:])
204
+ except Exception:
205
+ return None
206
+ return obj if isinstance(obj, dict) else None
207
+
208
+
209
+ def chat_json(prompt: str) -> dict | None:
210
+ return parse_json_response(chat(prompt))