folderlake 0.2.0__py3-none-any.whl

folderlake/catalog.py

@@ -0,0 +1,184 @@
+"""The DuckDB catalog: five tables, incremental scanning.
+
+Incremental rule: a file whose (size, mtime) matches the stored row is
+skipped on stat alone — no hashing, no extraction. Only new or changed
+files are hashed and processed. That is what makes re-opening a folder
+near-instant.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import duckdb
+
+from .extractors import extract
+from .topics import compute_topics, term_counts_for
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS files (
+    id VARCHAR PRIMARY KEY,
+    path VARCHAR, name VARCHAR, ext VARCHAR, kind VARCHAR,
+    size_bytes BIGINT, mtime DOUBLE, hash VARCHAR,
+    title VARCHAR, author VARCHAR, extra_json VARCHAR
+);
+CREATE TABLE IF NOT EXISTS file_topics (
+    file_id VARCHAR, tag VARCHAR, weight DOUBLE, source VARCHAR
+);
+CREATE TABLE IF NOT EXISTS file_entities (
+    file_id VARCHAR, entity VARCHAR
+);
+CREATE TABLE IF NOT EXISTS relations (
+    file_a VARCHAR, file_b VARCHAR, kind VARCHAR, detail VARCHAR
+);
+CREATE TABLE IF NOT EXISTS file_text (
+    file_id VARCHAR, summary VARCHAR, text VARCHAR
+);
+CREATE TABLE IF NOT EXISTS file_terms (
+    file_id VARCHAR, term VARCHAR, cnt BIGINT
+);
+CREATE TABLE IF NOT EXISTS file_topic_meta (
+    file_id VARCHAR PRIMARY KEY, hash VARCHAR, method VARCHAR
+);
+CREATE TABLE IF NOT EXISTS topic_nodes (
+    id VARCHAR PRIMARY KEY, parent_id VARCHAR, kind VARCHAR,
+    label VARCHAR, depth INTEGER, file_count INTEGER, cohesion DOUBLE
+);
+CREATE TABLE IF NOT EXISTS topic_node_files (
+    node_id VARCHAR, file_id VARCHAR
+);
+"""
+
+CATALOG_NAME = "folderlake.duckdb"
+
+
+def file_id_for(path: Path) -> str:
+    return hashlib.md5(str(path).encode()).hexdigest()[:16]
+
+
+def _hash_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def connect(db_path: Path) -> duckdb.DuckDBPyConnection:
+    con = duckdb.connect(str(db_path))
+    con.execute(SCHEMA)
+    try:  # catalogs built before the source column existed
+        con.execute("ALTER TABLE file_topics ADD COLUMN IF NOT EXISTS source VARCHAR")
+    except Exception:
+        pass
+    return con
+
+
+def _iter_files(root: Path, catalog_path: Path):
+    # The catalog itself and the generated Neo4j export are not content.
+    catalog_names = {catalog_path.name, catalog_path.name + ".wal", "folderlake.cypher"}
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        if path.name in catalog_names:
+            continue
+        if any(part.startswith(".") for part in path.relative_to(root).parts):
+            continue
+        yield path
+
+
+def scan(root: Path, db_path: Path, use_llm: bool = False,
+         progress=None) -> dict:
+    """Scan ``root`` into the catalog. ``progress``, when given, is
+    called as progress(done, total, current_name) while files are
+    walked, so a UI can show a live bar instead of a frozen wait."""
+    root = root.resolve()
+    con = connect(db_path)
+    stats = {"seen": 0, "added": 0, "updated": 0, "skipped": 0, "removed": 0}
+    existing = {
+        row[0]: (row[1], row[2])
+        for row in con.execute("SELECT path, size_bytes, mtime FROM files").fetchall()
+    }
+    seen_paths: set[str] = set()
+    changed_ids: set[str] = set()
+    try:
+        # One transaction for the whole scan: committing per file costs a
+        # disk sync per statement and made a 1,000-file scan take ~194s;
+        # batched, the same scan is extraction-bound instead.
+        con.execute("BEGIN TRANSACTION")
+        paths = list(_iter_files(root, db_path))
+        total = len(paths)
+        for idx, path in enumerate(paths):
+            if progress is not None:
+                progress(idx, total, path.name)
+            stats["seen"] += 1
+            key = str(path)
+            seen_paths.add(key)
+            st = path.stat()
+            prev = existing.get(key)
+            if prev is not None and prev[0] == st.st_size and abs(prev[1] - st.st_mtime) < 1e-6:
+                stats["skipped"] += 1
+                continue
+            fid = file_id_for(path)
+            result = extract(path)
+            con.execute("DELETE FROM files WHERE id = ?", [fid])
+            con.execute("DELETE FROM file_topics WHERE file_id = ?", [fid])
+            con.execute("DELETE FROM file_entities WHERE file_id = ?", [fid])
+            con.execute("DELETE FROM file_text WHERE file_id = ?", [fid])
+            con.execute("DELETE FROM file_terms WHERE file_id = ?", [fid])
+            con.execute("DELETE FROM file_topic_meta WHERE file_id = ?", [fid])
+            con.execute(
+                "INSERT INTO files VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+                [
+                    fid, key, path.name, path.suffix.lower(), result["kind"],
+                    st.st_size, st.st_mtime, _hash_file(path),
+                    result["title"], result["author"],
+                    json.dumps(result["extra"], ensure_ascii=False),
+                ],
+            )
+            # Term counts are the incremental base for the post-scan
+            # TF-IDF step: unchanged files keep theirs, so adding a
+            # file never re-reads the rest of the folder.
+            for term, cnt in term_counts_for(result, path.name).items():
+                con.execute("INSERT INTO file_terms VALUES (?,?,?)", [fid, term, cnt])
+            con.execute(
+                "INSERT INTO file_text VALUES (?,?,?)",
+                [fid, "", result["text"]],
+            )
+            changed_ids.add(fid)
+            stats["updated" if prev is not None else "added"] += 1
+        for old_path, _ in [(p, None) for p in existing if p not in seen_paths]:
+            fid = file_id_for(Path(old_path))
+            for tbl, col in (("files", "id"), ("file_topics", "file_id"),
+                             ("file_entities", "file_id"), ("file_text", "file_id"),
+                             ("file_terms", "file_id"), ("file_topic_meta", "file_id")):
+                con.execute(f"DELETE FROM {tbl} WHERE {col} = ?", [fid])
+            stats["removed"] += 1
+        con.execute("COMMIT")
+        if progress is not None:
+            progress(total, total, "")
+        # Post-scan topic pass on the same connection: TF-IDF over
+        # stored term counts (no file I/O for unchanged files); the
+        # LLM, when enabled, is called only for files added or updated
+        # in this scan.
+        stats["topics"] = compute_topics(
+            use_llm=use_llm, llm_file_ids=changed_ids, con=con
+        )
+    finally:
+        con.close()
+    return stats
+
+
+def try_build_fts(db_path: Path) -> bool:
+    """Best-effort full-text index over extracted text (used by search later)."""
+    con = duckdb.connect(str(db_path))
+    try:
+        con.execute("INSTALL fts; LOAD fts;")
+        con.execute("PRAGMA create_fts_index('file_text', 'file_id', 'text', overwrite=1)")
+        return True
+    except Exception:
+        return False
+    finally:
+        con.close()

folderlake/cli.py

@@ -0,0 +1,209 @@
+"""FolderLake command line.
+
+    folderlake scan <folder> [--db PATH] [--llm]
+    folderlake list [--db PATH]
+    folderlake preview <file-id-or-name> [--db PATH]
+    folderlake serve <folder> [--db PATH] [--port N] [--open]
+
+``serve`` is the one-command workflow from the README: it scans the
+folder (incremental — unchanged files are skipped), builds relations,
+and opens the three-pane viewer (filters | list/graph | preview).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+import webbrowser
+from pathlib import Path
+
+from .catalog import CATALOG_NAME, connect, scan, try_build_fts
+from .cypher import build_cypher
+from .hierarchy import build_hierarchy
+from .preview import build_preview
+from .relations import build_relations
+from .server import create_server
+
+
+def _default_db(folder: Path) -> Path:
+    return folder / CATALOG_NAME
+
+
+def _run_server(folder: Path, db_path: Path, use_llm: bool, host: str,
+                port: int, open_browser: bool) -> int:
+    stats = scan(folder, db_path, use_llm=use_llm)
+    build_relations(db_path)
+    build_hierarchy(db_path)
+    try_build_fts(db_path)
+    server = create_server(db_path, folder.resolve(), host, port)
+    url = f"http://{host}:{server.server_address[1]}/"
+    print(f"scanned: {stats}", flush=True)
+    print(f"FolderLake viewer: {url}  (folder: {folder}, catalog: {db_path})", flush=True)
+    print("Press Ctrl-C to stop.", flush=True)
+    if open_browser:
+        webbrowser.open(url)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nStopped.")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="folderlake")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    p_scan = sub.add_parser("scan", help="scan a folder into a DuckDB catalog")
+    p_scan.add_argument("folder")
+    p_scan.add_argument("--db", default=None)
+    p_scan.add_argument("--llm", action="store_true",
+                        help="use a local LLM (Ollama) for topics instead of rules")
+
+    p_list = sub.add_parser("list", help="list catalogued files with topics")
+    p_list.add_argument("--db", required=True)
+
+    p_prev = sub.add_parser("preview", help="preview one file by id or name")
+    p_prev.add_argument("target")
+    p_prev.add_argument("--db", required=True)
+
+    p_serve = sub.add_parser(
+        "serve", help="scan a folder and open the local viewer")
+    p_serve.add_argument("folder")
+    p_serve.add_argument("--db", default=None)
+    p_serve.add_argument("--llm", action="store_true",
+                         help="use a local LLM (Ollama) for topics instead of rules")
+    p_serve.add_argument("--port", type=int, default=0,
+                         help="port to listen on (0 = pick a free port)")
+    p_serve.add_argument("--host", default="127.0.0.1")
+    p_serve.add_argument("--open", action="store_true",
+                         help="open the viewer in the default browser")
+
+    p_cypher = sub.add_parser(
+        "cypher", help="export the catalog as a Neo4j Cypher script")
+    p_cypher.add_argument("folder")
+    p_cypher.add_argument("--db", default=None)
+    p_cypher.add_argument("-o", "--output", default=None,
+                          help="output file (default: <folder>/folderlake.cypher)")
+
+    p_app = sub.add_parser(
+        "app", help="web-app mode: open the FolderLake home page "
+                    "(history, new scans, settings) in your browser")
+    p_app.add_argument("--port", type=int, default=8770,
+                       help="port to listen on (default 8770)")
+    p_app.add_argument("--host", default="127.0.0.1")
+
+    p_mcp = sub.add_parser(
+        "mcp", help="serve the catalog to other agents over MCP (stdio)")
+    p_mcp.add_argument("folder", nargs="?", default=None,
+                       help="folder whose catalog to serve")
+    p_mcp.add_argument("--db", default=None,
+                       help="catalog path (overrides the folder default)")
+
+    args = parser.parse_args(argv)
+
+    if args.cmd == "scan":
+        folder = Path(args.folder)
+        if not folder.is_dir():
+            print(f"not a folder: {folder}", file=sys.stderr)
+            return 2
+        db_path = Path(args.db) if args.db else _default_db(folder)
+        started = time.perf_counter()
+        stats = scan(folder, db_path, use_llm=args.llm)
+        elapsed = time.perf_counter() - started
+        rel = build_relations(db_path)
+        hier = build_hierarchy(db_path)
+        fts = try_build_fts(db_path)
+        print(json.dumps({
+            **stats, "relations": rel, "hierarchy": hier, "fts_index": fts,
+            "seconds": round(elapsed, 3),
+            "catalog": str(db_path),
+            "catalog_bytes": db_path.stat().st_size,
+        }, indent=2))
+        return 0
+
+    if args.cmd == "list":
+        con = connect(Path(args.db))
+        try:
+            rows = con.execute(
+                """
+                SELECT f.name, f.kind,
+                       coalesce(string_agg(t.tag, ', '), '') AS tags
+                FROM files f LEFT JOIN file_topics t ON t.file_id = f.id
+                GROUP BY f.name, f.kind ORDER BY f.name
+                """
+            ).fetchall()
+            for name, kind, tags in rows:
+                print(f"{name}\t{kind}\t{tags}")
+            print(f"-- {len(rows)} files")
+        finally:
+            con.close()
+        return 0
+
+    if args.cmd == "preview":
+        con = connect(Path(args.db))
+        try:
+            row = con.execute(
+                "SELECT id FROM files WHERE id = ? OR name = ? LIMIT 1",
+                [args.target, args.target],
+            ).fetchone()
+        finally:
+            con.close()
+        if row is None:
+            print(f"not found: {args.target}", file=sys.stderr)
+            return 2
+        print(json.dumps(build_preview(Path(args.db), row[0]), indent=2, ensure_ascii=False))
+        return 0
+
+    if args.cmd == "cypher":
+        folder = Path(args.folder)
+        db_path = Path(args.db) if args.db else _default_db(folder)
+        if not db_path.exists():
+            print(f"no catalog found: {db_path} (run scan first)", file=sys.stderr)
+            return 2
+        script = build_cypher(db_path)
+        out = Path(args.output) if args.output else folder / "folderlake.cypher"
+        out.write_text(script, encoding="utf-8")
+        print(json.dumps({
+            "cypher": str(out),
+            "bytes": out.stat().st_size,
+            "statements": script.count(";\n") + (1 if script.rstrip().endswith(";") else 0),
+        }, indent=2))
+        return 0
+
+    if args.cmd == "serve":
+        folder = Path(args.folder)
+        if not folder.is_dir():
+            print(f"not a folder: {folder}", file=sys.stderr)
+            return 2
+        db_path = Path(args.db) if args.db else _default_db(folder)
+        return _run_server(folder, db_path, args.llm, args.host, args.port,
+                           args.open)
+
+    if args.cmd == "mcp":
+        from .mcp_server import main as mcp_main
+
+        if args.db:
+            db_path = Path(args.db)
+        elif args.folder:
+            db_path = _default_db(Path(args.folder))
+        else:
+            print("folderlake mcp: give a folder or --db", file=sys.stderr)
+            return 2
+        if not db_path.exists():
+            print(f"catalog not found: {db_path} (scan the folder first)",
+                  file=sys.stderr)
+            return 2
+        return mcp_main(["--db", str(db_path)])
+
+    if args.cmd == "app":
+        from .appserver import run_app
+
+        return run_app(args.host, args.port, open_browser=True)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

folderlake/cypher.py

@@ -0,0 +1,100 @@
+"""Export the catalog as a Neo4j Cypher script.
+
+Graph model:
+  (:File {id, name, path, kind, ext, size_bytes, title})
+  (:Tag {name})            (:Entity {name})
+  (File)-[:TAGGED {weight, source}]->(Tag)
+  (File)-[:MENTIONS]->(Entity)
+  (File)-[:RELATED {kind, detail}]->(File)
+
+The script is idempotent (MERGE everywhere) and self-contained: run it with
+cypher-shell or paste it into Neo4j Browser. Constraints use Neo4j 5 syntax.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import duckdb
+
+
+def _q(value) -> str:
+    """Quote a value as a Cypher literal."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return repr(value)
+    text = str(value).replace("\\", "\\\\").replace("'", "\\'")
+    return f"'{text}'"
+
+
+def build_cypher(db_path: Path) -> str:
+    con = duckdb.connect(str(db_path), read_only=True)
+    try:
+        files = con.execute(
+            "SELECT id, name, path, kind, ext, size_bytes, title FROM files ORDER BY name"
+        ).fetchall()
+        tags = con.execute(
+            "SELECT file_id, tag, weight, source FROM file_topics ORDER BY file_id, weight DESC"
+        ).fetchall()
+        entities = con.execute(
+            "SELECT file_id, entity FROM file_entities ORDER BY file_id, entity"
+        ).fetchall()
+        relations = con.execute(
+            "SELECT file_a, file_b, kind, detail FROM relations ORDER BY file_a, file_b"
+        ).fetchall()
+    finally:
+        con.close()
+
+    lines = [
+        "// FolderLake catalog export for Neo4j",
+        f"// {len(files)} files, {len(tags)} tag links, "
+        f"{len(entities)} entity links, {len(relations)} file relations",
+        "// Run with cypher-shell or paste into Neo4j Browser. Safe to re-run (MERGE).",
+        "",
+        "CREATE CONSTRAINT file_id_unique IF NOT EXISTS FOR (f:File) REQUIRE f.id IS UNIQUE;",
+        "CREATE CONSTRAINT tag_name_unique IF NOT EXISTS FOR (t:Tag) REQUIRE t.name IS UNIQUE;",
+        "CREATE CONSTRAINT entity_name_unique IF NOT EXISTS FOR (e:Entity) REQUIRE e.name IS UNIQUE;",
+        "",
+        "// ---- Files ----",
+    ]
+    for fid, name, path, kind, ext, size, title in files:
+        props = (
+            f"id: {_q(fid)}, name: {_q(name)}, path: {_q(path)}, kind: {_q(kind)}, "
+            f"ext: {_q(ext)}, size_bytes: {_q(size)}, title: {_q(title)}"
+        )
+        lines.append(f"MERGE (f:File {{id: {_q(fid)}}}) SET f += {{{props}}};")
+
+    lines.append("")
+    lines.append("// ---- Tags ----")
+    tag_names = sorted({t[1] for t in tags})
+    for name in tag_names:
+        lines.append(f"MERGE (:Tag {{name: {_q(name)}}});")
+    for fid, tag, weight, source in tags:
+        lines.append(
+            f"MATCH (f:File {{id: {_q(fid)}}}), (t:Tag {{name: {_q(tag)}}}) "
+            f"MERGE (f)-[:TAGGED {{weight: {_q(round(weight, 4))}, source: {_q(source)}}}]->(t);"
+        )
+
+    lines.append("")
+    lines.append("// ---- Entities ----")
+    entity_names = sorted({e[1] for e in entities})
+    for name in entity_names:
+        lines.append(f"MERGE (:Entity {{name: {_q(name)}}});")
+    for fid, entity in entities:
+        lines.append(
+            f"MATCH (f:File {{id: {_q(fid)}}}), (e:Entity {{name: {_q(entity)}}}) "
+            "MERGE (f)-[:MENTIONS]->(e);"
+        )
+
+    lines.append("")
+    lines.append("// ---- File relations ----")
+    for a, b, kind, detail in relations:
+        lines.append(
+            f"MATCH (a:File {{id: {_q(a)}}}), (b:File {{id: {_q(b)}}}) "
+            f"MERGE (a)-[:RELATED {{kind: {_q(kind)}, detail: {_q(detail)}}}]->(b);"
+        )
+    lines.append("")
+    return "\n".join(lines)