folderlake 0.2.0__py3-none-any.whl

folderlake/mcp_server.py

@@ -0,0 +1,331 @@
+"""MCP server — let other AI agents query a FolderLake catalog.
+
+The tool handlers are plain Python functions, independent of any SDK,
+registered in the ``TOOLS`` dict below (name -> (description, handler)).
+``serve_stdio`` wires them up through the official ``mcp`` Python SDK,
+imported lazily inside the function: the SDK is an optional dependency
+(``pip install folderlake[mcp]``), never a hard one — the same pattern
+the LLM layer uses with LiteLLM.
+
+Everything is read-only: handlers reuse the viewer's read APIs
+(``server.list_files`` / ``facets`` / ``file_detail``) or open the
+catalog read-only themselves. Results are capped and long text fields
+truncated — agent context is precious.
+
+Run it with:
+
+    python -m folderlake.mcp_server --db /path/to/folderlake.duckdb
+
+or set ``FOLDERLAKE_DB``. A typical MCP client config:
+
+    {"mcpServers": {"folderlake": {
+        "command": "python", "args": ["-m", "folderlake.mcp_server"],
+        "env": {"FOLDERLAKE_DB": "/path/to/folderlake.duckdb"}}}}
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from typing import Callable
+
+import duckdb
+
+from .server import facets, file_detail, list_files
+
+MAX_TEXT = 500          # summaries / text previews are cut to this
+MAX_DETAIL = 200        # relation details are short labels; cap anyway
+DEFAULT_LIMIT = 50
+MAX_LIMIT = 200
+MAX_RELATED = 100
+MAX_COLUMNS = 50
+
+
+def _truncate(text, limit: int = MAX_TEXT) -> str:
+    if text is None:
+        return ""
+    text = str(text)
+    return text if len(text) <= limit else text[:limit] + "…"
+
+
+def _file_summary(row: dict) -> dict:
+    """Slim a list_files row down to what an agent needs to triage."""
+    return {
+        "id": row["id"],
+        "name": row["name"],
+        "kind": row["kind"],
+        "ext": row["ext"],
+        "size_bytes": row["size_bytes"],
+        "title": row.get("title"),
+        "tags": row.get("tags", []),
+    }
+
+
+def _paged(rows: list[dict], limit: int) -> dict:
+    return {
+        "count": len(rows),
+        "returned": min(len(rows), limit),
+        "truncated": len(rows) > limit,
+        "files": [_file_summary(r) for r in rows[:limit]],
+    }
+
+
+# ----------------------------------------------------------------------
+# Tool handlers — plain functions: (db_path, **args) -> JSON-able result
+# ----------------------------------------------------------------------
+def _tool_stats(db_path: Path) -> dict:
+    f = facets(db_path)
+    return {
+        "total_files": f["total_files"],
+        "kinds": f["kinds"],
+        "topic_count": len(f["topics"]),
+    }
+
+
+def _tool_search(db_path: Path, query: str, kind: str | None = None) -> dict:
+    rows = list_files(db_path, kinds=[kind] if kind else None, query=query)
+    result = _paged(rows, MAX_LIMIT)
+    result["query"] = query
+    return result
+
+
+def _tool_list_files(db_path: Path, kind: str | None = None,
+                     topic: str | None = None,
+                     limit: int = DEFAULT_LIMIT) -> dict:
+    try:
+        limit = int(limit)
+    except (TypeError, ValueError):
+        limit = DEFAULT_LIMIT
+    limit = max(1, min(limit, MAX_LIMIT))
+    rows = list_files(db_path, kinds=[kind] if kind else None, topic=topic)
+    return _paged(rows, limit)
+
+
+def _tool_file_detail(db_path: Path, file_id: str) -> dict:
+    detail = file_detail(db_path, file_id)
+    if detail is None:
+        return {"error": f"not found: {file_id}"}
+    out: dict = {
+        "id": detail["id"],
+        "name": detail["name"],
+        "path": detail["path"],
+        "kind": detail["kind"],
+        "size_bytes": detail["size_bytes"],
+        "title": detail.get("title"),
+        "author": detail.get("author"),
+        "summary": _truncate(detail.get("summary")),
+        "tags": detail.get("tags", []),
+        "entities": detail.get("entities", []),
+    }
+    if "text_preview" in detail:
+        out["text_preview"] = _truncate(detail["text_preview"])
+    if "data_review" in detail:
+        review = detail["data_review"]
+        out["data_review"] = {
+            "columns": review.get("columns", [])[:MAX_COLUMNS],
+            "row_count": review.get("row_count"),
+            "nulls": dict(list((review.get("nulls") or {}).items())[:MAX_COLUMNS]),
+            "summarize": review.get("summarize", [])[:MAX_COLUMNS],
+            "sample_query": review.get("sample_query"),
+        }
+    return out
+
+
+def _tool_related(db_path: Path, file_id: str) -> dict:
+    con = duckdb.connect(str(db_path), read_only=True)
+    try:
+        rows = con.execute(
+            """
+            SELECT other_id, kind, detail, name, other_kind FROM (
+                SELECT CASE WHEN r.file_a = ? THEN r.file_b ELSE r.file_a END
+                           AS other_id,
+                       r.kind AS kind, r.detail AS detail,
+                       f.name AS name, f.kind AS other_kind
+                FROM relations r
+                JOIN files f
+                  ON f.id = CASE WHEN r.file_a = ? THEN r.file_b ELSE r.file_a END
+                WHERE r.file_a = ? OR r.file_b = ?
+            ) ORDER BY kind, name LIMIT ?
+            """,
+            [file_id, file_id, file_id, file_id, MAX_RELATED],
+        ).fetchall()
+    finally:
+        con.close()
+    return {
+        "file_id": file_id,
+        "count": len(rows),
+        "related": [
+            {
+                "file_id": r[0],
+                "name": r[3],
+                "kind": r[4],
+                "relation_kind": r[1],
+                "detail": _truncate(r[2], MAX_DETAIL),
+            }
+            for r in rows
+        ],
+    }
+
+
+def _topic_tree(db_path: Path) -> dict | None:
+    """Nested topic hierarchy from topic_nodes, or None if unavailable.
+
+    The hierarchy tables are built by a separate step and may not exist
+    in a catalog; any problem at all falls back to the flat topics.
+    """
+    con = duckdb.connect(str(db_path), read_only=True)
+    try:
+        exists = con.execute(
+            "SELECT count(*) FROM information_schema.tables "
+            "WHERE table_name = 'topic_nodes'"
+        ).fetchone()[0]
+        if not exists:
+            return None
+        rows = con.execute(
+            "SELECT id, parent_id, kind, label, file_count FROM topic_nodes"
+        ).fetchall()
+        if not rows:
+            return None  # table exists but no hierarchy was built
+    except Exception:
+        return None
+    finally:
+        con.close()
+    nodes: dict[str, dict] = {
+        r[0]: {
+            "id": r[0], "label": r[3], "kind": r[2],
+            "file_count": r[4], "children": [],
+        }
+        for r in rows
+    }
+    roots: list[dict] = []
+    for r in rows:
+        node = nodes[r[0]]
+        parent = nodes.get(r[1]) if r[1] else None
+        if parent is None:
+            roots.append(node)
+        else:
+            parent["children"].append(node)
+    return {"mode": "hierarchy", "roots": roots}
+
+
+def _tool_topics(db_path: Path) -> dict:
+    tree = _topic_tree(db_path)
+    if tree is not None:
+        return tree
+    return {"mode": "flat", "topics": facets(db_path)["topics"]}
+
+
+TOOLS: dict[str, tuple[str, Callable[..., dict]]] = {
+    "folderlake_stats": (
+        "Catalog overview: total file count, counts per kind "
+        "(document/data/text), and the number of distinct topics.",
+        _tool_stats,
+    ),
+    "folderlake_search": (
+        "Full-text search over file names, summaries and extracted text. "
+        "Args: query (required), kind (optional: document, data or text).",
+        _tool_search,
+    ),
+    "folderlake_list_files": (
+        "List catalogued files, optionally filtered by kind and/or a "
+        "topic tag. Args: kind, topic, limit (default 50, max 200).",
+        _tool_list_files,
+    ),
+    "folderlake_file_detail": (
+        "Detail for one file by id: tags, entities, summary, a text "
+        "preview for documents, and a data-review block (schema, row "
+        "count, sample query) for data files.",
+        _tool_file_detail,
+    ),
+    "folderlake_related": (
+        "Files related to the given file id, from the catalog's "
+        "relations (shared topic/entity, explicit reference, same "
+        "folder), each with the relation kind and detail.",
+        _tool_related,
+    ),
+    "folderlake_topics": (
+        "The topic organisation of the catalog: the hierarchical topic "
+        "tree when one has been built, otherwise the flat topic list "
+        "with per-topic file counts.",
+        _tool_topics,
+    ),
+}
+
+
+# ----------------------------------------------------------------------
+# SDK wiring (optional dependency, imported lazily)
+# ----------------------------------------------------------------------
+def serve_stdio(db_path: Path) -> None:
+    """Serve the TOOLS handlers over MCP stdio via the official SDK."""
+    server_cls = None
+    try:  # mcp 1.x
+        from mcp.server.fastmcp import FastMCP
+
+        server_cls = FastMCP
+    except ImportError:
+        try:  # mcp >= 2: FastMCP was renamed MCPServer (same shape)
+            from mcp.server.mcpserver import MCPServer
+
+            server_cls = MCPServer
+        except ImportError:
+            pass
+    if server_cls is None:
+        raise RuntimeError(
+            "The MCP SDK is not installed. Install it with: "
+            "pip install folderlake[mcp]"
+        )
+
+    server = server_cls("folderlake")
+
+    @server.tool(description=TOOLS["folderlake_stats"][0])
+    def folderlake_stats() -> dict:
+        return _tool_stats(db_path)
+
+    @server.tool(description=TOOLS["folderlake_search"][0])
+    def folderlake_search(query: str, kind: str | None = None) -> dict:
+        return _tool_search(db_path, query, kind)
+
+    @server.tool(description=TOOLS["folderlake_list_files"][0])
+    def folderlake_list_files(kind: str | None = None,
+                              topic: str | None = None,
+                              limit: int = DEFAULT_LIMIT) -> dict:
+        return _tool_list_files(db_path, kind, topic, limit)
+
+    @server.tool(description=TOOLS["folderlake_file_detail"][0])
+    def folderlake_file_detail(file_id: str) -> dict:
+        return _tool_file_detail(db_path, file_id)
+
+    @server.tool(description=TOOLS["folderlake_related"][0])
+    def folderlake_related(file_id: str) -> dict:
+        return _tool_related(db_path, file_id)
+
+    @server.tool(description=TOOLS["folderlake_topics"][0])
+    def folderlake_topics() -> dict:
+        return _tool_topics(db_path)
+
+    server.run()  # stdio transport by default
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="folderlake-mcp",
+        description="Serve a FolderLake catalog to other agents over MCP (stdio).",
+    )
+    parser.add_argument(
+        "--db", default=None,
+        help="path to the catalog (default: $FOLDERLAKE_DB or ./folderlake.duckdb)",
+    )
+    args = parser.parse_args(argv)
+    raw = args.db or os.environ.get("FOLDERLAKE_DB") or "folderlake.duckdb"
+    db_path = Path(raw)
+    if not db_path.exists():
+        print(f"catalog not found: {db_path}", file=sys.stderr)
+        return 2
+    serve_stdio(db_path)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

folderlake/preview.py

@@ -0,0 +1,57 @@
+"""Preview payload for one file.
+
+Documents/text: summary + extracted text.
+Data files get a small side feature: schema, row count, per-column nulls,
+and a ready-to-run DuckDB query — a review, not an analytics product.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import duckdb
+
+
+def build_preview(db_path: Path, file_id: str) -> dict | None:
+    con = duckdb.connect(str(db_path), read_only=True)
+    try:
+        row = con.execute(
+            "SELECT path, name, kind, size_bytes, title, author, extra_json "
+            "FROM files WHERE id = ?",
+            [file_id],
+        ).fetchone()
+        if row is None:
+            return None
+        path, name, kind, size, title, author, extra_json = row
+        extra = json.loads(extra_json or "{}")
+        text_row = con.execute(
+            "SELECT summary, text FROM file_text WHERE file_id = ?", [file_id]
+        ).fetchone()
+        summary, text = text_row if text_row else ("", "")
+        tags = [
+            r[0]
+            for r in con.execute(
+                "SELECT tag FROM file_topics WHERE file_id = ? ORDER BY weight DESC",
+                [file_id],
+            ).fetchall()
+        ]
+        preview: dict = {
+            "id": file_id, "name": name, "path": path, "kind": kind,
+            "size_bytes": size, "title": title, "author": author,
+            "summary": summary, "tags": tags,
+        }
+        if kind == "data":
+            escaped = path.replace("'", "''")
+            preview["data_review"] = {
+                "columns": extra.get("columns", []),
+                "row_count": extra.get("row_count"),
+                "nulls": extra.get("nulls", {}),
+                "summarize": extra.get("summarize", []),
+                "sample_query": f"SELECT * FROM '{escaped}' LIMIT 100;",
+            }
+        else:
+            preview["text_preview"] = (text or "")[:4000]
+        return preview
+    finally:
+        con.close()

folderlake/registry.py

@@ -0,0 +1,157 @@
+"""Local registry for the FolderLake app (home page).
+
+A small SQLite database — stdlib only — that remembers every catalog
+the app has built (so the home page can list history and reopen one in
+a click) and the user's settings (LLM provider/model, defaults). Each
+catalog record also stores a snapshot of the settings its last scan
+ran with, so the history shows *how* a catalog was generated, not just
+where it lives.
+
+Location: ``$FOLDERLAKE_HOME/registry.sqlite3``, defaulting to
+``~/.folderlake/registry.sqlite3``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sqlite3
+import time
+from pathlib import Path
+
+SETTING_KEYS = ("llm_provider", "llm_model", "llm_base_url",
+                "llm_api_key", "use_llm")
+
+
+def registry_home() -> Path:
+    home = Path(os.environ.get("FOLDERLAKE_HOME",
+                               Path.home() / ".folderlake"))
+    home.mkdir(parents=True, exist_ok=True)
+    return home
+
+
+def catalog_id_for(path: Path) -> str:
+    return hashlib.sha1(str(path.resolve()).encode()).hexdigest()[:16]
+
+
+class Registry:
+    def __init__(self, db_path: Path | None = None):
+        self.db_path = db_path or (registry_home() / "registry.sqlite3")
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._con = sqlite3.connect(str(self.db_path),
+                                    check_same_thread=False)
+        self._con.row_factory = sqlite3.Row
+        self._con.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS catalogs (
+                id TEXT PRIMARY KEY,
+                path TEXT UNIQUE NOT NULL,
+                name TEXT NOT NULL,
+                created_at REAL NOT NULL,
+                updated_at REAL NOT NULL,
+                last_opened_at REAL,
+                file_count INTEGER DEFAULT 0,
+                catalog_bytes INTEGER DEFAULT 0,
+                settings TEXT DEFAULT '{}'
+            );
+            CREATE TABLE IF NOT EXISTS settings (
+                key TEXT PRIMARY KEY,
+                value TEXT
+            );
+            """
+        )
+        self._con.commit()
+
+    # -- catalogs ----------------------------------------------------
+    def list_catalogs(self) -> list[dict]:
+        rows = self._con.execute(
+            "SELECT * FROM catalogs ORDER BY updated_at DESC").fetchall()
+        out = []
+        for r in rows:
+            d = dict(r)
+            d["settings"] = json.loads(d["settings"] or "{}")
+            d["exists"] = Path(d["path"]).is_dir()
+            out.append(d)
+        return out
+
+    def get_catalog(self, catalog_id: str) -> dict | None:
+        row = self._con.execute(
+            "SELECT * FROM catalogs WHERE id = ?", [catalog_id]).fetchone()
+        if row is None:
+            return None
+        d = dict(row)
+        d["settings"] = json.loads(d["settings"] or "{}")
+        d["exists"] = Path(d["path"]).is_dir()
+        return d
+
+    def upsert_catalog(self, path: Path, file_count: int,
+                       catalog_bytes: int, settings: dict) -> dict:
+        path = path.resolve()
+        cid = catalog_id_for(path)
+        now = time.time()
+        self._con.execute(
+            """
+            INSERT INTO catalogs (id, path, name, created_at, updated_at,
+                                  file_count, catalog_bytes, settings)
+            VALUES (?,?,?,?,?,?,?,?)
+            ON CONFLICT(path) DO UPDATE SET
+                updated_at=excluded.updated_at,
+                file_count=excluded.file_count,
+                catalog_bytes=excluded.catalog_bytes,
+                settings=excluded.settings
+            """,
+            [cid, str(path), path.name, now, now, file_count,
+             catalog_bytes, json.dumps(settings, ensure_ascii=False)],
+        )
+        self._con.commit()
+        return self.get_catalog(cid)  # type: ignore[return-value]
+
+    def touch_opened(self, catalog_id: str) -> None:
+        self._con.execute(
+            "UPDATE catalogs SET last_opened_at = ? WHERE id = ?",
+            [time.time(), catalog_id])
+        self._con.commit()
+
+    def delete_catalog(self, catalog_id: str) -> dict | None:
+        rec = self.get_catalog(catalog_id)
+        if rec is None:
+            return None
+        self._con.execute("DELETE FROM catalogs WHERE id = ?",
+                          [catalog_id])
+        self._con.commit()
+        return rec
+
+    # -- settings ------------------------------------------------------
+    def get_settings(self) -> dict:
+        rows = self._con.execute("SELECT key, value FROM settings").fetchall()
+        saved = {r["key"]: r["value"] for r in rows}
+        return {
+            "llm_provider": saved.get("llm_provider", "ollama"),
+            "llm_model": saved.get("llm_model", ""),
+            "llm_base_url": saved.get("llm_base_url", ""),
+            "llm_api_key_set": bool(saved.get("llm_api_key")),
+            "use_llm": saved.get("use_llm", "0") == "1",
+        }
+
+    def get_api_key(self) -> str:
+        row = self._con.execute(
+            "SELECT value FROM settings WHERE key = 'llm_api_key'"
+        ).fetchone()
+        return row["value"] if row and row["value"] else ""
+
+    def save_settings(self, values: dict) -> dict:
+        for key in SETTING_KEYS:
+            if key not in values:
+                continue
+            v = values[key]
+            if key == "llm_api_key" and (v is None or v == ""):
+                continue  # blank field = keep the saved key
+            if key == "use_llm":
+                v = "1" if v else "0"
+            self._con.execute(
+                "INSERT INTO settings (key, value) VALUES (?,?) "
+                "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+                [key, "" if v is None else str(v)])
+        self._con.commit()
+        return self.get_settings()

folderlake/relations.py

@@ -0,0 +1,103 @@
+"""Four simple, labelled relation kinds between files. Nothing fancier.
+
+shared_topic / shared_entity / reference / same_folder — every edge
+stores its reason in `detail`, so the graph never shows a mystery link.
+Implemented as plain SQL over the catalog tables.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import duckdb
+
+# Dataset names that are also ordinary English words: a capitalised
+# mention of these is too weak to count as a subject reference.
+_COMMON_STEMS = {"tips", "flights", "exercise", "attention", "data",
+                 "file", "files", "table"}
+
+
+def build_relations(db_path: Path) -> int:
+    con = duckdb.connect(str(db_path))
+    try:
+        con.execute("DELETE FROM relations")
+        con.execute(
+            """
+            INSERT INTO relations
+            SELECT a.file_id, b.file_id, 'shared_topic', a.tag
+            FROM file_topics a JOIN file_topics b
+              ON a.tag = b.tag AND a.file_id < b.file_id
+            """
+        )
+        con.execute(
+            """
+            INSERT INTO relations
+            SELECT a.file_id, b.file_id, 'shared_entity', a.entity
+            FROM file_entities a JOIN file_entities b
+              ON a.entity = b.entity AND a.file_id < b.file_id
+            """
+        )
+        # explicit reference: file A's text mentions file B's name
+        rows = con.execute(
+            """
+            SELECT a.file_id, b.id, b.name
+            FROM file_text a JOIN files b ON a.file_id <> b.id
+            WHERE a.text ILIKE '%' || b.name || '%'
+            """
+        ).fetchall()
+        ref_pairs: set[tuple[str, str]] = set()
+        for fa, fb, name in rows:
+            a, b = (fa, fb) if fa < fb else (fb, fa)
+            ref_pairs.add((a, b))
+            con.execute(
+                "INSERT INTO relations VALUES (?,?, 'reference', ?)", [a, b, name]
+            )
+        # subject reference: a text/document that mentions a data
+        # file's subject by its distinctive name links to that file —
+        # a Wikipedia article about the Titanic points at titanic.csv.
+        # The mention must be capitalised in the text ("Titanic",
+        # "Penguins", "MPG"), which keeps ordinary lowercase uses of
+        # common words from creating junk links.
+        data_files = con.execute(
+            "SELECT id, name FROM files WHERE kind = 'data'"
+        ).fetchall()
+        texts = con.execute(
+            """
+            SELECT t.file_id, t.text FROM file_text t
+            JOIN files f ON f.id = t.file_id
+            WHERE f.kind IN ('text', 'document')
+            """
+        ).fetchall()
+        for data_id, data_name in data_files:
+            stem = Path(data_name).stem.lower()
+            if len(stem) < 3 or stem in _COMMON_STEMS:
+                continue
+            forms = {stem.capitalize(), stem.upper(), stem.title()}
+            pat = re.compile(
+                r"\b(?:" + "|".join(re.escape(f) for f in forms) + r")\b")
+            for file_id, text in texts:
+                if file_id == data_id or not pat.search(text or ""):
+                    continue
+                a, b = ((file_id, data_id) if file_id < data_id
+                        else (data_id, file_id))
+                if (a, b) in ref_pairs:
+                    continue
+                ref_pairs.add((a, b))
+                con.execute(
+                    "INSERT INTO relations VALUES (?,?, 'reference', ?)",
+                    [a, b, f"mentions {stem}"],
+                )
+        con.execute(
+            """
+            INSERT INTO relations
+            SELECT a.id, b.id, 'same_folder',
+                   regexp_replace(a.path, '/[^/]*$', '')
+            FROM files a JOIN files b ON a.id < b.id
+            WHERE regexp_replace(a.path, '/[^/]*$', '')
+                = regexp_replace(b.path, '/[^/]*$', '')
+            """
+        )
+        return con.execute("SELECT count(*) FROM relations").fetchone()[0]
+    finally:
+        con.close()