weft-graph 1.0.0__py3-none-any.whl

weft/__init__.py

@@ -0,0 +1,3 @@
+"""Weft - A local-first knowledge graph for your browsing."""
+
+__version__ = "1.0.0"

weft/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m weft."""
+
+from weft.cli import main
+
+if __name__ == "__main__":
+    main()

weft/cli.py

@@ -0,0 +1,372 @@
+"""CLI dispatcher for weft."""
+
+import argparse
+import json
+import os
+import sys
+
+from weft import __version__
+
+
+def cmd_weave(args):
+    """Build knowledge graph from browser tabs."""
+    from weft.export.browser import export_chrome, export_firefox
+    from weft.export.graph import GraphOptions, build_tab_graph, load_tabs_from_windows
+
+    # Export from browsers
+    windows = []
+    had_error = False
+
+    if args.browser in ("chrome", "all"):
+        try:
+            chrome_windows = export_chrome()
+            windows.extend(chrome_windows)
+            if args.verbose:
+                tab_count = sum(len(w.get("tabs", [])) for w in chrome_windows)
+                print(f"[INFO] Exported {tab_count} tabs from Chrome", file=sys.stderr)
+        except Exception as e:
+            had_error = True
+            print(f"[WARN] Chrome export failed: {e}", file=sys.stderr)
+
+    if args.browser in ("firefox", "all"):
+        try:
+            firefox_windows = export_firefox(args.firefox_profile, args.verbose)
+            windows.extend(firefox_windows)
+            if args.verbose:
+                tab_count = sum(len(w.get("tabs", [])) for w in firefox_windows)
+                print(f"[INFO] Exported {tab_count} tabs from Firefox", file=sys.stderr)
+        except Exception as e:
+            had_error = True
+            print(f"[WARN] Firefox export failed: {e}", file=sys.stderr)
+
+    if not windows:
+        print("[ERROR] No tabs exported from any browser.", file=sys.stderr)
+        sys.exit(1)
+
+    # Flatten to tabs
+    tabs = load_tabs_from_windows(windows)
+
+    if args.verbose:
+        print(f"[INFO] Processing {len(tabs)} tabs...", file=sys.stderr)
+
+    # Build graph options
+    options = GraphOptions(
+        out=args.out,
+        cache=args.cache,
+        refresh=args.refresh,
+        no_crawl=args.no_crawl,
+        max_chars=args.max_chars,
+        embed_max_chars=args.embed_max_chars,
+        user_agent=args.user_agent,
+        timeout=args.timeout,
+        js=args.js,
+        summarize=args.summarize,
+        llm_backend=args.llm_backend,
+        ollama_model=args.ollama_model,
+        ollama_url=args.ollama_url,
+        ollama_timeout=args.ollama_timeout,
+        embed_model=args.embed_model,
+        embed_url=args.embed_url,
+        embed_timeout=args.embed_timeout,
+        no_embeddings=args.no_embeddings,
+        store_embeddings=args.store_embeddings,
+        gguf=args.gguf,
+        llama_n_ctx=args.llama_n_ctx,
+        llama_n_threads=args.llama_n_threads,
+        llama_n_gpu_layers=args.llama_n_gpu_layers,
+        edge_threshold=args.edge_threshold,
+        group_threshold=args.group_threshold,
+        domain_bonus=args.domain_bonus,
+        no_domain_group=args.no_domain_group,
+        domain_group_min=args.domain_group_min,
+        knn_k=args.knn_k,
+        no_mutual_knn=args.no_mutual_knn,
+        dedupe_hamming=args.dedupe_hamming,
+        keyword_count=args.keyword_count,
+        verbose=args.verbose,
+    )
+
+    # Build graph
+    graph = build_tab_graph(tabs, options)
+
+    # Write output
+    with open(args.out, "w", encoding="utf-8") as f:
+        json.dump(graph, f, ensure_ascii=False, indent=2)
+
+    stats = graph["stats"]
+    mode = "with LLM summaries" if args.summarize else "lightweight (keywords only)"
+    print(
+        f"[OK] Wrote {args.out} {mode}\n"
+        f"     {stats['tab_count']} tabs, {stats['group_count']} groups, "
+        f"{stats['edge_count']} edges, {stats['duplicates']} duplicates"
+    )
+    if stats["errors"]:
+        print(f"     {stats['errors']} errors during processing", file=sys.stderr)
+    if had_error:
+        print("[NOTE] Some browsers failed to export; see warnings above.", file=sys.stderr)
+
+
+def cmd_explore(args):
+    """Launch the terminal UI."""
+    from weft.tui.app import TabGraphApp, load_graph
+
+    if not os.path.exists(args.graph_json):
+        print(f"[ERROR] Graph file not found: {args.graph_json}", file=sys.stderr)
+        print("Run 'weft weave' first to generate the graph.", file=sys.stderr)
+        sys.exit(1)
+
+    graph = load_graph(args.graph_json)
+    app = TabGraphApp(graph)
+    app.run()
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="weft",
+        description="A local-first knowledge graph for your browsing",
+    )
+    parser.add_argument("--version", action="version", version=f"weft {__version__}")
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Weave command
+    weave_parser = subparsers.add_parser(
+        "weave",
+        help="Build knowledge graph from browser tabs",
+        description="Extract tabs from Chrome/Firefox and weave them into a knowledge graph with clustering.",
+    )
+
+    # Browser options
+    weave_parser.add_argument(
+        "--browser",
+        choices=["chrome", "firefox", "all"],
+        default="all",
+        help="Which browser(s) to export from (default: all)",
+    )
+    weave_parser.add_argument(
+        "--firefox-profile",
+        help="Override: path to a specific Firefox profile directory",
+    )
+
+    # Output options
+    weave_parser.add_argument(
+        "--out",
+        default="weft_graph.json",
+        help="Output graph JSON path (default: weft_graph.json)",
+    )
+
+    # LLM options
+    weave_parser.add_argument(
+        "--summarize",
+        action="store_true",
+        help="Enable LLM summarization (requires Ollama or GGUF model)",
+    )
+    weave_parser.add_argument(
+        "--llm-backend",
+        choices=["ollama", "gguf"],
+        default="ollama",
+        help="LLM backend (default: ollama)",
+    )
+    weave_parser.add_argument(
+        "--ollama-model",
+        default="llama3.1:8b",
+        help="Ollama model name (default: llama3.1:8b)",
+    )
+    weave_parser.add_argument(
+        "--ollama-url",
+        default="http://localhost:11434",
+        help="Ollama base URL",
+    )
+    weave_parser.add_argument(
+        "--ollama-timeout",
+        type=int,
+        default=120,
+        help="Ollama request timeout in seconds",
+    )
+    weave_parser.add_argument(
+        "--gguf",
+        help="Path to a GGUF model file (for --llm-backend=gguf)",
+    )
+    weave_parser.add_argument(
+        "--llama-n-ctx",
+        type=int,
+        default=4096,
+        help="Context size for llama-cpp",
+    )
+    weave_parser.add_argument(
+        "--llama-n-threads",
+        type=int,
+        default=max(1, os.cpu_count() or 1),
+        help="Threads for llama-cpp",
+    )
+    weave_parser.add_argument(
+        "--llama-n-gpu-layers",
+        type=int,
+        default=0,
+        help="GPU layers for llama-cpp",
+    )
+
+    # Embedding options
+    weave_parser.add_argument(
+        "--embed-model",
+        default="nomic-embed-text",
+        help="Ollama embedding model name",
+    )
+    weave_parser.add_argument(
+        "--embed-url",
+        help="Ollama base URL for embeddings (defaults to --ollama-url)",
+    )
+    weave_parser.add_argument(
+        "--embed-timeout",
+        type=int,
+        default=60,
+        help="Embedding request timeout",
+    )
+    weave_parser.add_argument(
+        "--no-embeddings",
+        action="store_true",
+        help="Disable embeddings (use keyword similarity only)",
+    )
+    weave_parser.add_argument(
+        "--store-embeddings",
+        action="store_true",
+        help="Include embeddings in output JSON",
+    )
+
+    # Crawling options
+    weave_parser.add_argument(
+        "--no-crawl",
+        action="store_true",
+        help="Skip URL crawling (use metadata only)",
+    )
+    weave_parser.add_argument(
+        "--max-chars",
+        type=int,
+        default=6000,
+        help="Max characters sent to LLM",
+    )
+    weave_parser.add_argument(
+        "--embed-max-chars",
+        type=int,
+        default=2000,
+        help="Max characters sent to embed model",
+    )
+    weave_parser.add_argument(
+        "--user-agent",
+        default="Mozilla/5.0",
+        help="User-Agent for HTTP requests",
+    )
+    weave_parser.add_argument(
+        "--timeout",
+        type=int,
+        default=20,
+        help="HTTP timeout in seconds",
+    )
+    weave_parser.add_argument(
+        "--js",
+        action="store_true",
+        help="Use Playwright to render JS-heavy pages",
+    )
+
+    # Caching options
+    weave_parser.add_argument(
+        "--cache",
+        default=os.path.join("data", "weft_cache.json"),
+        help="Cache file path",
+    )
+    weave_parser.add_argument(
+        "--refresh",
+        action="store_true",
+        help="Ignore cache and re-fetch everything",
+    )
+
+    # Clustering options
+    weave_parser.add_argument(
+        "--edge-threshold",
+        type=float,
+        default=0.2,
+        help="Edge weight threshold",
+    )
+    weave_parser.add_argument(
+        "--group-threshold",
+        type=float,
+        default=0.25,
+        help="Grouping similarity threshold",
+    )
+    weave_parser.add_argument(
+        "--domain-bonus",
+        type=float,
+        default=0.25,
+        help="Similarity bonus for same-domain tabs",
+    )
+    weave_parser.add_argument(
+        "--no-domain-group",
+        action="store_true",
+        help="Disable auto-grouping by domain",
+    )
+    weave_parser.add_argument(
+        "--domain-group-min",
+        type=int,
+        default=2,
+        help="Min tabs per domain to auto-group",
+    )
+    weave_parser.add_argument(
+        "--knn-k",
+        type=int,
+        default=6,
+        help="Mutual KNN size for grouping",
+    )
+    weave_parser.add_argument(
+        "--no-mutual-knn",
+        action="store_true",
+        help="Disable mutual-KNN grouping filter",
+    )
+    weave_parser.add_argument(
+        "--dedupe-hamming",
+        type=int,
+        default=3,
+        help="Simhash Hamming distance for dedupe",
+    )
+    weave_parser.add_argument(
+        "--keyword-count",
+        type=int,
+        default=8,
+        help="Number of keywords per tab",
+    )
+
+    # General options
+    weave_parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Verbose logging",
+    )
+
+    # Explore command
+    explore_parser = subparsers.add_parser(
+        "explore",
+        help="Launch terminal UI to explore the graph",
+        description="Interactive terminal UI for browsing and searching grouped tabs.",
+    )
+    explore_parser.add_argument(
+        "graph_json",
+        nargs="?",
+        default="weft_graph.json",
+        help="Path to graph JSON file (default: weft_graph.json)",
+    )
+
+    # Parse and dispatch
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.command == "weave":
+        cmd_weave(args)
+    elif args.command == "explore":
+        cmd_explore(args)
+
+
+if __name__ == "__main__":
+    main()

weft/export/__init__.py

@@ -0,0 +1 @@
+"""Export module for tab extraction and graph building."""

weft/export/browser.py

@@ -0,0 +1,156 @@
+"""Browser tab export functions for Chrome and Firefox on macOS."""
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+FF_BASE = Path.home() / "Library" / "Application Support" / "Firefox" / "Profiles"
+
+
+def run_jxa(script: str) -> str:
+    """Run JavaScript for Automation via osascript."""
+    try:
+        p = subprocess.run(
+            ["osascript", "-l", "JavaScript", "-e", script],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return p.stdout.strip()
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(e.stderr.strip() or str(e))
+
+
+def export_chrome() -> List[Dict]:
+    """Export all open tabs from Google Chrome using AppleScript."""
+    jxa = """
+    function run() {
+      var app = Application.currentApplication();
+      app.includeStandardAdditions = true;
+      var output = [];
+      try {
+        var chrome = Application('Google Chrome');
+        chrome.windows().forEach(function(w){
+          var tabs = w.tabs().map(function(t){ return {title: t.title(), url: t.url()}; });
+          if (tabs.length > 0) {
+            output.push({browser: 'chrome', windowId: w.id(), tabs: tabs});
+          }
+        });
+      } catch (e) {
+        // Chrome not running or AppleScript disabled
+      }
+      return JSON.stringify(output);
+    }
+    """
+    out = run_jxa(jxa)
+    return json.loads(out) if out else []
+
+
+def find_all_firefox_profiles() -> List[Path]:
+    """Find all Firefox profile directories."""
+    if not FF_BASE.exists():
+        return []
+    return [p for p in FF_BASE.iterdir() if p.is_dir()]
+
+
+def best_sessionstore_for_profile(prof: Path) -> Optional[Tuple[Path, float]]:
+    """Return (path, mtime) of the freshest sessionstore candidate or None."""
+    candidates = []
+    ssb = prof / "sessionstore-backups"
+    for name in ("recovery.jsonlz4", "previous.jsonlz4"):
+        p = ssb / name
+        if p.exists():
+            candidates.append(p)
+    candidates.extend((ssb).glob("upgrade.jsonlz4*"))
+    p_root = prof / "sessionstore.jsonlz4"
+    if p_root.exists():
+        candidates.append(p_root)
+    if not candidates:
+        return None
+    candidates = [(p, p.stat().st_mtime) for p in candidates if p.is_file()]
+    if not candidates:
+        return None
+    candidates.sort(key=lambda t: t[1], reverse=True)
+    return candidates[0]
+
+
+def choose_firefox_profile_with_fresh_session() -> Optional[Tuple[Path, Path]]:
+    """Return (profile_dir, sessionstore_path) with the freshest valid session."""
+    profiles = find_all_firefox_profiles()
+    scored = []
+    for prof in profiles:
+        best = best_sessionstore_for_profile(prof)
+        if best:
+            scored.append((prof, best[0], best[1]))
+    if not scored:
+        return None
+    scored.sort(key=lambda t: t[2], reverse=True)
+    return scored[0][0], scored[0][1]
+
+
+def export_firefox(
+    profile_override: Optional[str] = None, verbose: bool = False
+) -> List[Dict]:
+    """Export all open tabs from Firefox session backup."""
+    try:
+        from lz4.block import decompress as lz4_decompress
+    except ImportError as e:
+        raise RuntimeError(
+            "Python package 'lz4' is required for Firefox export. Install with: pip install lz4"
+        ) from e
+
+    session_path: Optional[Path] = None
+    selected_profile: Optional[Path] = None
+
+    if profile_override:
+        p = Path(profile_override).expanduser()
+        if not p.exists():
+            raise FileNotFoundError(f"Provided --firefox-profile does not exist: {p}")
+        best = best_sessionstore_for_profile(p)
+        if not best:
+            raise FileNotFoundError(
+                f"No sessionstore files found in provided profile: {p}"
+            )
+        selected_profile, session_path = p, best[0]
+    else:
+        chosen = choose_firefox_profile_with_fresh_session()
+        if not chosen:
+            raise FileNotFoundError(
+                "No Firefox sessionstore *.jsonlz4 files found in ANY profile. "
+                "Open Firefox (non-private window), ensure tabs are open, wait 10s, and try again."
+            )
+        selected_profile, session_path = chosen
+
+    if verbose:
+        print(f"[INFO] Using Firefox profile: {selected_profile}", file=sys.stderr)
+        print(
+            f"[INFO] Sessionstore file:    {session_path} (mtime={time.ctime(session_path.stat().st_mtime)})",
+            file=sys.stderr,
+        )
+
+    raw = session_path.read_bytes()
+    if raw[:8] != b"mozLz40\x00":
+        raise RuntimeError("Unexpected Firefox sessionstore header; not mozlz4.")
+
+    decomp = lz4_decompress(raw[8:])
+    session = json.loads(decomp.decode("utf-8"))
+
+    windows = session.get("windows", [])
+    payload = []
+    for w in windows:
+        tabs = []
+        for t in w.get("tabs", []):
+            idx = t.get("index", 1)
+            entries = t.get("entries", [])
+            if 1 <= idx <= len(entries):
+                e = entries[idx - 1]
+                url = e.get("url", "")
+                title = e.get("title", "")
+                if url:
+                    tabs.append({"title": title, "url": url})
+        if tabs:
+            payload.append({"browser": "firefox", "windowId": None, "tabs": tabs})
+    return payload

weft/export/cache.py

@@ -0,0 +1,38 @@
+"""Cache management for tab graph building."""
+
+import json
+import os
+from typing import Dict, Optional, Tuple
+
+from weft.utils.url import canonicalize_url
+
+
+def load_cache(path: Optional[str]) -> Dict[str, Dict]:
+    """Load cache from JSON file."""
+    if not path or not os.path.exists(path):
+        return {}
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_cache_entry(
+    cache: Dict[str, Dict], url: str
+) -> Tuple[Optional[Dict], Optional[str]]:
+    """Get cache entry for URL, trying both original and canonical forms."""
+    if not cache:
+        return None, None
+    for key in (url, canonicalize_url(url)):
+        if key in cache:
+            return cache[key], key
+    return None, None
+
+
+def save_cache(path: Optional[str], cache: Dict[str, Dict]) -> None:
+    """Save cache to JSON file."""
+    if not path:
+        return
+    dir_name = os.path.dirname(path)
+    if dir_name:
+        os.makedirs(dir_name, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(cache, f, ensure_ascii=False, indent=2)

weft/export/crawl.py

@@ -0,0 +1,55 @@
+"""HTML fetching and text extraction functions."""
+
+import re
+from typing import Optional
+
+import requests
+
+try:
+    import trafilatura
+except ImportError:
+    trafilatura = None
+
+
+def fetch_html_requests(url: str, timeout: int, user_agent: Optional[str] = None) -> str:
+    """Fetch HTML content from URL using requests library."""
+    headers = {}
+    if user_agent:
+        headers["User-Agent"] = user_agent
+    resp = requests.get(url, headers=headers, timeout=timeout)
+    resp.raise_for_status()
+    content_type = resp.headers.get("content-type", "")
+    if "text/html" not in content_type and "application/xhtml+xml" not in content_type:
+        return ""
+    return resp.text
+
+
+def extract_text(html: str, url: str) -> str:
+    """Extract readable text from HTML using trafilatura."""
+    if trafilatura is None:
+        raise ImportError("trafilatura is required. Install with: pip install trafilatura")
+
+    text = trafilatura.extract(
+        html,
+        url=url,
+        include_comments=False,
+        include_tables=False,
+        include_links=False,
+    )
+    if text:
+        return text
+    # Fallback for edge cases where trafilatura fails
+    text = re.sub(r"<[^>]+>", " ", html)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def truncate_text(text: str, max_chars: int) -> str:
+    """Truncate text to max_chars, breaking at word boundary."""
+    if len(text) <= max_chars:
+        return text
+    clipped = text[: max_chars - 1]
+    last_space = clipped.rfind(" ")
+    if last_space > 200:
+        clipped = clipped[:last_space]
+    return clipped