timelog-extract 0.2.3__py3-none-any.whl

collectors/__init__.py

@@ -0,0 +1 @@
+"""Collectors package for source-specific event extraction."""

collectors/ai_logs.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _read_jsonl_timestamps(jsonl_file, dt_from, dt_to):
+    results = []
+    try:
+        with open(jsonl_file, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+
+                ts_raw = obj.get("timestamp") or obj.get("ts") or obj.get("created_at") or obj.get("time")
+                if ts_raw is None:
+                    continue
+                try:
+                    if isinstance(ts_raw, (int, float)):
+                        divisor = 1000 if ts_raw > 1e11 else 1
+                        ts = datetime.fromtimestamp(ts_raw / divisor, tz=timezone.utc)
+                    else:
+                        ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+                except (ValueError, OSError):
+                    continue
+
+                if not (dt_from <= ts <= dt_to):
+                    continue
+
+                msg = obj.get("message", {})
+                if isinstance(msg, dict):
+                    content = msg.get("content", "")
+                    if isinstance(content, list):
+                        content = " ".join(c.get("text", "") for c in content if isinstance(c, dict))
+                    detail = str(content)[:70].replace("\n", " ")
+                elif isinstance(msg, str):
+                    detail = msg[:70]
+                else:
+                    detail = str(obj.get("type", ""))[:70]
+
+                results.append((ts, detail or "log", obj))
+    except (OSError, PermissionError):
+        pass
+    return results
+
+
+def collect_claude_code(profiles, dt_from, dt_to, home, classify_project, make_event):
+    results = []
+    projects_dir = home / ".claude" / "projects"
+    if not projects_dir.exists():
+        return results
+    for proj_dir in projects_dir.iterdir():
+        if not proj_dir.is_dir():
+            continue
+        dir_name = proj_dir.name.lower()
+        for jsonl_file in proj_dir.glob("*.jsonl"):
+            for ts, detail, _ in _read_jsonl_timestamps(jsonl_file, dt_from, dt_to):
+                project = classify_project(f"{dir_name} {detail}", profiles)
+                results.append(make_event("Claude Code CLI", ts, detail, project))
+    return results
+
+
+def collect_claude_desktop(profiles, dt_from, dt_to, home, classify_project, make_event):
+    results = []
+    sessions_dir = home / "Library" / "Application Support" / "Claude" / "local-agent-mode-sessions"
+    if not sessions_dir.exists():
+        return results
+    for jsonl_file in sessions_dir.glob("**/*.jsonl"):
+        for ts, detail, _ in _read_jsonl_timestamps(jsonl_file, dt_from, dt_to):
+            project = classify_project(detail, profiles)
+            results.append(make_event("Claude Desktop", ts, detail, project))
+    return results
+
+
+def collect_gemini_cli(profiles, dt_from, dt_to, home, classify_project, make_event):
+    results = []
+    base_dir = home / ".gemini" / "tmp"
+    if not base_dir.exists():
+        return results
+    for chat_file in base_dir.glob("*/chats/session-*.json"):
+        proj_name = chat_file.parent.parent.name.lower()
+        try:
+            data = json.loads(chat_file.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        for msg in data.get("messages", []):
+            ts_raw = msg.get("timestamp")
+            if not ts_raw:
+                continue
+            try:
+                ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+            except ValueError:
+                continue
+            if not (dt_from <= ts <= dt_to):
+                continue
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                content = " ".join(c.get("text", "") for c in content if isinstance(c, dict))
+            detail = str(content)[:70].replace("\n", " ")
+            role = msg.get("type", "")
+            project = classify_project(f"{proj_name} {detail}", profiles)
+            results.append(make_event("Gemini CLI", ts, f"[{role}] {detail}" if detail else "Gemini CLI", project))
+    return results
+
+
+def collect_codex_ide(
+    profiles,
+    dt_from,
+    dt_to,
+    codex_session_index: Path,
+    classify_project,
+    make_event,
+):
+    if not codex_session_index.is_file():
+        return []
+    results = []
+    try:
+        text = codex_session_index.read_text(encoding="utf-8")
+    except OSError:
+        return []
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        ts_raw = obj.get("updated_at")
+        if not ts_raw:
+            continue
+        try:
+            ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
+        except ValueError:
+            continue
+        if not (dt_from <= ts <= dt_to):
+            continue
+        thread = str(obj.get("thread_name") or "").strip() or "session"
+        sid = str(obj.get("id") or "").replace("-", "")[:10]
+        detail = f"{thread[:65]} — id {sid}…" if sid else thread[:70]
+        project = classify_project(thread, profiles)
+        results.append(make_event("Codex IDE", ts, detail, project))
+    return results

collectors/chrome.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import os
+import shutil
+import sqlite3
+import tempfile
+from datetime import datetime, timezone
+from typing import Callable, Dict
+from urllib.parse import urlparse
+
+
+def _like_escape(value: str) -> str:
+    """Escape SQLite LIKE wildcard characters so values match literally.
+
+    Uses backslash as the escape character; callers must append ESCAPE '\\\\' to
+    the predicate string.
+    """
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+def chrome_history_path(home):
+    return home / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "History"
+
+
+def query_chrome(history_path, where_clause, dt_from_cu, dt_to_cu, params=()):
+    if not history_path.exists():
+        return []
+
+    tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
+    tmp.close()
+    rows = []
+    try:
+        shutil.copy2(history_path, tmp.name)
+        conn = sqlite3.connect(tmp.name)
+        cursor = conn.cursor()
+        cursor.execute(
+            f"""
+            SELECT v.visit_time, u.url, u.title
+            FROM visits v
+            JOIN urls u ON v.url = u.id
+            WHERE v.visit_time BETWEEN ? AND ?
+              AND ({where_clause})
+            ORDER BY v.visit_time
+            """,
+            (dt_from_cu, dt_to_cu, *params),
+        )
+        rows = cursor.fetchall()
+        conn.close()
+    except Exception as exc:
+        print(f"  [Warning] Chrome history: {exc}")
+    finally:
+        try:
+            os.unlink(tmp.name)
+        except OSError:
+            pass
+    return rows
+
+
+def chrome_time_range(dt_from, dt_to, epoch_delta_us):
+    return (
+        int(dt_from.astimezone(timezone.utc).timestamp() * 1_000_000) + epoch_delta_us,
+        int(dt_to.astimezone(timezone.utc).timestamp() * 1_000_000) + epoch_delta_us,
+    )
+
+
+def chrome_ts(visit_time_cu, epoch_delta_us):
+    return datetime.fromtimestamp(
+        (visit_time_cu - epoch_delta_us) / 1_000_000, tz=timezone.utc
+    )
+
+
+def normalize_chrome_url(url):
+    if not url:
+        return ""
+    try:
+        parsed = urlparse(url)
+        path = parsed.path or "/"
+        if len(path) > 1 and path.endswith("/"):
+            path = path.rstrip("/")
+        return f"{parsed.netloc.lower()}{path.lower()}"
+    except Exception:
+        return (url or "")[:200].lower()
+
+
+def thin_chrome_visit_rows(rows, collapse_minutes, epoch_delta_us):
+    if collapse_minutes <= 0 or not rows:
+        return rows
+    window_s = collapse_minutes * 60
+    out = []
+    last_emit_ts_by_norm = {}
+    for visit_time_cu, url, title in rows:
+        ts = chrome_ts(visit_time_cu, epoch_delta_us)
+        norm = normalize_chrome_url(url)
+        if not norm:
+            out.append((visit_time_cu, url, title))
+            continue
+        prev = last_emit_ts_by_norm.get(norm)
+        if prev is not None and (ts - prev).total_seconds() < window_s:
+            continue
+        last_emit_ts_by_norm[norm] = ts
+        out.append((visit_time_cu, url, title))
+    return out
+
+
+def collect_claude_ai_urls(
+    profiles,
+    dt_from,
+    dt_to,
+    home,
+    epoch_delta_us,
+    uncategorized,
+    make_event: Callable,
+):
+    url_map: Dict[str, str] = {}
+    for profile in profiles:
+        for url in profile["tracked_urls"]:
+            if "claude.ai" not in str(url).lower():
+                continue
+            url_map[url] = profile["name"]
+    if not url_map:
+        return []
+
+    clauses = " OR ".join(["u.url LIKE ? ESCAPE '\\'" for _ in url_map])
+    clause_params = tuple(f"%{_like_escape(url)}%" for url in url_map)
+    dt_from_cu, dt_to_cu = chrome_time_range(dt_from, dt_to, epoch_delta_us)
+    history_path = chrome_history_path(home)
+    rows = query_chrome(history_path, clauses, dt_from_cu, dt_to_cu, clause_params)
+
+    results = []
+    for visit_time_cu, url, title in rows:
+        ts = chrome_ts(visit_time_cu, epoch_delta_us)
+        chat_id = url.split("/chat/")[-1].split("?")[0][:12] if "/chat/" in url else url[-20:]
+        project = next(
+            (name for tracked_url, name in url_map.items() if tracked_url in url),
+            uncategorized,
+        )
+        detail = f"chat/{chat_id}… — {(title or '')[:40]}"
+        results.append(make_event("Claude.ai (web)", ts, detail, project))
+    return results
+
+
+def collect_gemini_web_urls(
+    profiles,
+    dt_from,
+    dt_to,
+    home,
+    epoch_delta_us,
+    uncategorized,
+    make_event: Callable,
+):
+    url_map: Dict[str, str] = {}
+    for profile in profiles:
+        for url in profile["tracked_urls"]:
+            if "gemini.google.com" not in str(url).lower():
+                continue
+            url_map[url] = profile["name"]
+    if not url_map:
+        return []
+
+    clauses = " OR ".join(["u.url LIKE ? ESCAPE '\\'" for _ in url_map])
+    clause_params = tuple(f"%{_like_escape(url)}%" for url in url_map)
+    dt_from_cu, dt_to_cu = chrome_time_range(dt_from, dt_to, epoch_delta_us)
+    history_path = chrome_history_path(home)
+    rows = query_chrome(history_path, clauses, dt_from_cu, dt_to_cu, clause_params)
+
+    results = []
+    for visit_time_cu, url, title in rows:
+        ts = chrome_ts(visit_time_cu, epoch_delta_us)
+        match = None
+        best_len = -1
+        for tracked_url, name in url_map.items():
+            if tracked_url in url and len(tracked_url) > best_len:
+                match = name
+                best_len = len(tracked_url)
+        project = match or uncategorized
+        chat_id = url.split("/app/")[-1].split("?")[0][:20] if "/app/" in url else url[-24:]
+        detail = f"gemini/app/{chat_id}… — {(title or '')[:40]}"
+        results.append(make_event("Gemini (web)", ts, detail, project))
+    return results
+
+
+def collect_chrome(
+    profiles,
+    dt_from,
+    dt_to,
+    collapse_minutes,
+    home,
+    epoch_delta_us,
+    classify_project: Callable,
+    make_event: Callable,
+):
+    all_keywords = sorted(
+        {
+            kw.lower()
+            for profile in profiles
+            for kw in (profile["match_terms"] + [profile["name"]])
+            if kw
+        }
+    )
+    if not all_keywords:
+        return []
+
+    dt_from_cu, dt_to_cu = chrome_time_range(dt_from, dt_to, epoch_delta_us)
+    kw_clauses = " OR ".join(
+        ["(LOWER(u.url) LIKE ? ESCAPE '\\' OR LOWER(u.title) LIKE ? ESCAPE '\\')" for _ in all_keywords]
+    )
+    kw_params = tuple(p for kw in all_keywords for p in (f"%{_like_escape(kw)}%", f"%{_like_escape(kw)}%"))
+    where_clause = f"({kw_clauses}) AND u.url NOT LIKE ? AND u.url NOT LIKE ?"
+    clause_params = (*kw_params, "%claude.ai%", "%gemini.google.com%")
+    history_path = chrome_history_path(home)
+    rows = query_chrome(history_path, where_clause, dt_from_cu, dt_to_cu, clause_params)
+    rows = thin_chrome_visit_rows(rows, collapse_minutes, epoch_delta_us)
+    results = []
+    for visit_time_cu, url, title in rows:
+        ts = chrome_ts(visit_time_cu, epoch_delta_us)
+        detail = (title or url)[:70]
+        project = classify_project(f"{url} {title}", profiles)
+        results.append(make_event("Chrome", ts, detail, project))
+    return results

collectors/cursor.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+
+def load_cursor_workspaces(home: Path):
+    storage_dir = home / "Library" / "Application Support" / "Cursor" / "User" / "workspaceStorage"
+    workspace_map = {}
+    if not storage_dir.exists():
+        return workspace_map
+    for workspace_json in storage_dir.glob("*/workspace.json"):
+        workspace_id = workspace_json.parent.name
+        try:
+            data = json.loads(workspace_json.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        raw_uri = data.get("folder") or data.get("workspace")
+        if not raw_uri:
+            continue
+        parsed = urlparse(raw_uri)
+        path = unquote(parsed.path) if parsed.scheme == "file" else raw_uri
+        workspace_map[workspace_id] = path
+    return workspace_map
+
+
+def collect_cursor(profiles, dt_from, dt_to, home, local_tz, classify_project, make_event):
+    workspace_map = load_cursor_workspaces(home)
+    logs_dir = home / "Library" / "Application Support" / "Cursor" / "logs"
+    if not logs_dir.exists():
+        return []
+
+    results = []
+    ts_pattern = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+    ts_iso_bracket_pattern = re.compile(r"^\[(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?)(Z|[+-]\d{2}:\d{2})?\]")
+    workspace_id_pattern = re.compile(r"workspaceStorage/([0-9a-f]{32})|old id ([0-9a-f]{32})-")
+    workspace_path_pattern = re.compile(r"(/Users/[^\"'\s]+)")
+
+    def _parse_cursor_log_ts(line: str):
+        m = ts_pattern.match(line)
+        if m:
+            try:
+                return datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S").replace(tzinfo=local_tz)
+            except ValueError:
+                return None
+        m = ts_iso_bracket_pattern.match(line)
+        if m:
+            iso = (m.group(1) + (m.group(2) or "")).replace("Z", "+00:00")
+            try:
+                return datetime.fromisoformat(iso)
+            except ValueError:
+                return None
+        return None
+
+    for log_file in logs_dir.glob("**/*.log"):
+        try:
+            with open(log_file, encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    ts = _parse_cursor_log_ts(line)
+                    if not ts or not (dt_from <= ts <= dt_to):
+                        continue
+                    workspace_path = None
+                    m_id = workspace_id_pattern.search(line)
+                    if m_id and workspace_map:
+                        workspace_id = m_id.group(1) or m_id.group(2)
+                        workspace_path = workspace_map.get(workspace_id)
+                    if not workspace_path:
+                        m_path = workspace_path_pattern.search(line)
+                        if m_path:
+                            workspace_path = m_path.group(1)
+                    if not workspace_path:
+                        continue
+                    project = classify_project(f"{workspace_path} {line}", profiles)
+                    detail = f"{Path(workspace_path).name} — {line.strip()[:90]}"
+                    results.append(make_event("Cursor", ts, detail, project))
+        except OSError:
+            continue
+    return results
+
+
+def collect_cursor_checkpoints(
+    profiles,
+    dt_from,
+    dt_to,
+    checkpoints_dir: Path,
+    home: Path,
+    classify_project,
+    make_event,
+    source_name: str,
+):
+    if not checkpoints_dir.is_dir():
+        return []
+    workspace_map = load_cursor_workspaces(home)
+    results = []
+    for meta_path in checkpoints_dir.glob("*/metadata.json"):
+        try:
+            data = json.loads(meta_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        ms = data.get("startTrackingDateUnixMilliseconds")
+        if ms is None:
+            continue
+        try:
+            ts = datetime.fromtimestamp(ms / 1000.0, tz=timezone.utc)
+        except (OSError, ValueError, OverflowError):
+            continue
+        if not (dt_from <= ts <= dt_to):
+            continue
+        paths = []
+        for rf in data.get("requestFiles") or []:
+            p = rf.get("fsPath")
+            if p:
+                paths.append(str(p))
+        wid = data.get("workspaceId")
+        if wid:
+            mapped = workspace_map.get(wid)
+            if mapped:
+                paths.append(str(mapped))
+        hay = " ".join(paths)
+        if not hay:
+            continue
+        project = classify_project(hay, profiles)
+        agent_id = str(data.get("agentRequestId", "")).split("-")[0][:8]
+        label = Path(paths[0]).name if paths else "checkpoint"
+        detail = f"checkpoint {agent_id}… — {label}"
+        results.append(make_event(source_name, ts, detail, project))
+    return results

collectors/github.py

@@ -0,0 +1,169 @@
+"""GitHub public activity via REST API (optional; requires username)."""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from core.cli_options import package_version
+from core.sources import GITHUB_SOURCE
+
+USER_AGENT = (
+    f"timelog-extract/{package_version()} "
+    "(+https://github.com/mbjorke/timelog-extract)"
+)
+PER_PAGE = 100
+MAX_PAGES = 10
+
+
+def resolve_github_username(args: Any) -> str:
+    """CLI `--github-user` overrides `GITHUB_USER` / `GITHUB_LOGIN`."""
+    explicit = getattr(args, "github_user", None)
+    if explicit and str(explicit).strip():
+        return str(explicit).strip()
+    return (os.environ.get("GITHUB_USER") or os.environ.get("GITHUB_LOGIN") or "").strip()
+
+
+def github_source_enabled(args: Any) -> tuple[bool, Optional[str]]:
+    """Return (enabled, disable_reason)."""
+    mode = getattr(args, "github_source", "auto")
+    if mode == "off":
+        return False, "GitHub source disabled via --github-source off"
+    user = resolve_github_username(args)
+    if mode == "on" and not user:
+        return False, "GitHub on but no username (use --github-user or GITHUB_USER)"
+    if mode == "auto" and not user:
+        return False, "no GitHub username (set --github-user or GITHUB_USER for this source)"
+    if not user:
+        return False, "no GitHub username"
+    return True, None
+
+
+def _parse_github_ts(created_at: str) -> datetime:
+    if created_at.endswith("Z"):
+        created_at = created_at[:-1] + "+00:00"
+    return datetime.fromisoformat(created_at)
+
+
+def _detail_for_event(ev: Dict[str, Any]) -> str:
+    et = ev.get("type") or "unknown"
+    repo = (ev.get("repo") or {}).get("name") or "unknown/repo"
+    payload = ev.get("payload") or {}
+
+    if et == "PushEvent":
+        commits = payload.get("commits") or []
+        n = len(commits) if commits else payload.get("size") or 0
+        ref = (payload.get("ref") or "").split("/")[-1] or "default"
+        return f"push to {repo} ({n} commits, ref {ref})"
+
+    if et == "PullRequestEvent":
+        pr = payload.get("pull_request") or {}
+        title = (pr.get("title") or "").strip() or "(no title)"
+        action = payload.get("action") or "?"
+        num = pr.get("number", "")
+        return f"PR #{num} {action}: {title} ({repo})"
+
+    if et == "IssuesEvent":
+        issue = payload.get("issue") or {}
+        title = (issue.get("title") or "").strip() or "(no title)"
+        action = payload.get("action") or "?"
+        num = issue.get("number", "")
+        return f"issue #{num} {action}: {title} ({repo})"
+
+    if et == "CreateEvent":
+        ref = payload.get("ref") or ""
+        desc = payload.get("description") or ""
+        rt = payload.get("ref_type") or "ref"
+        extra = f" {desc}" if desc else ""
+        return f"created {rt} {ref} in {repo}{extra}"
+
+    if et == "DeleteEvent":
+        ref = payload.get("ref") or ""
+        return f"deleted {payload.get('ref_type', 'ref')} {ref} in {repo}"
+
+    if et == "ReleaseEvent":
+        rel = payload.get("release") or {}
+        tag = rel.get("tag_name") or rel.get("name") or "release"
+        return f"release {tag} ({repo})"
+
+    if et == "ForkEvent":
+        fork = (payload.get("forkee") or {}).get("full_name") or "fork"
+        return f"forked {repo} → {fork}"
+
+    if et == "WatchEvent":
+        return f"starred {repo}"
+
+    return f"{et} ({repo})"
+
+
+def collect_public_events(
+    profiles: List[Dict[str, Any]],
+    dt_from: datetime,
+    dt_to: datetime,
+    *,
+    username: str,
+    token: Optional[str],
+    classify_project: Callable[..., str],
+    make_event: Callable[..., Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """
+    Fetch `/users/{username}/events/public` (newest first; GitHub retains ~300 recent events).
+
+    Events outside the API window will not appear — sparse for old ranges.
+    """
+    if not username:
+        return []
+
+    results: List[Dict[str, Any]] = []
+    # Normalize bounds to aware UTC for comparison
+    if dt_from.tzinfo is None:
+        dt_from = dt_from.replace(tzinfo=timezone.utc)
+    if dt_to.tzinfo is None:
+        dt_to = dt_to.replace(tzinfo=timezone.utc)
+    dt_from_utc = dt_from.astimezone(timezone.utc)
+    dt_to_utc = dt_to.astimezone(timezone.utc)
+
+    for page in range(1, MAX_PAGES + 1):
+        url = f"https://api.github.com/users/{username}/events/public?per_page={PER_PAGE}&page={page}"
+        req = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/vnd.github+json"})
+        if token:
+            req.add_header("Authorization", f"Bearer {token}")
+        try:
+            with urlopen(req, timeout=30) as resp:
+                raw = resp.read().decode("utf-8")
+        except HTTPError as exc:
+            raise RuntimeError(f"GitHub API HTTP {exc.code}: {exc.reason}") from exc
+        except URLError as exc:
+            raise RuntimeError(f"GitHub API network error: {exc.reason}") from exc
+
+        batch = json.loads(raw)
+        if not batch:
+            break
+
+        stop_paging = False
+        for ev in batch:
+            created = ev.get("created_at")
+            if not created:
+                continue
+            ts = _parse_github_ts(created)
+            if ts > dt_to_utc:
+                continue
+            if ts < dt_from_utc:
+                stop_paging = True
+                break
+            detail = _detail_for_event(ev)
+            repo = (ev.get("repo") or {}).get("name") or ""
+            haystack = f"{repo} {detail}"
+            project = classify_project(haystack, profiles)
+            results.append(make_event(GITHUB_SOURCE, ts, detail, project))
+
+        if stop_paging:
+            break
+        if len(batch) < PER_PAGE:
+            break
+
+    return results