code-meter 0.1.0__py3-none-any.whl

code_meter/providers/codex.py

@@ -0,0 +1,356 @@
+"""Codex provider for scanning local OpenAI Codex CLI session logs (~/.codex)."""
+
+import hashlib
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from code_meter.models.usage import SessionCodeChange, SessionPrompt, UsageRecord
+from code_meter.providers.base import FileScanState, ScanResult, UsageProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CodexProvider(UsageProvider):
+    """Scanner for local Codex / OpenAI Codex CLI session files."""
+
+    def __init__(self, codex_dir: Path):
+        self.codex_dir = Path(os.path.expanduser(codex_dir)).resolve()
+
+    @property
+    def provider_name(self) -> str:
+        return "codex"
+
+    def scan(self, existing_states: Optional[Dict[str, FileScanState]] = None) -> ScanResult:
+        """Incrementally scan session log files in Codex directory."""
+        existing = existing_states or {}
+        records: List[UsageRecord] = []
+        prompts: List[SessionPrompt] = []
+        code_changes: List[SessionCodeChange] = []
+        updated_states: List[FileScanState] = []
+        files_scanned = 0
+        malformed_lines = 0
+
+        if not self.codex_dir.exists():
+            logger.warning(f"Codex directory does not exist: {self.codex_dir}")
+            return ScanResult(records=[], prompts=[], code_changes=[], updated_states=[], files_scanned=0, malformed_lines=0)
+
+        log_files = self._find_log_files()
+
+        for file_path in log_files:
+            try:
+                stat = file_path.stat()
+                file_str = str(file_path.resolve())
+                size = stat.st_size
+                mtime = stat.st_mtime
+
+                prev_state = existing.get(file_str)
+
+                if prev_state:
+                    if size == prev_state.file_size and mtime == prev_state.modified_time:
+                        updated_states.append(prev_state)
+                        continue
+
+                    start_offset = 0 if size < prev_state.file_size else prev_state.read_offset
+                else:
+                    start_offset = 0
+
+                files_scanned += 1
+                new_records, new_prompts, new_changes, new_offset, malformed = self._scan_file(
+                    file_path=file_path,
+                    start_offset=start_offset,
+                )
+
+                records.extend(new_records)
+                prompts.extend(new_prompts)
+                code_changes.extend(new_changes)
+                malformed_lines += malformed
+
+                updated_states.append(
+                    FileScanState(
+                        file_path=file_str,
+                        file_size=size,
+                        modified_time=mtime,
+                        read_offset=new_offset,
+                    )
+                )
+
+            except Exception as e:
+                logger.error(f"Error scanning Codex file {file_path}: {e}", exc_info=True)
+
+        return ScanResult(
+            records=records,
+            prompts=prompts,
+            code_changes=code_changes,
+            updated_states=updated_states,
+            files_scanned=files_scanned,
+            malformed_lines=malformed_lines,
+        )
+
+    def _find_log_files(self) -> List[Path]:
+        """Recursively find session JSON / JSONL files under self.codex_dir."""
+        files: List[Path] = []
+        try:
+            for root, _, filenames in os.walk(self.codex_dir):
+                for fname in filenames:
+                    if fname.endswith(".jsonl") or fname.endswith(".json"):
+                        files.append(Path(root) / fname)
+        except PermissionError as e:
+            logger.warning(f"Permission denied accessing {self.codex_dir}: {e}")
+        return files
+
+    def _scan_file(
+        self, file_path: Path, start_offset: int
+    ) -> Tuple[List[UsageRecord], List[SessionPrompt], List[SessionCodeChange], int, int]:
+        """Scan log file from start_offset."""
+        records: List[UsageRecord] = []
+        prompts: List[SessionPrompt] = []
+        changes: List[SessionCodeChange] = []
+        malformed = 0
+        current_offset = start_offset
+
+        session_id_from_path = file_path.stem
+        project_name = file_path.parent.name if file_path.parent != self.codex_dir else None
+
+        line_index = 0
+
+        # Handle full JSON files vs JSONL line files
+        if file_path.suffix == ".json":
+            try:
+                with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+                    content = f.read()
+                    data = json.loads(content)
+                    current_offset = len(content.encode("utf-8"))
+                    
+                    items = data if isinstance(data, list) else [data]
+                    for item in items:
+                        line_index += 1
+                        rec, pmt, chg = self._process_data_item(item, str(file_path.resolve()), session_id_from_path, project_name, line_index)
+                        if rec:
+                            records.append(rec)
+                        if pmt:
+                            prompts.append(pmt)
+                        if chg:
+                            changes.extend(chg)
+            except Exception:
+                malformed += 1
+            return records, prompts, changes, current_offset, malformed
+
+        # JSONL files
+        with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+            if start_offset > 0:
+                f.seek(start_offset)
+
+            while True:
+                line = f.readline()
+                if not line:
+                    break
+
+                current_offset = f.tell()
+                line_index += 1
+                line_str = line.strip()
+                if not line_str:
+                    continue
+
+                try:
+                    data = json.loads(line_str)
+                except Exception:
+                    malformed += 1
+                    continue
+
+                rec, pmt, chg = self._process_data_item(data, str(file_path.resolve()), session_id_from_path, project_name, line_index)
+                if rec:
+                    records.append(rec)
+                if pmt:
+                    prompts.append(pmt)
+                if chg:
+                    changes.extend(chg)
+
+        return records, prompts, changes, current_offset, malformed
+
+    def _process_data_item(
+        self,
+        data: dict,
+        file_path: str,
+        session_id_from_path: str,
+        project_name: Optional[str],
+        line_index: int,
+    ) -> Tuple[Optional[UsageRecord], Optional[SessionPrompt], List[SessionCodeChange]]:
+        if not isinstance(data, dict):
+            return None, None, []
+
+        session_id = data.get("session_id") or data.get("sessionId") or session_id_from_path or "unknown_session"
+        cwd = data.get("cwd") or data.get("workdir") or data.get("project_path")
+        p_name = data.get("project") or (os.path.basename(cwd) if cwd else project_name)
+
+        rec = self._parse_usage_record(data, file_path, session_id, cwd, p_name)
+        pmt = self._parse_user_prompt(data, session_id, p_name, line_index)
+        chg = self._parse_code_changes(data, session_id, p_name, line_index)
+
+        return rec, pmt, chg
+
+    def _parse_usage_record(
+        self,
+        data: dict,
+        file_path: str,
+        session_id: str,
+        cwd: Optional[str],
+        project_name: Optional[str],
+    ) -> Optional[UsageRecord]:
+        usage = data.get("usage") or data.get("response", {}).get("usage")
+        if not isinstance(usage, dict):
+            return None
+
+        input_tokens = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0)
+        output_tokens = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0)
+        
+        prompt_details = usage.get("prompt_tokens_details") if isinstance(usage.get("prompt_tokens_details"), dict) else {}
+        cache_read_tokens = int(prompt_details.get("cached_tokens") or usage.get("cache_read_tokens") or 0)
+        cache_write_tokens = int(usage.get("cache_write_tokens") or 0)
+
+        if (input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) == 0:
+            return None
+
+        model = data.get("model") or data.get("response", {}).get("model") or "gpt-4o"
+        if not isinstance(model, str):
+            model = str(model)
+
+        raw_ts = data.get("timestamp") or data.get("created_at") or data.get("time")
+        ts = self._parse_timestamp(raw_ts)
+
+        request_id = data.get("id") or data.get("request_id") or data.get("uuid")
+        if not request_id or not isinstance(request_id, str):
+            request_id = self._generate_deterministic_id(
+                file_path=file_path,
+                timestamp=ts,
+                model=model,
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                cache_read_tokens=cache_read_tokens,
+                cache_write_tokens=cache_write_tokens,
+            )
+
+        return UsageRecord(
+            provider="codex",
+            request_id=request_id,
+            session_id=session_id,
+            timestamp=ts,
+            user_id=data.get("user_id"),
+            project_id=project_name,
+            project_path=cwd,
+            model=model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            cache_read_tokens=cache_read_tokens,
+            cache_write_tokens=cache_write_tokens,
+            metadata={"file_path": file_path, "type": data.get("type")},
+        )
+
+    def _parse_user_prompt(
+        self, data: dict, session_id: str, project_id: Optional[str], line_index: int
+    ) -> Optional[SessionPrompt]:
+        role = data.get("role") or data.get("message", {}).get("role") or data.get("type")
+        if role not in ["user", "prompt", "user_input"]:
+            return None
+
+        content = data.get("prompt") or data.get("content") or data.get("message", {}).get("content")
+        prompt_text = ""
+
+        if isinstance(content, str):
+            prompt_text = content.strip()
+        elif isinstance(content, list):
+            texts = []
+            for item in content:
+                if isinstance(item, str):
+                    texts.append(item)
+                elif isinstance(item, dict) and item.get("text"):
+                    texts.append(item.get("text", ""))
+            prompt_text = "\n".join(texts).strip()
+
+        if not prompt_text:
+            return None
+
+        raw_ts = data.get("timestamp") or data.get("created_at")
+        ts = self._parse_timestamp(raw_ts)
+
+        prompt_id = data.get("prompt_id") or data.get("id") or f"codex_prompt_{session_id}_{line_index}"
+
+        return SessionPrompt(
+            provider="codex",
+            session_id=session_id,
+            prompt_id=str(prompt_id),
+            timestamp=ts,
+            prompt_text=prompt_text,
+            project_id=project_id,
+        )
+
+    def _parse_code_changes(
+        self, data: dict, session_id: str, project_id: Optional[str], line_index: int
+    ) -> List[SessionCodeChange]:
+        changes: List[SessionCodeChange] = []
+        raw_ts = data.get("timestamp") or data.get("created_at")
+        ts = self._parse_timestamp(raw_ts)
+
+        tool_calls = data.get("tool_calls") or data.get("message", {}).get("tool_calls")
+        if isinstance(tool_calls, list):
+            for idx, call in enumerate(tool_calls):
+                if isinstance(call, dict):
+                    fn = call.get("function", {})
+                    fname = fn.get("name", "")
+                    if fname in ["edit_file", "write_file", "apply_patch", "str_replace", "replace_file_content"]:
+                        args_raw = fn.get("arguments", {})
+                        args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw if isinstance(args_raw, dict) else {})
+                        
+                        fpath = args.get("file_path") or args.get("path") or args.get("TargetFile") or "unknown_file"
+                        old_code = args.get("old_content") or args.get("TargetContent") or ""
+                        new_code = args.get("new_content") or args.get("ReplacementContent") or args.get("content") or ""
+
+                        diff_lines = []
+                        if old_code:
+                            diff_lines.append(f"- {old_code[:150]}")
+                        if new_code:
+                            diff_lines.append(f"+ {new_code[:150]}")
+                        diff_summary = "\n".join(diff_lines) if diff_lines else f"Codex action: {fname}"
+
+                        change_id = call.get("id") or f"codex_edit_{session_id}_{line_index}_{idx}"
+                        changes.append(
+                            SessionCodeChange(
+                                provider="codex",
+                                session_id=session_id,
+                                change_id=str(change_id),
+                                timestamp=ts,
+                                file_path=str(fpath),
+                                change_type=fname.lower(),
+                                diff_summary=diff_summary,
+                                project_id=project_id,
+                            )
+                        )
+
+        return changes
+
+    def _parse_timestamp(self, raw_ts: Optional[str]) -> datetime:
+        if not raw_ts:
+            return datetime.now(timezone.utc)
+        if isinstance(raw_ts, (int, float)):
+            return datetime.fromtimestamp(raw_ts if raw_ts < 1e11 else raw_ts / 1000.0, tz=timezone.utc)
+        try:
+            ts_str = str(raw_ts).replace("Z", "+00:00")
+            return datetime.fromisoformat(ts_str)
+        except ValueError:
+            return datetime.now(timezone.utc)
+
+    def _generate_deterministic_id(
+        self,
+        file_path: str,
+        timestamp: datetime,
+        model: str,
+        input_tokens: int,
+        output_tokens: int,
+        cache_read_tokens: int,
+        cache_write_tokens: int,
+    ) -> str:
+        seed = f"codex:{file_path}:{timestamp.isoformat()}:{model}:{input_tokens}:{output_tokens}:{cache_read_tokens}:{cache_write_tokens}"
+        return "gen_codex_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]

code_meter/storage/__init__.py

@@ -0,0 +1,6 @@
+"""Storage package for SQLite database operations and repositories."""
+
+from code_meter.storage.database import Database
+from code_meter.storage.repository import UsageRepository
+
+__all__ = ["Database", "UsageRepository"]

code_meter/storage/database.py

@@ -0,0 +1,147 @@
+"""SQLite Database connection and schema initialization."""
+
+import os
+import sqlite3
+from pathlib import Path
+from typing import Generator
+
+
+class Database:
+    """Manages SQLite database connections and schema migrations."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = Path(os.path.expanduser(db_path)).resolve()
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def get_connection(self) -> sqlite3.Connection:
+        """Create a sqlite3 connection with WAL journal mode enabled for concurrency."""
+        conn = sqlite3.connect(str(self.db_path), timeout=30.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode = WAL;")
+        conn.execute("PRAGMA foreign_keys = ON;")
+        return conn
+
+    def connection_context(self) -> Generator[sqlite3.Connection, None, None]:
+        """Context manager for sqlite connection handling."""
+        conn = self.get_connection()
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def initialize_schema(self) -> None:
+        """Create database tables and indexes if they do not exist."""
+        conn = self.get_connection()
+        try:
+            cursor = conn.cursor()
+
+            # usage_events table
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS usage_events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    provider TEXT NOT NULL,
+                    request_id TEXT NOT NULL,
+                    session_id TEXT,
+                    timestamp TEXT NOT NULL,
+                    user_id TEXT,
+                    project_id TEXT,
+                    project_path TEXT,
+                    model TEXT NOT NULL,
+                    input_tokens INTEGER NOT NULL DEFAULT 0,
+                    output_tokens INTEGER NOT NULL DEFAULT 0,
+                    cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+                    cache_write_tokens INTEGER NOT NULL DEFAULT 0,
+                    metadata_json TEXT,
+                    agent_id TEXT,
+                    task_id TEXT,
+                    tool_name TEXT,
+                    created_at TEXT NOT NULL,
+                    UNIQUE(provider, request_id)
+                );
+                """
+            )
+
+            # session_prompts table
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS session_prompts (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    provider TEXT NOT NULL,
+                    session_id TEXT NOT NULL,
+                    prompt_id TEXT NOT NULL,
+                    timestamp TEXT NOT NULL,
+                    prompt_text TEXT NOT NULL,
+                    project_id TEXT,
+                    created_at TEXT NOT NULL,
+                    UNIQUE(provider, prompt_id)
+                );
+                """
+            )
+
+            # session_code_changes table (tracks git diffs / code edits over time)
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS session_code_changes (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    provider TEXT NOT NULL,
+                    session_id TEXT NOT NULL,
+                    change_id TEXT NOT NULL,
+                    timestamp TEXT NOT NULL,
+                    file_path TEXT NOT NULL,
+                    change_type TEXT NOT NULL,
+                    diff_summary TEXT NOT NULL,
+                    project_id TEXT,
+                    created_at TEXT NOT NULL,
+                    UNIQUE(provider, change_id)
+                );
+                """
+            )
+
+            # scan_state table
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS scan_state (
+                    file_path TEXT PRIMARY KEY,
+                    file_size INTEGER NOT NULL,
+                    modified_time REAL NOT NULL,
+                    read_offset INTEGER NOT NULL,
+                    updated_at TEXT NOT NULL
+                );
+                """
+            )
+
+            # pricing table
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS pricing (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    provider TEXT NOT NULL,
+                    model TEXT NOT NULL,
+                    input_price_per_million REAL NOT NULL,
+                    output_price_per_million REAL NOT NULL,
+                    cache_read_price_per_million REAL NOT NULL DEFAULT 0.0,
+                    cache_write_price_per_million REAL NOT NULL DEFAULT 0.0,
+                    effective_from TEXT NOT NULL,
+                    UNIQUE(provider, model, effective_from)
+                );
+                """
+            )
+
+            # Indexes for high performance queries
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_usage_ts ON usage_events(timestamp);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_usage_model ON usage_events(model);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_usage_project ON usage_events(project_id);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_usage_provider ON usage_events(provider);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_usage_session ON usage_events(session_id);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_prompts_session ON session_prompts(session_id);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_changes_session ON session_code_changes(session_id);")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_changes_file ON session_code_changes(file_path);")
+
+            conn.commit()
+        finally:
+            conn.close()