dotscope 0.1.0__py3-none-any.whl

dotscope/storage/timing.py

@@ -0,0 +1,84 @@
+"""Lightweight timing instrumentation for core operations."""
+
+import json
+import os
+import time
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class TimingEntry:
+    operation: str  # "resolve", "check", "ingest"
+    duration_ms: float
+    timestamp: str
+
+
+_MAX_TIMING_LINES = 5000
+
+
+def record_timing(repo_root: str, operation: str, duration_ms: float) -> None:
+    """Append a timing entry to .dotscope/timings.jsonl. Truncates at 5000 lines."""
+    dot_dir = os.path.join(repo_root, ".dotscope")
+    if not os.path.isdir(dot_dir):
+        return  # No .dotscope dir — skip silently
+
+    path = os.path.join(dot_dir, "timings.jsonl")
+    entry = {
+        "operation": operation,
+        "duration_ms": round(duration_ms, 2),
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    }
+    with open(path, "a", encoding="utf-8") as f:
+        f.write(json.dumps(entry) + "\n")
+
+    # Truncate if too large (keep most recent half)
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+        if len(lines) > _MAX_TIMING_LINES:
+            with open(path, "w", encoding="utf-8") as f:
+                f.writelines(lines[len(lines) // 2:])
+    except (IOError, OSError):
+        pass
+
+
+def load_timings(repo_root: str) -> List[TimingEntry]:
+    """Load all timing entries."""
+    path = os.path.join(repo_root, ".dotscope", "timings.jsonl")
+    if not os.path.exists(path):
+        return []
+
+    entries = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                try:
+                    d = json.loads(line)
+                    entries.append(TimingEntry(
+                        operation=d["operation"],
+                        duration_ms=d["duration_ms"],
+                        timestamp=d.get("timestamp", ""),
+                    ))
+                except (json.JSONDecodeError, KeyError):
+                    continue
+    return entries
+
+
+def median(values: List[float]) -> float:
+    if not values:
+        return 0.0
+    s = sorted(values)
+    n = len(s)
+    if n % 2 == 0:
+        return (s[n // 2 - 1] + s[n // 2]) / 2
+    return s[n // 2]
+
+
+def percentile(values: List[float], p: int) -> float:
+    if not values:
+        return 0.0
+    s = sorted(values)
+    idx = int(len(s) * p / 100)
+    return s[min(idx, len(s) - 1)]

dotscope/timing.py

@@ -0,0 +1,2 @@
+"""Backward-compatibility stub. Moved to dotscope.storage.timing."""
+from .storage.timing import *  # noqa: F401,F403

dotscope/tokens.py

@@ -0,0 +1,53 @@
+"""Token estimation for scope files and resolved contexts.
+
+Uses tiktoken if available, otherwise falls back to len(text) // 4.
+"""
+
+
+import os
+from typing import List
+
+# Try tiktoken for accurate counts, fall back to approximation
+try:
+    import tiktoken
+
+    _encoder = tiktoken.encoding_for_model("gpt-4")
+
+    def estimate_tokens(text: str) -> int:
+        """Estimate token count using tiktoken."""
+        return len(_encoder.encode(text))
+
+except ImportError:
+
+    def estimate_tokens(text: str) -> int:
+        """Estimate token count (~4 chars per token for English)."""
+        return len(text) // 4
+
+
+def estimate_file_tokens(path: str) -> int:
+    """Read a file and estimate its token count."""
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            return estimate_tokens(f.read())
+    except (OSError, IOError):
+        return 0
+
+
+def estimate_scope_tokens(files: List[str]) -> int:
+    """Estimate total tokens across a list of files."""
+    return sum(estimate_file_tokens(f) for f in files)
+
+
+def estimate_context_tokens(context: str) -> int:
+    """Estimate tokens for a context string."""
+    if not context:
+        return 0
+    return estimate_tokens(context)
+
+
+def file_size_bytes(path: str) -> int:
+    """Get file size in bytes, 0 if not accessible."""
+    try:
+        return os.path.getsize(path)
+    except OSError:
+        return 0

dotscope/utility.py

@@ -0,0 +1,123 @@
+"""Utility scoring: historical file relevance from observation data.
+
+Computes per-file utility ratios from session + observation logs.
+Files agents actually touch get higher scores. Budget allocation
+uses these scores instead of static heuristics.
+
+The utility floor prevents a death spiral: core abstractions that are
+rarely edited but frequently read always retain a base weight.
+"""
+
+import json
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from .models.state import FileUtilityScore, ObservationLog, SessionLog  # noqa: F401
+
+
+BASE_UTILITY_WEIGHT = 0.5
+MAX_UTILITY_BONUS = 1.0
+MIN_SAMPLE_SIZE = 3
+
+
+def compute_utility_scores(
+    sessions: List[SessionLog],
+    observations: List[ObservationLog],
+) -> Dict[str, FileUtilityScore]:
+    """Build utility scores from session + observation logs."""
+    scores: Dict[str, FileUtilityScore] = {}
+
+    # Count resolves per file
+    for session in sessions:
+        for f in session.predicted_files:
+            if f not in scores:
+                scores[f] = FileUtilityScore(path=f)
+            scores[f].resolve_count += 1
+            scores[f].last_resolved = max(scores[f].last_resolved, session.timestamp)
+
+    # Count touches per file
+    obs_by_session: Dict[str, ObservationLog] = {
+        obs.session_id: obs for obs in observations
+    }
+    for session in sessions:
+        obs = obs_by_session.get(session.session_id)
+        if not obs:
+            continue
+        for f in obs.actual_files_modified:
+            if f not in scores:
+                scores[f] = FileUtilityScore(path=f)
+            scores[f].touch_count += 1
+            scores[f].last_touched = max(scores[f].last_touched, obs.timestamp)
+
+    # Compute ratios
+    for score in scores.values():
+        if score.resolve_count > 0:
+            score.utility_ratio = round(score.touch_count / score.resolve_count, 3)
+
+    return scores
+
+
+def effective_score(
+    base_score: float,
+    utility: Optional[FileUtilityScore],
+    is_explicit_include: bool,
+) -> float:
+    """Compute effective score with utility floor protection.
+
+    Explicit includes always get a base weight. Utility observations
+    add on top, never subtract below the floor.
+    """
+    floor = BASE_UTILITY_WEIGHT if is_explicit_include else 0.0
+
+    bonus = 0.0
+    if utility and utility.resolve_count >= MIN_SAMPLE_SIZE:
+        bonus = utility.utility_ratio * MAX_UTILITY_BONUS
+        # Recency bonus: touched in last 30 days
+        if utility.last_touched > time.time() - (30 * 86400):
+            bonus *= 1.1
+
+    return base_score * max(floor + bonus, floor) if floor else base_score * (1.0 + bonus)
+
+
+def save_utility_scores(dot_dir: Path, scores: Dict[str, FileUtilityScore]) -> None:
+    """Write utility scores to .dotscope/utility/file_scores.json."""
+    utility_dir = dot_dir / "utility"
+    utility_dir.mkdir(parents=True, exist_ok=True)
+
+    data = {
+        path: {
+            "resolve_count": s.resolve_count,
+            "touch_count": s.touch_count,
+            "utility_ratio": s.utility_ratio,
+            "last_touched": s.last_touched,
+            "last_resolved": s.last_resolved,
+        }
+        for path, s in scores.items()
+    }
+
+    (utility_dir / "file_scores.json").write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+
+def load_utility_scores(dot_dir: Path) -> Dict[str, FileUtilityScore]:
+    """Load utility scores from .dotscope/utility/file_scores.json."""
+    path = dot_dir / "utility" / "file_scores.json"
+    if not path.exists():
+        return {}
+
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return {
+            k: FileUtilityScore(path=k, **v)
+            for k, v in data.items()
+        }
+    except (json.JSONDecodeError, TypeError):
+        return {}
+
+
+def rebuild_utility(dot_dir: Path, sessions: List[SessionLog],
+                    observations: List[ObservationLog]) -> Dict[str, FileUtilityScore]:
+    """Rebuild and save utility scores from event logs."""
+    scores = compute_utility_scores(sessions, observations)
+    save_utility_scores(dot_dir, scores)
+    return scores

dotscope/virtual.py

@@ -0,0 +1,3 @@
+"""Backward-compatibility stub. Moved to dotscope.passes.virtual."""
+from .passes.virtual import *  # noqa: F401,F403
+from .models.passes import VirtualScope  # noqa: F401