dotscope 0.1.0__py3-none-any.whl

dotscope/paths.py

@@ -0,0 +1,32 @@
+"""Shared path helpers — normalize, relative, exists checks."""
+
+
+import os
+from typing import Optional
+
+
+def normalize(base: str, rel_path: str) -> str:
+    """Join and normalize a base + relative path."""
+    return os.path.normpath(os.path.join(base, rel_path))
+
+
+def make_relative(abs_path: str, root: Optional[str]) -> str:
+    """Make a path relative to root, falling back to absolute."""
+    if root:
+        try:
+            return os.path.relpath(abs_path, root)
+        except ValueError:
+            pass
+    return abs_path
+
+
+def path_exists(base: str, rel_path: str) -> bool:
+    """Check if a path (possibly with trailing /) exists."""
+    full = normalize(base, rel_path)
+    return os.path.exists(full.rstrip("/"))
+
+
+def strip_inline_comment(text: str) -> str:
+    """Strip trailing '# comment' from a path or value string."""
+    parts = text.split("#", 1)
+    return parts[0].strip()

dotscope/progress.py

@@ -0,0 +1,44 @@
+"""Streaming progress for long-running pipeline steps.
+
+Emits terse status lines to stderr as each step begins and completes.
+The developer watches the tool think instead of staring at silence.
+"""
+
+import sys
+import time
+
+
+class ProgressEmitter:
+    """Emit streaming progress for pipeline steps."""
+
+    def __init__(self, quiet: bool = False, stream=None):
+        self._quiet = quiet
+        self._stream = stream or sys.stderr
+        self._start_time = 0.0
+
+    def start(self, action: str) -> None:
+        """Print action with trailing ... (no newline)."""
+        if self._quiet:
+            return
+        self._stream.write(f"dotscope: {action}...")
+        self._stream.flush()
+        self._start_time = time.perf_counter()
+
+    def finish(self, result: str) -> None:
+        """Complete the current line with the result."""
+        if self._quiet:
+            return
+        elapsed = time.perf_counter() - self._start_time
+        pad = " " * max(1, 45 - len(result))
+        if elapsed > 1.0:
+            self._stream.write(f"{pad}{result} ({elapsed:.1f}s)\n")
+        else:
+            self._stream.write(f"{pad}{result}\n")
+        self._stream.flush()
+
+    def skip(self, action: str, reason: str) -> None:
+        """Show a skipped step."""
+        if self._quiet:
+            return
+        self._stream.write(f"dotscope: {action}... skipped ({reason})\n")
+        self._stream.flush()

dotscope/regression.py

@@ -0,0 +1,147 @@
+"""Observation regression suite: freeze successful sessions as test cases.
+
+When dotscope's internals change, replay frozen sessions to verify the new
+version resolves the same or better context.
+"""
+
+import hashlib
+import json
+import os
+from dataclasses import asdict
+from typing import List, Optional
+from uuid import uuid4
+
+from .models.state import RegressionCase, ReplayResult  # noqa: F401
+
+
+def maybe_freeze_session(
+    observation: object,
+    session: object,
+    repo_root: str,
+    min_recall: float = 0.8,
+) -> Optional[str]:
+    """Freeze a successful session as a regression test case.
+
+    Returns the case ID if frozen, None otherwise.
+    """
+    recall = getattr(observation, "recall", 0.0)
+    if recall < min_recall:
+        return None
+
+    predicted = getattr(session, "predicted_files", [])
+    context_hash = getattr(session, "context_hash", "")
+    scope_expr = getattr(session, "scope_expr", "")
+
+    case = RegressionCase(
+        id=f"regression_{uuid4().hex[:8]}",
+        scope_expr=scope_expr,
+        expected_files=list(predicted),
+        expected_context_hash=context_hash,
+        actual_recall=recall,
+        timestamp=getattr(observation, "timestamp", ""),
+    )
+
+    reg_dir = os.path.join(repo_root, ".dotscope", "regressions")
+    os.makedirs(reg_dir, exist_ok=True)
+
+    path = os.path.join(reg_dir, f"{case.id}.json")
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(asdict(case), f, indent=2)
+
+    return case.id
+
+
+def load_regressions(repo_root: str) -> List[RegressionCase]:
+    """Load all frozen regression cases."""
+    reg_dir = os.path.join(repo_root, ".dotscope", "regressions")
+    if not os.path.isdir(reg_dir):
+        return []
+
+    cases = []
+    for fname in sorted(os.listdir(reg_dir)):
+        if not fname.endswith(".json"):
+            continue
+        path = os.path.join(reg_dir, fname)
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                d = json.load(f)
+            cases.append(RegressionCase(
+                id=d["id"],
+                scope_expr=d.get("scope_expr", ""),
+                budget=d.get("budget"),
+                task=d.get("task"),
+                expected_files=d.get("expected_files", []),
+                expected_context_hash=d.get("expected_context_hash", ""),
+                actual_recall=d.get("actual_recall", 0.0),
+                timestamp=d.get("timestamp", ""),
+            ))
+        except (json.JSONDecodeError, KeyError, IOError):
+            continue
+
+    return cases
+
+
+def replay_regression(
+    case: RegressionCase,
+    repo_root: str,
+) -> ReplayResult:
+    """Replay a frozen session against current codebase state."""
+    from .composer import compose
+    from .budget import apply_budget
+
+    resolved = compose(case.scope_expr, root=repo_root, follow_related=True)
+    if case.budget:
+        resolved = apply_budget(resolved, case.budget)
+
+    new_files = set(resolved.files)
+    expected = set(case.expected_files)
+
+    new_hash = hashlib.sha256(resolved.context.encode()).hexdigest()[:16]
+
+    return ReplayResult(
+        case=case,
+        new_files=sorted(new_files),
+        new_context_hash=new_hash,
+        files_added=sorted(new_files - expected),
+        files_dropped=sorted(expected - new_files),
+        context_changed=(new_hash != case.expected_context_hash),
+        is_regression=len(expected - new_files) > 0,
+    )
+
+
+def format_replay_report(results: List[ReplayResult]) -> str:
+    """Format replay results for terminal output."""
+    if not results:
+        return "No regression cases found. Sessions are auto-frozen after successful observations."
+
+    lines = [f"dotscope test-compiler: replaying {len(results)} historical sessions\n"]
+    passed = 0
+    regressions = 0
+
+    for r in results:
+        prefix = f"  {r.case.id}  {r.case.scope_expr}"
+        if r.case.budget:
+            prefix += f" (budget {r.case.budget})"
+
+        if not r.is_regression and not r.files_added:
+            lines.append(f"{prefix}")
+            lines.append(f"    Files: {len(r.new_files)}/{len(r.case.expected_files)} same  OK")
+            passed += 1
+        elif not r.is_regression and r.files_added:
+            lines.append(f"{prefix}")
+            added = ", ".join(r.files_added[:3])
+            lines.append(f"    Files: +{len(r.files_added)} added ({added})  OK (improvement)")
+            passed += 1
+        else:
+            lines.append(f"{prefix}")
+            dropped = ", ".join(r.files_dropped[:3])
+            lines.append(f"    REGRESSION: {dropped} no longer resolved")
+            regressions += 1
+
+        if r.context_changed:
+            lines.append(f"    Context hash: changed")
+
+        lines.append("")
+
+    lines.append(f"  {passed}/{len(results)} passed, {regressions} regression(s) detected")
+    return "\n".join(lines)

dotscope/resolver.py

@@ -0,0 +1,203 @@
+"""Core resolution engine: .scope config → concrete file list.
+
+Walks includes, applies excludes, follows related scopes with cycle detection.
+"""
+
+
+import fnmatch
+import os
+from pathlib import Path
+from typing import List, Optional, Set
+
+from .constants import SKIP_DIRS
+from .models import ResolvedScope, ScopeConfig
+from .tokens import estimate_file_tokens, estimate_context_tokens
+
+
+def resolve(
+    config: ScopeConfig,
+    follow_related: bool = True,
+    max_depth: int = 3,
+    root: Optional[str] = None,
+) -> ResolvedScope:
+    """Resolve a ScopeConfig to a concrete file list.
+
+    Args:
+        config: Parsed .scope configuration
+        follow_related: Whether to follow related scope references
+        max_depth: Maximum depth for related scope traversal
+        root: Repository root (for resolving related scope paths)
+    """
+    return _resolve_inner(config, follow_related, max_depth, root, set(), 0)
+
+
+def _resolve_inner(
+    config: ScopeConfig,
+    follow_related: bool,
+    max_depth: int,
+    root: Optional[str],
+    _visited: Set[str],
+    _depth: int,
+) -> ResolvedScope:
+    """Internal resolver with cycle detection state."""
+
+    # Cycle detection
+    abs_path = os.path.abspath(config.path)
+    if abs_path in _visited:
+        return ResolvedScope(scope_chain=[abs_path])
+    _visited.add(abs_path)
+
+    scope_dir = config.directory
+    if root is None:
+        from .discovery import find_repo_root
+        root = find_repo_root(scope_dir) or scope_dir
+
+    files = _collect_includes(config.includes, root)
+    files = _apply_excludes(files, config.excludes, root)
+    context = config.context_str
+    file_tokens = sum(estimate_file_tokens(f) for f in files)
+    context_tokens = estimate_context_tokens(context)
+
+    result = ResolvedScope(
+        files=files,
+        context=context,
+        token_estimate=file_tokens + context_tokens,
+        scope_chain=[abs_path],
+        truncated=False,
+    )
+
+    if follow_related and config.related and _depth < max_depth:
+        for related_path in config.related:
+            related_config = _load_related(related_path, scope_dir, root)
+            if related_config is None:
+                continue
+
+            related_resolved = _resolve_inner(
+                related_config,
+                follow_related=True,
+                max_depth=max_depth,
+                root=root,
+                _visited=_visited,
+                _depth=_depth + 1,
+            )
+            result = result.merge(related_resolved)
+
+    return result
+
+
+def _collect_includes(includes: List[str], scope_dir: str) -> List[str]:
+    """Expand include paths to concrete file list."""
+    files = []
+    seen: Set[str] = set()
+
+    for pattern in includes:
+        # Resolve relative to scope directory
+        full_path = os.path.normpath(os.path.join(scope_dir, pattern))
+
+        if pattern.endswith("/"):
+            # Directory: recursive walk
+            _walk_directory(full_path.rstrip("/"), files, seen)
+        elif any(c in pattern for c in "*?["):
+            # Glob pattern
+            _glob_pattern(pattern, scope_dir, files, seen)
+        elif os.path.isfile(full_path):
+            # Exact file
+            if full_path not in seen:
+                files.append(full_path)
+                seen.add(full_path)
+        elif os.path.isdir(full_path):
+            # Directory without trailing slash — still treat as recursive
+            _walk_directory(full_path, files, seen)
+
+    return files
+
+
+def _walk_directory(dir_path: str, files: List[str], seen: Set[str]) -> None:
+    """Recursively collect all files in a directory."""
+    skip_dirs = SKIP_DIRS
+
+    if not os.path.isdir(dir_path):
+        return
+
+    for dirpath, dirnames, filenames in os.walk(dir_path):
+        dirnames[:] = [d for d in dirnames if d not in skip_dirs]
+        for filename in sorted(filenames):
+            full = os.path.join(dirpath, filename)
+            if full not in seen:
+                files.append(full)
+                seen.add(full)
+
+
+def _glob_pattern(pattern: str, scope_dir: str, files: List[str], seen: Set[str]) -> None:
+    """Expand a glob pattern relative to scope directory."""
+    base = Path(scope_dir)
+    for match in sorted(base.glob(pattern)):
+        if match.is_file():
+            full = str(match)
+            if full not in seen:
+                files.append(full)
+                seen.add(full)
+
+
+def _apply_excludes(files: List[str], excludes: List[str], scope_dir: str) -> List[str]:
+    """Filter out files matching exclude patterns."""
+    if not excludes:
+        return files
+
+    result = []
+    for f in files:
+        rel_path = os.path.relpath(f, scope_dir)
+        excluded = False
+
+        for pattern in excludes:
+            # Check against relative path
+            if fnmatch.fnmatch(rel_path, pattern):
+                excluded = True
+                break
+            # Check against filename only
+            if fnmatch.fnmatch(os.path.basename(f), pattern):
+                excluded = True
+                break
+            # Check if file is under an excluded directory
+            if pattern.endswith("/"):
+                dir_prefix = pattern.rstrip("/")
+                if rel_path.startswith(dir_prefix + "/") or rel_path.startswith(dir_prefix + os.sep):
+                    excluded = True
+                    break
+            # Also match with ** prefix for nested patterns
+            if "/" in pattern and fnmatch.fnmatch(rel_path, "**/" + pattern):
+                excluded = True
+                break
+
+        if not excluded:
+            result.append(f)
+
+    return result
+
+
+def _load_related(
+    related_path: str, scope_dir: str, root: str
+) -> Optional[ScopeConfig]:
+    """Load a related scope file."""
+    from .parser import parse_scope_file
+
+    from .paths import strip_inline_comment
+    related_path = strip_inline_comment(related_path)
+
+    # Try relative to scope directory first
+    candidate = os.path.normpath(os.path.join(scope_dir, related_path))
+    if os.path.isfile(candidate):
+        try:
+            return parse_scope_file(candidate)
+        except (ValueError, IOError):
+            return None
+
+    # Try relative to root
+    candidate = os.path.normpath(os.path.join(root, related_path))
+    if os.path.isfile(candidate):
+        try:
+            return parse_scope_file(candidate)
+        except (ValueError, IOError):
+            return None
+
+    return None

dotscope/scanner.py

@@ -0,0 +1,246 @@
+"""Auto-generate .scope from directory analysis.
+
+Scans directory structure, detects language, finds cross-directory imports,
+and produces a reasonable starting .scope configuration.
+"""
+
+
+import os
+import re
+from collections import Counter
+from typing import List, Optional, Set, Tuple
+
+from .constants import LANG_MAP, SKIP_DIRS
+from .context import parse_context
+from .models import ScopeConfig
+from .tokens import estimate_file_tokens
+
+
+# Directories to always exclude
+_DEFAULT_EXCLUDES = [
+    "__pycache__/",
+    "node_modules/",
+    ".git/",
+    "*.pyc",
+    "dist/",
+    "build/",
+    "*.egg-info/",
+    ".tox/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    "venv/",
+    ".venv/",
+    "*.min.js",
+    "*.generated.*",
+]
+
+# Directories that are typically test/fixture/migration content
+_TEST_DIRS = {"tests", "test", "__tests__", "spec", "specs"}
+_FIXTURE_DIRS = {"fixtures", "fixture", "testdata", "test_data", "mocks"}
+_MIGRATION_DIRS = {"migrations", "migrate", "alembic"}
+
+
+def scan_directory(path: str) -> ScopeConfig:
+    """Analyze a directory and generate a starter .scope configuration.
+
+    The human then edits the context field — that's the part that can't be automated.
+    """
+    path = os.path.abspath(path)
+    dir_name = os.path.basename(path)
+
+    # Collect file info
+    files, lang_counts, total_tokens = _scan_files(path)
+
+    # Detect primary language
+    primary_lang = _detect_language(lang_counts)
+
+    includes = [f"{dir_name}/"]
+
+    # Build excludes
+    excludes = _build_excludes(path)
+
+    # Detect cross-directory imports
+    external_deps = _find_external_deps(path, files, primary_lang)
+
+    # Add external dependencies to includes
+    for dep in external_deps:
+        if dep not in includes:
+            includes.append(dep)
+
+    # Build description
+    file_count = len(files)
+    description = f"{dir_name} -- {primary_lang or 'mixed'} ({file_count} files)"
+
+    # Build tags
+    tags = _infer_tags(path, dir_name, files)
+
+    context = parse_context(
+        "# TODO: Add architectural context here.\n"
+        "# What invariants does this module maintain?\n"
+        "# What gotchas should an agent know about?\n"
+        "# What conventions does it follow?"
+    )
+
+    return ScopeConfig(
+        path=os.path.join(path, ".scope"),
+        description=description,
+        includes=includes,
+        excludes=excludes,
+        context=context,
+        related=[],
+        owners=[],
+        tags=tags,
+        tokens_estimate=total_tokens,
+    )
+
+
+def _scan_files(path: str) -> Tuple[List[str], Counter, int]:
+    """Walk directory, collect files, count languages, estimate tokens."""
+    files = []
+    lang_counts: Counter = Counter()
+    total_tokens = 0
+
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
+
+        for filename in filenames:
+            full = os.path.join(dirpath, filename)
+            files.append(full)
+
+            ext = os.path.splitext(filename)[1].lower()
+            if ext in LANG_MAP:
+                lang_counts[LANG_MAP[ext]] += 1
+
+            total_tokens += estimate_file_tokens(full)
+
+    return files, lang_counts, total_tokens
+
+
+def _detect_language(lang_counts: Counter) -> Optional[str]:
+    """Detect the primary language from extension counts."""
+    if not lang_counts:
+        return None
+    return lang_counts.most_common(1)[0][0]
+
+
+def _build_excludes(path: str) -> List[str]:
+    """Build excludes from detected directories."""
+    excludes = []
+    dir_name = os.path.basename(path)
+
+    for entry in os.listdir(path):
+        full = os.path.join(path, entry)
+        if not os.path.isdir(full):
+            continue
+
+        entry_lower = entry.lower()
+        if entry_lower in _TEST_DIRS:
+            excludes.append(f"{dir_name}/{entry}/fixtures/")
+        if entry_lower in _FIXTURE_DIRS:
+            excludes.append(f"{dir_name}/{entry}/")
+        if entry_lower in _MIGRATION_DIRS:
+            excludes.append(f"{dir_name}/{entry}/")
+        if entry in SKIP_DIRS:
+            excludes.append(f"{dir_name}/{entry}/")
+
+    # Add common glob excludes
+    excludes.extend([
+        "*.pyc",
+        f"{dir_name}/__pycache__/",
+    ])
+
+    return list(dict.fromkeys(excludes))  # dedupe preserving order
+
+
+def _find_external_deps(
+    path: str, files: List[str], lang: Optional[str]
+) -> List[str]:
+    """Parse imports to find cross-directory dependencies."""
+    if lang == "Python":
+        return _find_python_imports(path, files)
+    elif lang in ("JavaScript", "TypeScript"):
+        return _find_js_imports(path, files)
+    return []
+
+
+def _find_python_imports(path: str, files: List[str]) -> List[str]:
+    """Find Python imports that reference outside the scanned directory."""
+    external: Set[str] = set()
+    parent = os.path.dirname(path)
+
+    for f in files:
+        if not f.endswith(".py"):
+            continue
+        try:
+            with open(f, "r", encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    line = line.strip()
+                    # from foo.bar import baz
+                    m = re.match(r"from\s+([\w.]+)\s+import", line)
+                    if m:
+                        module = m.group(1).split(".")[0]
+                        candidate = os.path.join(parent, module)
+                        if os.path.isdir(candidate) and candidate != path:
+                            rel = os.path.relpath(candidate, parent)
+                            external.add(f"{rel}/")
+                        candidate_file = os.path.join(parent, module + ".py")
+                        if os.path.isfile(candidate_file) and os.path.dirname(candidate_file) != path:
+                            rel = os.path.relpath(candidate_file, parent)
+                            external.add(rel)
+        except (IOError, OSError):
+            continue
+
+    return sorted(external)
+
+
+def _find_js_imports(path: str, files: List[str]) -> List[str]:
+    """Find JS/TS imports that reference outside the scanned directory."""
+    external: Set[str] = set()
+    parent = os.path.dirname(path)
+
+    for f in files:
+        if not any(f.endswith(ext) for ext in (".js", ".ts", ".jsx", ".tsx")):
+            continue
+        try:
+            with open(f, "r", encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    # import ... from '../foo/bar'
+                    # require('../foo/bar')
+                    for m in re.finditer(r"""(?:from|require\()\s*['"](\.\./[^'"]+)['"]""", line):
+                        rel_import = m.group(1)
+                        abs_import = os.path.normpath(os.path.join(os.path.dirname(f), rel_import))
+                        if not abs_import.startswith(path):
+                            rel = os.path.relpath(abs_import, parent)
+                            if os.path.isdir(abs_import):
+                                external.add(f"{rel}/")
+                            else:
+                                external.add(rel)
+        except (IOError, OSError):
+            continue
+
+    return sorted(external)
+
+
+def _infer_tags(path: str, dir_name: str, files: List[str]) -> List[str]:
+    """Infer tags from directory name and file contents."""
+    tags = [dir_name.lower()]
+
+    # Infer from common file/directory names
+    name_lower = dir_name.lower()
+    tag_hints = {
+        "auth": ["authentication", "security"],
+        "api": ["rest", "endpoint"],
+        "payment": ["billing", "stripe"],
+        "user": ["account", "profile"],
+        "admin": ["dashboard", "management"],
+        "config": ["configuration", "settings"],
+        "deploy": ["infrastructure", "ci-cd"],
+        "test": ["testing"],
+        "model": ["database", "orm"],
+    }
+
+    for hint, extra_tags in tag_hints.items():
+        if hint in name_lower:
+            tags.extend(extra_tags)
+
+    return list(dict.fromkeys(tags))

dotscope/sessions.py

@@ -0,0 +1,2 @@
+"""Backward-compatibility stub. Moved to dotscope.storage.session_manager."""
+from .storage.session_manager import *  # noqa: F401,F403