devtime-ei 0.1.0__py3-none-any.whl

devtime/privacy.py

@@ -0,0 +1,96 @@
+"""Privacy and security implementation (Builder Edition, Chapter 19).
+
+Privacy is architecture, not a footer.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from devtime import config, paths
+
+# Secret patterns (Chapter 19).
+SECRET_PATTERNS = [
+    r"AKIA[0-9A-Z]{16}",
+    r"-----BEGIN PRIVATE KEY-----",
+    r"sk-[A-Za-z0-9_-]{20,}",
+    r"""(?i)(api_key|secret|token|password)\s*=\s*['"][^'"]+['"]""",
+]
+
+
+def redact_secret_like_values(text: str) -> str:
+    for pattern in SECRET_PATTERNS:
+        text = re.sub(pattern, "<redacted-secret>", text)
+    return text
+
+
+def privacy_report(root: Path | None = None) -> dict:
+    root = root or paths.repo_root()
+    cfg = config.load_config(root)
+    priv = cfg["privacy"]
+
+    good: list[str] = []
+    warning: list[str] = []
+    recommended: list[str] = []
+
+    good.append("AI disabled" if not priv["ai_enabled"] else "AI enabled")
+    good.append("Cloud disabled" if not priv["cloud_enabled"] else "Cloud enabled")
+    good.append("Telemetry off" if not priv["telemetry_enabled"] else "Telemetry on")
+
+    if (root / ".devtimeignore").exists():
+        good.append(".devtimeignore active")
+    if (root / ".env").exists():
+        good.append(".env ignored")
+
+    status = _devtime_ignored(root)
+    if status is True:
+        good.append(".devtime/ is git-ignored")
+    elif status is False:
+        warning.append(".devtime/ is not ignored by git")
+        recommended.append(
+            "Add `.devtime/` to `.gitignore` unless you intentionally share local memory."
+        )
+    else:  # unknown (git unavailable)
+        warning.append("Could not confirm whether .devtime/ is ignored (git unavailable)")
+        recommended.append(
+            "Verify `.devtime/` is ignored, e.g. add it to `.gitignore`."
+        )
+
+    return {"good": good, "warning": warning, "recommended": recommended}
+
+
+def _devtime_ignored(root: Path) -> bool | None:
+    """Return True/False if .devtime/ is git-ignored, or None if undeterminable.
+
+    Trust Repair (v0.0.6): prefer `git check-ignore`, which honors parent
+    .gitignore rules, so a nested repo whose parent ignores .devtime/ is not
+    falsely warned. Falls back to a local .gitignore text check.
+    """
+    import subprocess
+
+    # Probe a path *inside* .devtime/ so the dir-only pattern matches even when the
+    # directory does not exist yet, and so parent .gitignore rules are honored.
+    target = ".devtime/devtime.sqlite"
+    try:
+        proc = subprocess.run(
+            ["git", "check-ignore", "-q", target],
+            cwd=str(root),
+            capture_output=True,
+            text=True,
+        )
+        # exit 0 = ignored, 1 = not ignored, 128 = not a git repo / error.
+        if proc.returncode == 0:
+            return True
+        if proc.returncode == 1:
+            return False
+    except FileNotFoundError:
+        pass
+
+    # Fallback: local .gitignore text only (cannot see parent rules).
+    gitignore = root / ".gitignore"
+    if gitignore.exists() and ".devtime/" in gitignore.read_text(
+        encoding="utf-8", errors="ignore"
+    ):
+        return True
+    return None

devtime/scanner/extractors/base.py

@@ -0,0 +1,83 @@
+"""Shared signal model for extractors (Builder Edition, Chapter 8).
+
+A signal is a small extracted fact. It does not have to be perfect, but it must
+be typed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from devtime.scanner.file_walker import WalkedFile
+
+
+@dataclass
+class Signal:
+    kind: str
+    name: str | None
+    file_rel_path: str
+    value: str | None = None
+    start_line: int | None = None
+    end_line: int | None = None
+    confidence: float = 0.5
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def signal(
+    kind: str,
+    *,
+    name: str | None = None,
+    file: WalkedFile,
+    value: str | None = None,
+    start_line: int | None = None,
+    end_line: int | None = None,
+    confidence: float = 0.5,
+    metadata: dict[str, Any] | None = None,
+) -> Signal:
+    return Signal(
+        kind=kind,
+        name=name,
+        file_rel_path=file.rel_path,
+        value=value,
+        start_line=start_line,
+        end_line=end_line,
+        confidence=confidence,
+        metadata=metadata or {},
+    )
+
+
+def read_text(file: WalkedFile) -> str:
+    try:
+        return file.path.read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return ""
+
+
+def classify_jwt_purpose(text: str, rel_path: str) -> str:
+    """Classify what a JWT is used for (Trust Repair v0.0.6).
+
+    Returns "access", "invitation", or "unclear". Invitation/verification tokens
+    are not access tokens and must not be claimed as such.
+    """
+    hay = (text + " " + rel_path).lower()
+    access_signals = (
+        "access token", "accesstoken", "access_token", "bearer", "authorization",
+        "login", "signin", "sign-in", "refresh token", "refresh_token",
+        "req.cookies", "set-cookie", "auth middleware", "current_user",
+        "get_current_user",
+    )
+    invitation_signals = (
+        "invite", "invitation", "verify email", "verify-email", "email_verification",
+        "password reset", "password-reset", "reset_token", "magic link", "magic-link",
+        "one-time", "onetime",
+    )
+    has_access = any(s in hay for s in access_signals)
+    has_invite = any(s in hay for s in invitation_signals)
+    if has_access and not has_invite:
+        return "access"
+    if has_invite and not has_access:
+        return "invitation"
+    if has_access and has_invite:
+        return "access"  # an access path that also issues invites still does access auth
+    return "unclear"

devtime/scanner/extractors/config_files.py

@@ -0,0 +1,41 @@
+"""Config and dependency-manifest extractor (Builder Edition, Chapter 8)."""
+
+from __future__ import annotations
+
+import json
+import re
+
+from devtime.scanner.extractors.base import Signal, read_text, signal
+from devtime.scanner.file_walker import WalkedFile
+
+_ENV_REF_RE = re.compile(r"\b([A-Z][A-Z0-9_]{3,})\b")
+
+
+def extract_config_signals(file: WalkedFile) -> list[Signal]:
+    name = file.path.name.lower()
+    text = read_text(file)
+    signals: list[Signal] = []
+
+    # package.json / requirements: dependency signals.
+    if name == "package.json":
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError:
+            data = {}
+        for section in ("dependencies", "devDependencies"):
+            for dep in (data.get(section) or {}):
+                signals.append(signal("dependency", name=dep, file=file, confidence=0.6))
+    elif name in ("requirements.txt", "requirements-dev.txt"):
+        for line in text.splitlines():
+            dep = re.split(r"[=<>!~ ]", line.strip(), 1)[0]
+            if dep and not dep.startswith("#"):
+                signals.append(signal("dependency", name=dep, file=file, confidence=0.6))
+
+    # .env.example and config files: env var names are concept hints (never values).
+    if name.startswith(".env") or name.endswith((".env", ".env.example")):
+        for match in _ENV_REF_RE.finditer(text):
+            signals.append(
+                signal("config", name=match.group(1), file=file, confidence=0.5)
+            )
+
+    return signals

devtime/scanner/extractors/docs.py

@@ -0,0 +1,35 @@
+"""Docs and decision-record extractor (Builder Edition, Chapter 8)."""
+
+from __future__ import annotations
+
+import re
+
+from devtime.scanner.extractors.base import Signal, read_text, signal
+from devtime.scanner.file_walker import WalkedFile
+
+_HEADING_RE = re.compile(r"^#{1,3}\s+(.+?)\s*$", re.M)
+
+
+def extract_doc_signals(file: WalkedFile) -> list[Signal]:
+    text = read_text(file)
+    signals: list[Signal] = []
+
+    is_decision = "/decisions/" in file.rel_path.lower() or re.match(
+        r"^\d{3,4}-", file.path.name
+    )
+
+    for match in _HEADING_RE.finditer(text):
+        heading = match.group(1).strip()
+        line = text.count("\n", 0, match.start()) + 1
+        kind = "decision" if is_decision else "doc"
+        signals.append(
+            signal(
+                kind,
+                name=heading,
+                file=file,
+                start_line=line,
+                confidence=0.7 if is_decision else 0.4,
+            )
+        )
+
+    return signals

devtime/scanner/extractors/nextjs.py

@@ -0,0 +1,82 @@
+"""Next.js App Router signal extractor.
+
+Added during V0 Reality Validation: Snapilio and SaaSVoice are Next.js App Router
+apps whose API surface is file-based (`app/api/**/route.ts` exporting HTTP method
+handlers). The Express/FastAPI extractors saw none of it, so every concept
+degraded to weak dependency evidence. This extractor parses route handlers and
+derives the route path from the directory structure.
+"""
+
+from __future__ import annotations
+
+import re
+
+from devtime.scanner.extractors.base import Signal, read_text, signal
+from devtime.scanner.file_walker import WalkedFile
+
+# export async function GET(...) / export function POST(...) / export const DELETE = ...
+_METHOD_RE = re.compile(
+    r"export\s+(?:async\s+)?function\s+(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\b"
+    r"|export\s+const\s+(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s*=",
+)
+
+_ROUTE_FILES = {"route.ts", "route.js", "route.tsx", "route.jsx"}
+
+
+def is_app_router_route(rel_path: str) -> bool:
+    name = rel_path.rsplit("/", 1)[-1]
+    if name not in _ROUTE_FILES:
+        return False
+    return "/app/" in rel_path or rel_path.startswith("app/")
+
+
+def derive_route_path(rel_path: str) -> str:
+    """Turn app/api/(payments)/checkout/[id]/route.ts -> /api/checkout/[id].
+
+    Route groups like (payments) are stripped; dynamic segments like [id] and
+    [...slug] are kept. Paths are normalised so no empty segments survive.
+    """
+    parts = rel_path.split("/")
+    # Locate the segment named "app" (handles src/app/... too); start after it.
+    try:
+        app_idx = len(parts) - 1 - parts[::-1].index("app")
+    except ValueError:
+        app_idx = -1
+    segments = parts[app_idx + 1 : -1]  # drop everything up to app/ and the route file
+
+    cleaned: list[str] = []
+    for seg in segments:
+        if not seg:
+            continue
+        # Route groups (parentheses) and parallel/intercept routes do not affect the URL.
+        if seg.startswith("(") and seg.endswith(")"):
+            continue
+        if seg.startswith("@"):
+            continue
+        cleaned.append(seg)
+    return "/" + "/".join(cleaned)
+
+
+def extract_nextjs_signals(file: WalkedFile) -> list[Signal]:
+    if not is_app_router_route(file.rel_path):
+        return []
+    text = read_text(file)
+    methods: list[str] = []
+    for match in _METHOD_RE.finditer(text):
+        methods.append(match.group(1) or match.group(2))
+    if not methods:
+        return []
+
+    route_path = derive_route_path(file.rel_path)
+    signals: list[Signal] = []
+    for method in methods:
+        signals.append(
+            signal(
+                "route",
+                name=f"{method} {route_path}",
+                file=file,
+                confidence=0.82,
+                metadata={"method": method, "path": route_path, "framework": "nextjs"},
+            )
+        )
+    return signals

devtime/scanner/extractors/python.py

@@ -0,0 +1,81 @@
+"""Python signal extractor (Builder Edition, Chapter 8)."""
+
+from __future__ import annotations
+
+import re
+
+from devtime.scanner.extractors.base import (
+    Signal,
+    classify_jwt_purpose,
+    read_text,
+    signal,
+)
+from devtime.scanner.file_walker import WalkedFile
+
+_IMPORT_RE = re.compile(r"""^\s*(?:from\s+(\w[\w.]*)\s+import|import\s+(\w[\w.]*))""", re.M)
+_ROUTE_RE = re.compile(
+    r"""@(?:app|router)\.(get|post|put|patch|delete)\(\s*['"]([^'"]+)['"]"""
+)
+
+
+def extract_python_signals(file: WalkedFile) -> list[Signal]:
+    text = read_text(file)
+    signals: list[Signal] = []
+
+    for match in _IMPORT_RE.finditer(text):
+        module = (match.group(1) or match.group(2) or "").split(".")[0]
+        if module:
+            signals.append(signal("dependency", name=module, file=file, confidence=0.6))
+
+    for match in _ROUTE_RE.finditer(text):
+        method = match.group(1).upper()
+        path = match.group(2)
+        signals.append(
+            signal(
+                "route",
+                name=f"{method} {path}",
+                file=file,
+                confidence=0.8,
+                metadata={"method": method, "path": path, "framework": "fastapi"},
+            )
+        )
+
+    if "Depends(" in text and ("current_user" in text or "get_current_user" in text):
+        signals.append(
+            signal("auth_dependency", name="current_user", file=file, confidence=0.75)
+        )
+
+    if "@celery.task" in text or ".task(" in text or "@shared_task" in text:
+        signals.append(
+            signal("background_job", name=file.rel_path, file=file, confidence=0.75)
+        )
+
+    if re.search(r"\bjwt\.(encode|decode)\b|\bPyJWT\b", text):
+        purpose = classify_jwt_purpose(text, file.rel_path)
+        signals.append(
+            signal(
+                "token_usage",
+                name="jwt",
+                file=file,
+                confidence=0.8,
+                metadata={"purpose": purpose},
+            )
+        )
+
+    # File uploads: FastAPI UploadFile / multipart / werkzeug request.files.
+    if re.search(r"\bUploadFile\b|=\s*File\(|multipart/form-data|request\.files", text):
+        signals.append(
+            signal("upload_endpoint", name="upload", file=file, confidence=0.8)
+        )
+
+    if "stripe.Webhook.construct_event" in text:
+        signals.append(
+            signal(
+                "webhook_signature_verification",
+                name="stripe",
+                file=file,
+                confidence=0.9,
+            )
+        )
+
+    return signals

devtime/scanner/extractors/tests.py

@@ -0,0 +1,61 @@
+"""Test-file extractor (Builder Edition, Chapter 8).
+
+Behavior-specific tests are strong evidence, so test names are first-class signals.
+"""
+
+from __future__ import annotations
+
+import re
+
+from devtime.scanner.extractors.base import Signal, read_text, signal
+from devtime.scanner.file_walker import WalkedFile
+
+_TEST_NAME_RE = re.compile(
+    r"""(?:it|test|describe)\(\s*['"]([^'"]+)['"]"""  # JS/TS
+    r"""|def\s+(test_\w+)"""  # pytest
+)
+# Imports a test references - used to attach tests to the implementation they cover
+# (Evidence Precision v0.0.7): a truthful "imports the implementation" reason.
+_IMPORT_RE = re.compile(
+    r"""from\s+([\w.]+)\s+import|import\s+([\w.]+)"""  # python
+    r"""|from\s+['"]([^'"]+)['"]""",  # JS/TS
+)
+
+
+def _extract_imports(text: str) -> list[str]:
+    mods: list[str] = []
+    for m in _IMPORT_RE.finditer(text):
+        mod = m.group(1) or m.group(2) or m.group(3)
+        if mod:
+            mods.append(mod.lower())
+    return mods
+
+
+def _is_e2e(rel_path: str) -> bool:
+    """E2E UI specs match concept keywords by accident and should be weak evidence.
+
+    Reality Validation finding: `tests-e2e/specs/sidebar-navigation.e2e.spec.ts`
+    was defining File Uploads, and other e2e specs polluted Data Export.
+    """
+    low = rel_path.lower()
+    return ".e2e." in low or "/tests-e2e/" in low or "/e2e/" in low or "playwright" in low
+
+
+def extract_test_signals(file: WalkedFile) -> list[Signal]:
+    text = read_text(file)
+    e2e = _is_e2e(file.rel_path)
+    imports = _extract_imports(text)
+    signals: list[Signal] = []
+    for match in _TEST_NAME_RE.finditer(text):
+        name = match.group(1) or match.group(2)
+        if name:
+            signals.append(
+                signal(
+                    "test",
+                    name=name,
+                    file=file,
+                    confidence=0.4 if e2e else 0.8,
+                    metadata={"e2e": e2e, "imports": imports},
+                )
+            )
+    return signals

devtime/scanner/extractors/typescript.py

@@ -0,0 +1,99 @@
+"""TypeScript / JavaScript signal extractor (Builder Edition, Chapter 8)."""
+
+from __future__ import annotations
+
+import re
+
+from devtime.scanner.extractors.base import (
+    Signal,
+    classify_jwt_purpose,
+    read_text,
+    signal,
+)
+from devtime.scanner.file_walker import WalkedFile
+
+_IMPORT_RE = re.compile(r"""import\s+.*?from\s+['"]([^'"]+)['"]""")
+# Match app/router as well as named routers (authRouter, exportRouter, etc).
+_ROUTE_RE = re.compile(
+    r"""\b(?:app|\w*[Rr]outer)\.(get|post|put|patch|delete)\(\s*['"]([^'"]+)['"]""",
+    re.I,
+)
+_MIDDLEWARE_RE = re.compile(
+    r"""\b(requireAuth|authMiddleware|isAuthenticated|ensureAuth|requireAdmin)\b"""
+)
+_BULLMQ_WORKER_RE = re.compile(r"""new\s+Worker\(\s*['"]([^'"]+)['"]""")
+_BULLMQ_QUEUE_RE = re.compile(r"""new\s+Queue\(\s*['"]([^'"]+)['"]""")
+
+
+def extract_typescript_signals(file: WalkedFile) -> list[Signal]:
+    text = read_text(file)
+    signals: list[Signal] = []
+
+    for match in _IMPORT_RE.finditer(text):
+        module = match.group(1)
+        # Skip relative imports; external dependencies are the useful signal.
+        if module.startswith("."):
+            continue
+        signals.append(signal("dependency", name=module, file=file, confidence=0.6))
+
+    for match in _ROUTE_RE.finditer(text):
+        method = match.group(1).upper()
+        path = match.group(2)
+        signals.append(
+            signal(
+                "route",
+                name=f"{method} {path}",
+                file=file,
+                confidence=0.8,
+                metadata={"method": method, "path": path, "framework": "express"},
+            )
+        )
+
+    if _MIDDLEWARE_RE.search(text):
+        signals.append(
+            signal("middleware", name="auth", file=file, confidence=0.7)
+        )
+
+    if "stripe.webhooks.constructEvent" in text:
+        signals.append(
+            signal(
+                "webhook_signature_verification",
+                name="stripe",
+                file=file,
+                confidence=0.9,
+            )
+        )
+
+    if re.search(r"\bjsonwebtoken\b|\bjwt\.(sign|verify)\b", text):
+        purpose = classify_jwt_purpose(text, file.rel_path)
+        signals.append(
+            signal(
+                "token_usage",
+                name="jwt",
+                file=file,
+                confidence=0.8,
+                metadata={"purpose": purpose},
+            )
+        )
+
+    # File uploads: multipart / multer / busboy / formData with a file part.
+    if re.search(r"multipart/form-data|\bmulter\b|\bbusboy\b|\.formData\(", text):
+        signals.append(
+            signal("upload_endpoint", name="upload", file=file, confidence=0.75)
+        )
+
+    for match in _BULLMQ_WORKER_RE.finditer(text):
+        signals.append(
+            signal(
+                "background_job",
+                name=f"worker:{match.group(1)}",
+                file=file,
+                confidence=0.8,
+            )
+        )
+    for match in _BULLMQ_QUEUE_RE.finditer(text):
+        signals.append(
+            signal("queue", name=f"queue:{match.group(1)}", file=file, confidence=0.8)
+        )
+
+    return signals

devtime/scanner/file_walker.py

@@ -0,0 +1,96 @@
+"""Safe repository file walker (Builder Edition, Chapter 7).
+
+Walks files without executing repository code. Ignored directories are pruned
+*before* traversal (Reality Hardening): the walker never descends into
+node_modules, .git, build output, .devtime, etc., instead of walking them and
+filtering after the fact. Symlinks, large files, binaries, and ignored/secret
+files are still skipped per file.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from devtime.scanner import ignore as ignore_mod
+from devtime.scanner.language import is_doc_path, is_test_path
+
+
+@dataclass
+class WalkedFile:
+    path: Path
+    rel_path: str
+    size_bytes: int
+    extension: str
+    is_test: bool
+    is_doc: bool
+
+
+def _normalize(rel: str) -> str:
+    while "//" in rel:
+        rel = rel.replace("//", "/")
+    return rel
+
+
+def walk_repository(
+    root: Path,
+    ignore_matcher: ignore_mod.IgnoreMatcher,
+    max_size_bytes: int,
+    *,
+    follow_symlinks: bool = False,
+    stats: dict | None = None,
+):
+    root = Path(root)
+    for dirpath, dirnames, filenames in os.walk(root, followlinks=follow_symlinks):
+        rel_dir = _normalize(Path(dirpath).relative_to(root).as_posix())
+
+        # --- Prune ignored directories before descending into them. ---
+        kept: list[str] = []
+        for d in sorted(dirnames):
+            if ignore_mod.is_pruned_dirname(d):
+                if stats is not None:
+                    stats["pruned_dirs"] = stats.get("pruned_dirs", 0) + 1
+                continue
+            child_rel = d if rel_dir in ("", ".") else f"{rel_dir}/{d}"
+            child_rel = _normalize(child_rel)
+            if ignore_matcher.match_dir(child_rel):
+                if stats is not None:
+                    stats["pruned_dirs"] = stats.get("pruned_dirs", 0) + 1
+                continue
+            child_path = Path(dirpath) / d
+            if child_path.is_symlink() and not follow_symlinks:
+                continue
+            kept.append(d)
+        dirnames[:] = kept  # in-place prune controls os.walk descent
+
+        for fn in sorted(filenames):
+            rel = _normalize(fn if rel_dir in ("", ".") else f"{rel_dir}/{fn}")
+            if ignore_matcher.match(rel):
+                if stats is not None:
+                    stats["skipped_files"] = stats.get("skipped_files", 0) + 1
+                continue
+            path = Path(dirpath) / fn
+            if path.is_symlink() and not follow_symlinks:
+                continue
+            extension = path.suffix.lower()
+            if ignore_mod.is_binary_extension(extension):
+                if stats is not None:
+                    stats["skipped_files"] = stats.get("skipped_files", 0) + 1
+                continue
+            try:
+                size = path.stat().st_size
+            except OSError:
+                continue
+            if size > max_size_bytes:
+                if stats is not None:
+                    stats["skipped_files"] = stats.get("skipped_files", 0) + 1
+                continue
+            yield WalkedFile(
+                path=path,
+                rel_path=rel,
+                size_bytes=size,
+                extension=extension,
+                is_test=is_test_path(rel),
+                is_doc=is_doc_path(rel),
+            )