devtime-ei 0.1.0__py3-none-any.whl

devtime/scanner/ignore.py

@@ -0,0 +1,96 @@
+"""Ignore-rule matching (Builder Edition, Chapter 7).
+
+Respects .gitignore and .devtimeignore, and always refuses secrets, VCS internals,
+and common generated directories even if the ignore files are missing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pathspec
+
+# Always-on safety net. These must never become evidence (Chapter 7 safety rules).
+HARD_DENY = [
+    ".git/",
+    ".env",
+    ".env.*",
+    "*.pem",
+    "*.key",
+    "*.p12",
+    "*.pfx",
+    "id_rsa",
+    "id_ed25519",
+    "secrets.*",
+    "credentials.*",
+    "service-account*.json",
+    ".devtime/",
+]
+
+# Directories pruned before traversal regardless of ignore files (Reality
+# Hardening): never descend into these, so large trees are skipped, not filtered.
+PRUNE_DIRS = {
+    "node_modules", ".git", "dist", "build", "coverage", ".next", "out",
+    "__pycache__", ".pytest_cache", ".venv", "venv", ".cache", ".devtime",
+    ".turbo", ".sst", ".open-next", ".svelte-kit", ".nuxt",
+}
+
+
+def is_pruned_dirname(name: str) -> bool:
+    return name in PRUNE_DIRS or name.endswith(".egg-info")
+
+
+BINARY_EXTENSIONS = {
+    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".svg",
+    ".pdf", ".zip", ".gz", ".tar", ".7z", ".rar",
+    ".woff", ".woff2", ".ttf", ".eot", ".otf",
+    ".mp3", ".mp4", ".mov", ".avi", ".wav",
+    ".exe", ".dll", ".so", ".dylib", ".bin", ".class", ".o", ".a",
+    ".lock", ".sqlite", ".db",
+}
+
+
+class IgnoreMatcher:
+    def __init__(self, patterns: list[str]):
+        self._spec = pathspec.PathSpec.from_lines("gitignore", patterns)
+
+    def match(self, rel_path: str) -> bool:
+        return self._spec.match_file(rel_path)
+
+    def match_dir(self, rel_dir: str) -> bool:
+        """True if a directory is ignored (so it can be pruned before traversal).
+
+        gitignore patterns may target a directory with or without a trailing
+        slash, so both forms are checked.
+        """
+        rel_dir = rel_dir.rstrip("/")
+        return self._spec.match_file(rel_dir) or self._spec.match_file(rel_dir + "/")
+
+
+def _read_patterns(path: Path) -> list[str]:
+    if not path.exists():
+        return []
+    out: list[str] = []
+    for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
+        line = line.strip()
+        if line and not line.startswith("#"):
+            out.append(line)
+    return out
+
+
+def build_matcher(
+    root: Path,
+    *,
+    respect_gitignore: bool = True,
+    respect_devtimeignore: bool = True,
+) -> IgnoreMatcher:
+    patterns = list(HARD_DENY)
+    if respect_gitignore:
+        patterns += _read_patterns(root / ".gitignore")
+    if respect_devtimeignore:
+        patterns += _read_patterns(root / ".devtimeignore")
+    return IgnoreMatcher(patterns)
+
+
+def is_binary_extension(extension: str) -> bool:
+    return extension.lower() in BINARY_EXTENSIONS

devtime/scanner/language.py

@@ -0,0 +1,36 @@
+"""Language and file-role classification (Builder Edition, Chapter 7)."""
+
+from __future__ import annotations
+
+TEST_HINTS = (".test.", ".spec.", "_test.", "test_")
+TEST_DIRS = ("/tests/", "/test/", "/__tests__/", "tests/", "test/")
+DOC_DIRS = ("/docs/", "docs/")
+DOC_EXTS = (".md", ".mdx", ".rst")
+
+
+def classify_language(path: str) -> str | None:
+    if path.endswith((".ts", ".tsx", ".js", ".jsx")):
+        return "typescript"
+    if path.endswith(".py"):
+        return "python"
+    if path.endswith((".md", ".mdx")):
+        return "markdown"
+    if path.endswith((".yml", ".yaml")):
+        return "yaml"
+    if path.endswith(".json"):
+        return "json"
+    return None
+
+
+def is_test_path(rel_path: str) -> bool:
+    lower = rel_path.lower()
+    if any(hint in lower for hint in TEST_HINTS):
+        return True
+    return any(d in lower for d in TEST_DIRS)
+
+
+def is_doc_path(rel_path: str) -> bool:
+    lower = rel_path.lower()
+    if lower.endswith(DOC_EXTS):
+        return True
+    return any(d in lower for d in DOC_DIRS)

devtime/scanner/signals.py

@@ -0,0 +1,252 @@
+"""Scan orchestration (Builder Edition, Chapter 7 pipeline).
+
+scan repository -> load config -> load ignore rules -> walk files safely ->
+classify -> hash -> extract signals -> persist files and signals -> detect
+concepts -> build evidence -> generate claims -> compute scores -> write summary.
+
+Never executes repository code, never makes network calls.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devtime import __version__, config, paths
+from devtime.db import connection, migrations, repository
+from devtime.intelligence import claims as claims_mod
+from devtime.intelligence import concepts as concepts_mod
+from devtime.intelligence import evidence as evidence_mod
+from devtime.scanner import ignore as ignore_mod
+from devtime.scanner.extractors import (
+    config_files,
+    docs,
+    nextjs,
+    python,
+    tests,
+    typescript,
+)
+from devtime.scanner.extractors.base import Signal
+from devtime.scanner.file_walker import WalkedFile, walk_repository
+from devtime.scanner.language import classify_language
+
+
+SUPPORTED_CONCEPT_FAMILIES = (
+    "Authentication", "Billing Webhooks", "Background Jobs",
+    "Data Export", "Admin Permissions", "File Uploads",
+)
+
+# Frameworks whose routes/controllers V0 does not parse, keyed by a dependency token.
+UNSUPPORTED_FRAMEWORKS = {
+    "django": "Django routes are not parsed in V0; route coverage may be incomplete.",
+    "@nestjs/core": "NestJS controller parsing is not supported in V0; backend route coverage may be incomplete.",
+    "@nestjs/common": "NestJS controller parsing is not supported in V0; backend route coverage may be incomplete.",
+    "rails": "Rails routes are not parsed in V0; route coverage may be incomplete.",
+    "laravel/framework": "Laravel routes are not parsed in V0; route coverage may be incomplete.",
+}
+
+
+@dataclass
+class ScanResult:
+    file_count: int
+    signal_count: int
+    concept_count: int
+    intelligence: list[claims_mod.ConceptIntelligence]
+    duration_seconds: float = 0.0
+    pruned_dirs: int = 0
+    skipped_files: int = 0
+    framework_warnings: list[str] = field(default_factory=list)
+
+
+def detect_framework_warnings(signals: list[Signal]) -> list[str]:
+    """Surface coverage gaps for frameworks V0 cannot parse (Trust Repair v0.0.6)."""
+    deps = {
+        (s.name or "").lower()
+        for s in signals
+        if s.kind == "dependency" and s.name
+    }
+    warnings: list[str] = []
+    seen: set[str] = set()
+    for token, message in UNSUPPORTED_FRAMEWORKS.items():
+        if token in deps and message not in seen:
+            warnings.append(f"Framework coverage warning: {message}")
+            seen.add(message)
+    return warnings
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _hash_file(path: Path) -> str:
+    h = hashlib.sha256()
+    try:
+        with path.open("rb") as fh:
+            for chunk in iter(lambda: fh.read(65536), b""):
+                h.update(chunk)
+    except OSError:
+        return ""
+    return h.hexdigest()
+
+
+def _is_migration_path(rel_path: str) -> bool:
+    """DB migrations describe schema history, not live behavior or workers.
+
+    Reality Validation finding: an Alembic migration named *_add_jobs.py was being
+    used as Background Jobs evidence. Migration files must not produce
+    concept-defining signals.
+    """
+    low = rel_path.lower()
+    return (
+        "/migrations/" in low
+        or low.startswith("migrations/")
+        or "/alembic/versions/" in low
+        or "alembic/versions/" in low
+    )
+
+
+def _extract(file: WalkedFile, language: str | None) -> list[Signal]:
+    # Migrations are scanned/hashed but never become concept evidence.
+    if _is_migration_path(file.rel_path):
+        return []
+
+    out: list[Signal] = []
+    if file.is_test:
+        out += tests.extract_test_signals(file)
+    if file.is_doc or language == "markdown":
+        out += docs.extract_doc_signals(file)
+    if language == "typescript":
+        out += typescript.extract_typescript_signals(file)
+        out += nextjs.extract_nextjs_signals(file)
+    elif language == "python":
+        out += python.extract_python_signals(file)
+    # Config/manifest extraction by filename, regardless of language.
+    out += config_files.extract_config_signals(file)
+    return out
+
+
+def run_scan(
+    root: Path | None = None, *, refresh: bool = False, progress: bool = False
+) -> ScanResult:
+    import sys
+    import time
+
+    started = time.perf_counter()
+    root = root or paths.repo_root()
+    if not paths.is_initialized(root):
+        migrations.init_repo(root)
+
+    cfg = config.load_config(root)
+    scanner_cfg = cfg["scanner"]
+    max_bytes = int(scanner_cfg["max_file_size_kb"]) * 1024
+
+    matcher = ignore_mod.build_matcher(
+        root,
+        respect_gitignore=bool(scanner_cfg["respect_gitignore"]),
+        respect_devtimeignore=bool(scanner_cfg["respect_devtimeignore"]),
+    )
+
+    conn = connection.connect(root)
+    try:
+        repo_id = repository.add_repository_if_missing(conn, root)
+        scan_id = f"scan-{uuid.uuid4().hex[:10]}"
+        conn.execute(
+            "INSERT INTO scans(id, repository_id, started_at, status, devtime_version) "
+            "VALUES (?,?,?,?,?)",
+            (scan_id, repo_id, _now(), "running", __version__),
+        )
+        conn.commit()
+
+        all_signals: list[Signal] = []
+        file_count = 0
+        walk_stats: dict = {"pruned_dirs": 0, "skipped_files": 0}
+
+        if progress:
+            print("Scanning locally. No code execution. No network.", file=sys.stderr)
+
+        for wf in walk_repository(
+            root, matcher, max_bytes,
+            follow_symlinks=bool(scanner_cfg["follow_symlinks"]),
+            stats=walk_stats,
+        ):
+            if progress and file_count and file_count % 1000 == 0:
+                print(f"Progress: {file_count} files scanned...", file=sys.stderr)
+            language = classify_language(wf.rel_path)
+            file_id = f"file-{uuid.uuid4().hex[:10]}"
+            conn.execute(
+                "INSERT OR REPLACE INTO files"
+                "(id, repository_id, path, extension, language, sha256, size_bytes, "
+                " is_test, is_doc, ignored, last_seen_scan_id) "
+                "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+                (
+                    file_id,
+                    repo_id,
+                    wf.rel_path,
+                    wf.extension,
+                    language,
+                    _hash_file(wf.path),
+                    wf.size_bytes,
+                    1 if wf.is_test else 0,
+                    1 if wf.is_doc else 0,
+                    0,
+                    scan_id,
+                ),
+            )
+            file_count += 1
+
+            for s in _extract(wf, language):
+                all_signals.append(s)
+                conn.execute(
+                    "INSERT INTO signals"
+                    "(id, scan_id, file_id, kind, name, value, start_line, end_line, "
+                    " confidence, metadata_json) VALUES (?,?,?,?,?,?,?,?,?,?)",
+                    (
+                        f"sig-{uuid.uuid4().hex[:10]}",
+                        scan_id,
+                        file_id,
+                        s.kind,
+                        s.name,
+                        s.value,
+                        s.start_line,
+                        s.end_line,
+                        s.confidence,
+                        "{}",
+                    ),
+                )
+        conn.commit()
+
+        # Detect concepts -> build evidence -> generate claims + uncertainty.
+        candidates = concepts_mod.detect_concepts(all_signals)
+        intelligence: list[claims_mod.ConceptIntelligence] = []
+        for cand in candidates:
+            ev = evidence_mod.build_evidence(cand)
+            ci = claims_mod.build_concept_intelligence(cand, ev)
+            intelligence.append(ci)
+
+        # Persist derived memory (replace machine-derived rows; keep human decisions).
+        repository.clear_derived_memory(conn)
+        repository.save_intelligence(conn, repo_id, intelligence)
+
+        conn.execute(
+            "UPDATE scans SET finished_at=?, status=?, file_count=?, signal_count=? "
+            "WHERE id=?",
+            (_now(), "completed", file_count, len(all_signals), scan_id),
+        )
+        conn.commit()
+
+        return ScanResult(
+            file_count=file_count,
+            signal_count=len(all_signals),
+            concept_count=len(intelligence),
+            intelligence=intelligence,
+            duration_seconds=round(time.perf_counter() - started, 2),
+            pruned_dirs=walk_stats.get("pruned_dirs", 0),
+            skipped_files=walk_stats.get("skipped_files", 0),
+            framework_warnings=detect_framework_warnings(all_signals),
+        )
+    finally:
+        conn.close()

devtime_ei-0.1.0.dist-info/METADATA

@@ -0,0 +1,289 @@
+Metadata-Version: 2.4
+Name: devtime-ei
+Version: 0.1.0
+Summary: Local-first Engineering Intelligence for software repositories
+Author-email: Aviad Shakargi <aviad94@gmail.com>
+Maintainer-email: Aviad Shakargi <aviad94@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/Shakargy/devtime
+Project-URL: Repository, https://github.com/Shakargy/devtime
+Project-URL: Issues, https://github.com/Shakargy/devtime/issues
+Project-URL: Release Notes, https://github.com/Shakargy/devtime/releases/tag/v0.1.0
+Project-URL: Demo, https://youtu.be/1Hiu3Y9J_SI
+Keywords: devtools,cli,static-analysis,repository-analysis,engineering-intelligence,local-first
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.12
+Requires-Dist: rich>=13.7
+Requires-Dist: pydantic>=2.7
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pathspec>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# DevTime
+
+**Local-first Engineering Intelligence for software repositories.**
+
+DevTime helps a codebase explain itself from evidence.
+
+It scans code, tests, configs, routes, and decisions to identify supported software
+concepts, link claims to files, surface uncertainty, and warn about a narrow set of
+risky changes.
+
+> No cloud. No telemetry. No code execution. No AI required.
+
+[![DevTime demo - Repository memory from evidence](assets/devtime-demo-thumbnail-v0.1.0.png)](https://youtu.be/1Hiu3Y9J_SI)
+
+Watch the 2-minute demo: DevTime scans a repo locally, explains concepts from
+evidence, surfaces uncertainty, catches a risky diff, and shows how a corroborated
+decision improves understanding.
+
+---
+
+## Try DevTime in 60 seconds
+
+```bash
+git clone https://github.com/Shakargy/devtime.git
+cd devtime
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+cd examples/demo-saas
+dtc init
+dtc scan
+dtc concepts
+dtc explain "Billing Webhooks"
+```
+
+On Windows PowerShell:
+
+```powershell
+git clone https://github.com/Shakargy/devtime.git
+cd devtime
+python -m venv .venv
+.venv\Scripts\Activate.ps1
+pip install -e ".[dev]"
+cd examples/demo-saas
+dtc init
+dtc scan
+dtc concepts
+dtc explain "Billing Webhooks"
+```
+
+You should see Billing Webhooks explained from evidence, including supported claims,
+file references, uncertainty, Understanding Score, and Understanding Debt.
+
+To test risk review, make a local change first, then run:
+
+```bash
+dtc risk --diff
+```
+
+A full, copy-pasteable walkthrough (including the risk-diff and corroborated-decision
+steps) is in **[DEMO_SCRIPT.md](DEMO_SCRIPT.md)**.
+
+## Why this exists
+
+Git remembers code. It does not remember understanding. It does not tell you why a
+behavior exists, what evidence supports it, or what nobody has decided yet. As AI
+tools generate code faster than teams can review it, that missing understanding
+becomes the bottleneck.
+
+DevTime builds evidence-backed repository memory: a local layer that says what a
+repository can prove, and - just as importantly - what it cannot prove yet.
+
+## Who it is for
+
+DevTime is for developers reviewing unfamiliar code, teams using AI coding tools, and
+maintainers who want repository understanding to be backed by evidence instead of
+generated summaries.
+
+It is useful when you want to ask:
+
+- where is authentication actually implemented?
+- what files prove that billing webhooks exist?
+- what is still uncertain?
+- did this diff touch a risky concept?
+- is there a decision explaining this behavior?
+
+## What DevTime does
+
+- Detects concepts from routes, tests, configs, dependencies, and docs.
+- Explains from evidence by linking claims to files and signals.
+- Surfaces uncertainty when evidence is missing or weak.
+- Scores understanding with an Understanding Score and Understanding Debt label.
+- Reviews narrow risky diffs with advisory findings from local memory.
+- Records decisions locally so rationale can reduce uncertainty when corroborated by code.
+
+## Supported concepts
+
+V0 detects six supported concept families. It does not discover arbitrary domain
+concepts yet:
+
+- Authentication
+- Billing Webhooks
+- Background Jobs
+- Data Export
+- Admin Permissions
+- File Uploads
+
+Anything outside these six is out of scope for V0. See [LIMITATIONS.md](LIMITATIONS.md).
+
+## What DevTime does not do
+
+- It does not execute your code.
+- It does not send code or data over the network.
+- It does not require or call an AI model.
+- It does not guarantee correctness or safe changes.
+- It does not replace code review or architecture decisions.
+- It is **not** a documentation generator, a static analyzer, an observability tool,
+  a productivity tracker, or an AI coding agent.
+
+## Trust model
+
+- DevTime stores local repository memory in `.devtime/` (a local SQLite database).
+- **No network access** during a scan.
+- **No code execution** during a scan.
+- Ignored directories are pruned *before* scanning; ignored files and secrets must
+  never become evidence or claims.
+- Every claim must link to evidence - *no claim without evidence*.
+- Weak evidence produces **uncertainty**, not confidence.
+- *Usage is not decision*: that a dependency is used does not mean someone decided why.
+- Risk review is **advisory** by default - it does not block PRs.
+
+## Commands
+
+| Command | Purpose |
+|---------|---------|
+| `dtc init` | Create local `.devtime` memory. |
+| `dtc scan` | Scan the current repository and extract evidence-backed signals. |
+| `dtc concepts` | List detected concepts with confidence and Understanding Debt. |
+| `dtc explain <concept>` | Explain a concept: claims, evidence, confidence, uncertainty, Understanding Debt. |
+| `dtc context <concept>` | Create a governed Context Pack for agents or humans. |
+| `dtc risk --diff` | Review a git diff for risky changes using local evidence (advisory). |
+| `dtc decision add` | Add a local decision record that can reduce uncertainty. |
+
+(Also available: `dtc evidence`, `dtc debt`, `dtc status`, `dtc doctor --privacy`,
+`dtc export`, `dtc reset`.)
+
+Requires **Python >= 3.11** and git. See **[QUICKSTART.md](QUICKSTART.md)** for a
+step-by-step first run and troubleshooting.
+
+## Installation
+
+Recommended source install for now:
+
+```bash
+git clone https://github.com/Shakargy/devtime.git
+cd devtime
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+```
+
+Windows PowerShell:
+
+```powershell
+git clone https://github.com/Shakargy/devtime.git
+cd devtime
+python -m venv .venv
+.venv\Scripts\Activate.ps1
+pip install -e ".[dev]"
+```
+
+Planned PyPI install (after the package is published and verified):
+
+```bash
+pipx install devtime
+```
+
+## Example output
+
+```
+$ dtc explain "Billing Webhooks"
+Concept: Billing Webhooks
+
+Supported claims:
+  - Billing Webhooks is present and supported by behavior evidence.
+    type: concept  confidence: 0.86  evidence: src/billing/stripe-webhook.ts, tests/stripe-signature.test.ts
+  - Billing Webhooks has active route handling.
+    type: behavior  confidence: 0.82  evidence: src/billing/stripe-webhook.ts
+  - Billing Webhooks verifies webhook signatures.
+    type: behavior  confidence: 0.85  evidence: src/billing/stripe-webhook.ts
+
+Uncertainty:
+  - No decision was found explaining key choices for Billing Webhooks.
+
+Understanding Score: 58 / 100
+Understanding Debt: medium
+causes:
+  - missing or uncorroborated decision evidence
+  - no confirmed owner
+```
+
+> Understanding Score is higher = better understanding; Understanding Debt is a
+> label (low/medium/high), not the same number.
+
+## Proof
+
+DevTime runs on `examples/demo-saas` and on real repositories. During Reality
+Validation it detected - and then learned from - real failures (Next.js App Router
+blindness, a false Billing Webhooks detection on a generic webhook system, a DB
+migration mis-counted as Background Jobs evidence, and more). Each failure became a
+fixture so it cannot silently regress.
+
+- Tests grew from 13 to 88 as real failures became fixtures.
+- Scan time on a 355-file real repo dropped from ~27.3s to ~0.48s after ignored-
+  directory pruning.
+
+Full evidence, before/after examples, and the validation reports are in
+**[PROOF.md](PROOF.md)** and `reports/reality-validation/`.
+
+## Privacy and safety
+
+- Runs entirely locally; nothing leaves your machine during a scan.
+- No code execution and no network calls during scanning.
+- Secrets and ignored files are excluded from evidence by design (`dtc doctor
+  --privacy` reports the boundaries).
+- `dtc reset` deletes local memory; your source code is never modified.
+
+## Known limitations
+
+DevTime is a **heuristic scanner**, not a full compiler or semantic analyzer. It is
+currently strongest on TypeScript / Next.js / Express / FastAPI-style repositories
+that resemble its fixtures. False positives and false negatives are possible.
+Understanding Debt is a product signal, not an objective universal truth.
+
+Read the full list - including framework coverage, risk-review scope, and what is
+intentionally not built yet - in **[LIMITATIONS.md](LIMITATIONS.md)**.
+
+## Roadmap
+
+This is an early, local-first V0 focused on being trustworthy before being large.
+Not yet built (intentionally): git-history signals, wired MCP transport, an AI
+provider, a UI, and any cloud/team/enterprise features. See **[ROADMAP.md](ROADMAP.md)**.
+
+## Contributing
+
+The most valuable contribution is a **fixture**: a small repository pattern plus the
+expected concepts, allowed claims, forbidden claims, and required uncertainty. If
+DevTime gets something wrong on your code, that wrong output can become a fixture so
+it never regresses. See **[CONTRIBUTING.md](CONTRIBUTING.md)**.
+
+## License
+
+Licensed under the **Apache License 2.0**. See [LICENSE](LICENSE).