codepulse-engine 0.1.0__py3-none-any.whl

codepulse/indexer/ignore.py

@@ -0,0 +1,120 @@
+"""
+Phase 0.8 -- .gitignore / .codepulseignore discovery and ignore-rule handling.
+
+Loads a repository's root-level `.gitignore` and `.codepulseignore` files
+(if present) and answers whether a given repository-relative path should be
+excluded from indexing, before that path is ever read or parsed.
+
+Semantics (per CodePulse_Strategy_and_Roadmap.md §8/§16 and roadmap sprint
+item 0.8 / backlog item B6 -- "Ignored files never appear in the index"):
+
+  - `.gitignore` is respected using standard gitignore pattern syntax
+    (§16: "Respect `.gitignore` during file discovery").
+  - `.codepulseignore` uses the "same syntax" (§16) for CodePulse-specific
+    exclusions "beyond git's own" -- i.e. ADDITIVE to `.gitignore`, never a
+    replacement/override of it.
+  - The two files are evaluated INDEPENDENTLY, each with its own negation
+    patterns resolved only within itself, and a path is excluded if EITHER
+    file's ruleset matches it. A negation (`!pattern`) in one file cannot
+    reach across and un-ignore something the other file matches -- this
+    keeps ".codepulseignore extends .gitignore" unambiguous instead of
+    inventing cross-file negation semantics the roadmap doesn't specify.
+  - Only the REPOSITORY-ROOT `.gitignore`/`.codepulseignore` are read.
+    Nested, per-directory ignore files (as real git supports) are NOT
+    implemented here -- the roadmap doesn't call for it, and it's a
+    materially larger feature than "respect the repository's ignore
+    rules." Documented limitation (see CODEPULSE_PROGRESS.md).
+  - An absent ignore file is treated as an empty ruleset -- not an error.
+  - If a *present* ignore file cannot be read, or contains a pattern
+    `pathspec` cannot parse, that file's ENTIRE ruleset is treated as
+    absent (empty) for this indexing run, and a warning is logged naming
+    the file and the error. Indexing proceeds rather than aborting, but
+    the failure is not silently swallowed, and a malformed file never
+    causes MORE or FEWER exclusions than "this file contributed nothing."
+  - The ignore files themselves (`.gitignore`, `.codepulseignore`) are not
+    given any special exemption from these rules, but in practice they
+    already never become indexed `FileModel`/`SymbolModel` rows regardless
+    -- no `LanguageParser` recognizes them (see `parser/factory.py`), and
+    `IndexingEngine.index_directory` only persists files a parser accepts.
+  - No file-size or file-type caps are implemented here (§8 lists them
+    alongside ignore-file handling, but they're a distinct, not-yet-built
+    concern -- out of this milestone's scope).
+"""
+from __future__ import annotations
+
+import logging
+import os
+
+import pathspec
+
+logger = logging.getLogger(__name__)
+
+GITIGNORE_FILENAME = ".gitignore"
+CODEPULSEIGNORE_FILENAME = ".codepulseignore"
+
+
+def _load_spec(repo_path: str, filename: str) -> pathspec.PathSpec | None:
+    """
+    Loads one root-level ignore file into a `PathSpec` using gitwildmatch
+    syntax. Returns None if the file doesn't exist, can't be read, or
+    contains a pattern that can't be parsed (logging a warning in the
+    latter two cases) -- callers treat None as "this file contributes no
+    rules," never as an error to propagate.
+    """
+    file_path = os.path.join(repo_path, filename)
+    if not os.path.isfile(file_path):
+        return None
+
+    try:
+        with open(file_path, "r", encoding="utf-8", errors="replace") as fh:
+            lines = fh.readlines()
+    except OSError as e:
+        logger.warning(
+            "Could not read %s: %s -- its rules are not applied for this indexing run.",
+            file_path,
+            e,
+        )
+        return None
+
+    try:
+        return pathspec.PathSpec.from_lines("gitignore", lines)
+    except Exception as e:
+        logger.warning(
+            "Could not parse %s (malformed pattern): %s -- its rules are not applied "
+            "for this indexing run.",
+            file_path,
+            e,
+        )
+        return None
+
+
+class RepositoryIgnoreRules:
+    """
+    Combined `.gitignore` + `.codepulseignore` rules for one repository
+    root. Loaded ONCE per indexing operation (see module docstring for
+    precedence/combination semantics) and reused for every discovered path
+    -- callers should construct one instance per `index_directory()` call,
+    not re-parse the ignore files per file/directory.
+    """
+
+    def __init__(self, repo_path: str) -> None:
+        self.repo_path = os.path.abspath(repo_path)
+        self._gitignore_spec = _load_spec(self.repo_path, GITIGNORE_FILENAME)
+        self._codepulseignore_spec = _load_spec(self.repo_path, CODEPULSEIGNORE_FILENAME)
+
+    def is_ignored(self, rel_path: str, is_dir: bool = False) -> bool:
+        """
+        `rel_path` must be repository-root-relative, using forward slashes
+        (the same path representation the rest of the indexing pipeline
+        uses for `FileParseResult`/`Symbol`/DB rows). A trailing slash is
+        added automatically for directories, matching gitwildmatch's
+        convention for directory-only patterns (e.g. `build/`).
+        """
+        rel_path = rel_path.replace("\\", "/").lstrip("/")
+        match_path = rel_path.rstrip("/") + "/" if is_dir else rel_path
+
+        if self._gitignore_spec is not None and self._gitignore_spec.match_file(match_path):
+            return True
+        if self._codepulseignore_spec is not None and self._codepulseignore_spec.match_file(match_path):
+            return True
+        return False

codepulse/ingestion_safety.py

@@ -0,0 +1,169 @@
+"""
+Phase 1.5 — Ingestion safety: path allow-list, SSRF check, size caps
+(roadmap §37 item 1.5; §15 Security Architecture's "Repository-ingestion
+safety (the dangerous endpoint)" paragraph; backlog B10, depends on B8).
+
+Roadmap text this milestone implements, verbatim (§15):
+  "for local-path ingestion, restrict to an explicit allow-listed root
+  directory (configurable), reject `..`-traversal and symlink escapes;
+  for Git-URL ingestion, validate the URL scheme (`https`/`ssh` only),
+  resolve and check the target host isn't a private/link-local IP (SSRF
+  prevention), and enforce a repository size/file-count cap before
+  beginning a full clone."
+Plus §15's "Resource limits: max files per repo, max file size parsed
+[...]".
+
+Scope, precisely (do not infer requirements from later milestones):
+  - `validate_local_path`: resolves a requested local path (following
+    symlinks, collapsing `..`) and, if one or more allow-listed roots are
+    configured, rejects it unless the resolved path is contained in one
+    of them. Catches both textual `..`-traversal and symlink-escape
+    attempts the same way, because both are neutralized by resolution
+    before the containment check ever runs.
+  - `validate_git_url`: validates a git URL's scheme is `https`/`ssh`,
+    resolves its host, and rejects it if any resolved address is
+    private/loopback/link-local/reserved/multicast (SSRF prevention).
+    This is a real, independently testable safety check -- see the note
+    below on why it exists even though this codebase has no git-clone
+    implementation yet.
+  - File-count and per-file-size caps are enforced inside
+    `IndexingEngine.index_directory()` itself (not here -- see that
+    method's own comments), because the file list they check against is
+    only available after the discovery walk `index_directory()` already
+    performs; duplicating that walk here just to pre-count would be
+    wasteful and is exactly the kind of "second identity system"/
+    duplicated-logic this project's established house style avoids.
+    `IngestionSafetyError` (this module) is the one exception type both
+    that engine-level check and this module's own boundary checks raise,
+    so `codepulse/api/app.py` has a single, consistent type to catch.
+
+IMPORTANT scope note on `validate_git_url`: this codebase has **no git
+clone/fetch implementation at all** (grep-confirmed: no `subprocess`,
+`GitPython`, or network-clone code exists anywhere). `IndexRepoRequest
+.source` (`"local"`/`"github"`/`"gitlab"`) has always been accepted but
+never actually branched on -- every prior phase's `/v1/repositories/index`
+treated `req.url` as a local filesystem path regardless of `source`,
+silently indexing zero files for a real git URL. Implementing actual
+git-URL cloning is a much larger, unrequested feature no sprint item in
+this roadmap assigns a milestone to; it is NOT this milestone's job to
+invent it. What Phase 1.5 *does* own, literally, is the "SSRF check" --
+so this module implements and tests that check as a real, standalone
+safety primitive (ready for whenever a real fetch path exists), and
+`codepulse/api/app.py` now honestly rejects a non-local `source` (after
+still validating the URL for safety first) with HTTP 501, rather than
+continuing to silently mis-process it as a phantom local path.
+
+Deliberately NOT in scope for Phase 1.5 (do not infer these belong here):
+  - Actually cloning/fetching a git repository (see above).
+  - Request-body size caps (roadmap §15 lists this as a separate
+    "Resource limit" alongside file/repo caps; FastAPI/Starlette/uvicorn
+    request-body size limits are a server/ASGI-layer configuration
+    concern, not application code this milestone adds).
+  - Max concurrent indexing jobs "per tenant" (roadmap §15) -- Phase 1.3
+    already added a *per-repository* advisory one-active-job lock; a
+    global/per-tenant concurrent-job cap is a distinct, unrequested
+    control this milestone does not add.
+  - A structured `{"error": {...}}` response envelope -- Phase 1.6.
+  - A dedicated security test suite -- Phase 1.7.
+
+This module lives outside `codepulse/indexer/` and `codepulse/api/`
+deliberately -- a top-level module, mirroring `codepulse/auth.py`,
+`codepulse/jobs.py`, and `codepulse/rate_limit.py`'s placement, per the
+roadmap's "control plane"/API-boundary grouping of these safety concerns,
+reusable identically by a future CLI or MCP surface.
+"""
+from __future__ import annotations
+
+import ipaddress
+import os
+import socket
+from pathlib import Path
+from urllib.parse import urlparse
+
+ALLOWED_GIT_URL_SCHEMES = ("https", "ssh")
+
+
+class IngestionSafetyError(Exception):
+    """
+    Raised for any ingestion request rejected on safety grounds: a local
+    path outside the configured allow-list (or escaping it via `..`/a
+    symlink), an unsafe git URL scheme, an SSRF-unsafe resolved host, or
+    (raised by `IndexingEngine.index_directory` itself, not this module)
+    a repository exceeding the configured file-count cap.
+    """
+
+
+def validate_local_path(path: str, allowed_roots: list[str]) -> str:
+    """
+    Resolves `path` (following symlinks, collapsing `..`) and, if
+    `allowed_roots` is non-empty, rejects it unless the resolved path is
+    contained in (or equal to) one of them. Returns the resolved,
+    canonical path on success.
+
+    An empty `allowed_roots` means "unrestricted" -- see
+    `CodePulseConfig.ingestion_allowed_roots`'s docstring for why that is
+    this project's deliberate default, preserving the local-first
+    zero-configuration experience.
+    """
+    resolved = str(Path(path).resolve())
+
+    if allowed_roots:
+        for root in allowed_roots:
+            resolved_root = str(Path(root).resolve())
+            if resolved == resolved_root or resolved.startswith(resolved_root + os.sep):
+                break
+        else:
+            raise IngestionSafetyError(
+                f"Path '{path}' resolves to '{resolved}', which is outside every "
+                f"configured ingestion_allowed_roots entry"
+            )
+
+    # Checked AFTER the allow-list on purpose: reporting "does not exist"
+    # for a path outside the allowed roots would let a caller probe which
+    # paths exist outside them. A nonexistent path used to be accepted and
+    # silently created an empty phantom repository (2026-10-02 audit).
+    if not os.path.isdir(resolved):
+        raise IngestionSafetyError(f"Path '{path}' is not an existing directory")
+
+    return resolved
+
+
+def validate_git_url(url: str) -> None:
+    """
+    Validates a git URL's scheme and resolved host for SSRF safety.
+    Raises `IngestionSafetyError` if the scheme isn't https/ssh, the host
+    can't be resolved, or any resolved address is
+    private/loopback/link-local/reserved/multicast/unspecified.
+    """
+    parsed = urlparse(url)
+
+    if parsed.scheme not in ALLOWED_GIT_URL_SCHEMES:
+        raise IngestionSafetyError(
+            f"Unsupported git URL scheme '{parsed.scheme}' -- only "
+            f"{'/'.join(ALLOWED_GIT_URL_SCHEMES)} are allowed"
+        )
+
+    host = parsed.hostname
+    if not host:
+        raise IngestionSafetyError(f"Git URL '{url}' has no resolvable host")
+
+    try:
+        resolved = socket.getaddrinfo(host, None)
+    except socket.gaierror as e:
+        raise IngestionSafetyError(f"Could not resolve host '{host}': {e}") from e
+
+    for family, _, _, _, sockaddr in resolved:
+        ip = ipaddress.ip_address(sockaddr[0])
+        if (
+            ip.is_private
+            or ip.is_loopback
+            or ip.is_link_local
+            or ip.is_reserved
+            or ip.is_multicast
+            or ip.is_unspecified
+        ):
+            raise IngestionSafetyError(
+                f"Host '{host}' resolves to '{sockaddr[0]}', a "
+                f"private/loopback/link-local/reserved address -- rejected "
+                f"as an SSRF risk"
+            )

codepulse/jobs.py

@@ -0,0 +1,257 @@
+"""
+Phase 1.3 — IndexJob state machine + /v1/jobs/{id} (roadmap §37 item 1.3;
+§8 "Concurrency/failure handling"; §21; backlog B9).
+
+Scope, precisely (do not infer requirements from later milestones):
+  - Create a real, DB-backed job row for one indexing run, in the initial
+    `queued` state.
+  - Transition it through the roadmap §8 state machine: `queued ->
+    running -> succeeded|failed`. `cancelled` is a valid schema value
+    (see `IndexJobModel`) but this module exposes no way to reach it --
+    no cancel endpoint/flow exists yet.
+  - An "advisory lock" (roadmap §8's own wording) preventing a second
+    concurrent indexing run for the same repository: `get_active_job`
+    lets a caller check for -- and refuse to start alongside -- an
+    existing `queued`/`running` job for that repository. This is an
+    application-level check, not a hard database constraint: this
+    project runs indexing as in-process background tasks, not multiple
+    concurrent worker processes, so a stronger lock is not needed at this
+    scale, and the roadmap itself only calls for "advisory."
+  - A read path (`get_job`) returning everything `GET /v1/jobs/{job_id}`
+    needs to report real, measured status -- never a fabricated/estimated
+    one.
+
+Deliberately NOT in scope for Phase 1.3 (reserved for later milestones --
+do not infer these belong here):
+  - A heartbeat-based crash reaper reclaiming a `running` job whose worker
+    died (roadmap §17 mentions this as a *reliability* mechanism, but no
+    sprint item assigns it a specific milestone; this project has no
+    subprocess/worker infrastructure for a "crashed worker" to even be a
+    distinct case from "this process's background task raised" -- the
+    latter IS handled here, via `mark_failed`, since `run_indexing`'s own
+    try/except in `api/app.py` calls it on any exception). Backend-hardening
+    addition (not a roadmap sprint item): `reap_stale_jobs` below IS that
+    reaper, added later -- see its own docstring for why a startup-time
+    reconciliation, not a heartbeat/timeout, is the correct mechanism for
+    this project's actual single-process architecture.
+  - `RepositorySnapshot` / git commit tracking / incremental indexing
+    (roadmap §9, explicitly Phase 3 sprint items 3.1-3.5) -- this module
+    has no concept of a snapshot; `IndexingEngine.index_directory()` is
+    still a full re-index every time, unchanged by this milestone.
+  - A cancel endpoint/flow, job expiry/cleanup, rate limiting (Phase 1.4),
+    ingestion safety (Phase 1.5), a structured error envelope (Phase 1.6).
+
+This module lives outside `codepulse/indexer/` deliberately, mirroring
+`codepulse/auth.py`'s placement: `IndexingEngine.index_directory()` has no
+knowledge of jobs at all and is not modified by this milestone -- the
+caller (currently `codepulse/api/app.py`'s background task) is entirely
+responsible for calling `mark_running`/`mark_succeeded`/`mark_failed`
+around its own call to `index_directory()`. It is a top-level module
+(not under `codepulse/api/`) so a future CLI or MCP surface that also
+triggers indexing (roadmap: "REST, CLI, and MCP must call the same
+underlying intelligence functions") can reuse it identically, exactly as
+`codepulse/auth.py` is already structured for reuse beyond REST alone.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from codepulse.storage.db import IndexJobModel, StorageManager
+
+ACTIVE_STATUSES = ("queued", "running")
+TERMINAL_STATUSES = ("succeeded", "failed", "cancelled")
+
+
+@dataclass(frozen=True)
+class JobInfo:
+    """
+    A snapshot of one IndexJob row's state, safe to hold after its
+    originating session has closed (see `StorageManager.SessionLocal`'s
+    `expire_on_commit=True` default -- the same reason
+    `codepulse.auth.CreatedAPIKey` is a frozen dataclass rather than a raw
+    ORM object, established in Phase 1.1).
+    """
+
+    id: int
+    repo_id: int
+    status: str
+    created_at: datetime
+    started_at: datetime | None
+    finished_at: datetime | None
+    error_message: str | None
+    result: dict[str, Any] | None
+
+
+def _to_job_info(job: IndexJobModel) -> JobInfo:
+    return JobInfo(
+        id=job.id,
+        repo_id=job.repo_id,
+        status=job.status,
+        created_at=job.created_at,
+        started_at=job.started_at,
+        finished_at=job.finished_at,
+        error_message=job.error_message,
+        result=json.loads(job.result_summary) if job.result_summary else None,
+    )
+
+
+def get_active_job(storage: StorageManager, repo_id: int) -> JobInfo | None:
+    """
+    Roadmap §8: "one active job per repository (advisory lock on
+    repo_id)". Returns the most recent `queued`/`running` job for
+    `repo_id`, if any -- callers use this to refuse starting a second
+    concurrent indexing run for the same repository, rather than letting
+    two runs write conflicting data at once.
+    """
+    session = storage.get_session()
+    try:
+        job = (
+            session.query(IndexJobModel)
+            .filter(IndexJobModel.repo_id == repo_id, IndexJobModel.status.in_(ACTIVE_STATUSES))
+            .order_by(IndexJobModel.id.desc())
+            .first()
+        )
+        return _to_job_info(job) if job else None
+    finally:
+        session.close()
+
+
+def create_job(storage: StorageManager, repo_id: int) -> JobInfo:
+    """Creates a new IndexJob row in the initial `queued` state."""
+    session = storage.get_session()
+    try:
+        job = IndexJobModel(repo_id=repo_id, status="queued", created_at=datetime.utcnow())
+        session.add(job)
+        session.commit()
+        return _to_job_info(job)
+    finally:
+        session.close()
+
+
+def mark_running(storage: StorageManager, job_id: int) -> None:
+    """Transitions a job `queued -> running`, recording `started_at`."""
+    session = storage.get_session()
+    try:
+        job = session.query(IndexJobModel).filter(IndexJobModel.id == job_id).first()
+        if job is not None:
+            job.status = "running"
+            job.started_at = datetime.utcnow()
+            session.commit()
+    finally:
+        session.close()
+    # Phase 7.4 (roadmap §37 item 7.4; §18: "indexing_jobs_total{status}"):
+    # real state-transition counter, incremented at the exact moment
+    # this transition actually happens -- not a synthetic/estimated value.
+    from codepulse.observability import indexing_jobs_total
+    indexing_jobs_total.labels(status="running").inc()
+
+
+def mark_succeeded(storage: StorageManager, job_id: int, result: dict[str, Any]) -> None:
+    """
+    Transitions a job `running -> succeeded`, recording `finished_at` and
+    `result` -- `result` must be the real, measured return value of
+    `IndexingEngine.index_directory()` (never an estimated/fabricated
+    summary), per this project's "no fabricated metrics" invariant.
+    """
+    session = storage.get_session()
+    try:
+        job = session.query(IndexJobModel).filter(IndexJobModel.id == job_id).first()
+        if job is not None:
+            job.status = "succeeded"
+            job.finished_at = datetime.utcnow()
+            job.result_summary = json.dumps(result)
+            session.commit()
+    finally:
+        session.close()
+    from codepulse.observability import indexing_jobs_total
+    indexing_jobs_total.labels(status="succeeded").inc()
+
+
+def mark_failed(storage: StorageManager, job_id: int, error: str) -> None:
+    """Transitions a job `running -> failed`, recording `finished_at` and `error`."""
+    session = storage.get_session()
+    try:
+        job = session.query(IndexJobModel).filter(IndexJobModel.id == job_id).first()
+        if job is not None:
+            job.status = "failed"
+            job.finished_at = datetime.utcnow()
+            job.error_message = error
+            session.commit()
+    finally:
+        session.close()
+    from codepulse.observability import indexing_jobs_total
+    indexing_jobs_total.labels(status="failed").inc()
+
+
+def reap_stale_jobs(storage: StorageManager) -> list[int]:
+    """
+    Backend-hardening addition (not a roadmap sprint item) -- closes the
+    one remaining gap this module's own docstring has disclosed since
+    Phase 1.3: a `running` job whose background task is killed by a real
+    process crash used to stay `running` forever, permanently blocking
+    `get_active_job`'s advisory lock for that repository (every future
+    indexing attempt would see an "active" job that can never finish and
+    refuse to start).
+
+    Deliberately a STARTUP-TIME reconciliation, not a heartbeat/timeout
+    mechanism: this project runs indexing as in-process `BackgroundTasks`
+    coroutines (roadmap §8's own "advisory lock... this project runs
+    indexing as in-process background tasks, not multiple concurrent
+    worker processes" framing, already established for `get_active_job`),
+    with no persisted "which process owns this job" tracking and no
+    heartbeat column anywhere in the schema. In that architecture, a job
+    still `queued`/`running` at the moment a NEW `IndexingEngine` is
+    constructed can only be a leftover from a PREVIOUS process that no
+    longer exists -- the current process could not have created it (it
+    just started), and no other process could be concurrently progressing
+    it (this project has no distributed worker pool). A timeout-based
+    heuristic would need a new column, a migration, and an arbitrary
+    staleness threshold that either reaps a genuinely slow-but-healthy
+    job too early or a genuinely crashed one too late; this mechanism
+    needs neither and cannot produce a false positive for THIS
+    architecture's actual failure mode.
+
+    Reuses `mark_failed` (not a hand-rolled column update) so this stays
+    the single source of truth for what a `-> failed` transition means,
+    including its own existing `indexing_jobs_total{status="failed"}`
+    metric increment -- a reaped job is counted exactly like any other
+    real failure, not a separate, uncounted code path.
+
+    Called once from `IndexingEngine.__init__`, mirroring `hydrate_all`'s
+    own "reconcile whatever happened before this process began"
+    philosophy and timing. Returns the ids of every job reaped (empty if
+    none were stale) -- primarily for logging/testing, not a public API.
+    """
+    session = storage.get_session()
+    try:
+        stale_ids = [
+            job.id
+            for job in session.query(IndexJobModel)
+            .filter(IndexJobModel.status.in_(ACTIVE_STATUSES))
+            .all()
+        ]
+    finally:
+        session.close()
+
+    for job_id in stale_ids:
+        mark_failed(
+            storage,
+            job_id,
+            "Reaped at startup: this job was still queued/running when the "
+            "previous process ended (crash, restart, or forced shutdown) "
+            "and can never complete.",
+        )
+    return stale_ids
+
+
+def get_job(storage: StorageManager, job_id: int) -> JobInfo | None:
+    """Returns the current state of one job, or None if no such job exists."""
+    session = storage.get_session()
+    try:
+        job = session.query(IndexJobModel).filter(IndexJobModel.id == job_id).first()
+        return _to_job_info(job) if job else None
+    finally:
+        session.close()