ado-git-repo-insights 1.2.1__py3-none-any.whl

ado_git_repo_insights/persistence/models.py

@@ -0,0 +1,145 @@
+"""SQLite database schema and models for ado-git-repo-insights.
+
+This module defines the SQLite schema that maps directly to the CSV output contract.
+Schema changes must preserve invariants 1-4, 14-16 from INVARIANTS.md.
+"""
+
+from __future__ import annotations
+
+# SQL schema that will be executed to create tables
+# Mirrors the CSV output contract exactly
+
+SCHEMA_SQL = """
+-- Metadata table for incremental extraction state (Invariant 6)
+CREATE TABLE IF NOT EXISTS extraction_metadata (
+    id INTEGER PRIMARY KEY,
+    organization_name TEXT NOT NULL,
+    project_name TEXT NOT NULL,
+    last_extraction_date TEXT NOT NULL,  -- ISO 8601 (YYYY-MM-DD)
+    last_extraction_timestamp TEXT NOT NULL,  -- ISO 8601 with time
+    UNIQUE(organization_name, project_name)
+);
+
+-- Core entity tables (matching CSV output contract - Invariants 1-4)
+
+-- organizations.csv: organization_name
+CREATE TABLE IF NOT EXISTS organizations (
+    organization_name TEXT PRIMARY KEY
+);
+
+-- projects.csv: organization_name, project_name
+CREATE TABLE IF NOT EXISTS projects (
+    organization_name TEXT NOT NULL,
+    project_name TEXT NOT NULL,
+    PRIMARY KEY (organization_name, project_name),
+    FOREIGN KEY (organization_name) REFERENCES organizations(organization_name)
+);
+
+-- repositories.csv: repository_id, repository_name, project_name, organization_name
+-- Invariant 14: repository_id is the stable ADO ID
+CREATE TABLE IF NOT EXISTS repositories (
+    repository_id TEXT PRIMARY KEY,
+    repository_name TEXT NOT NULL,
+    project_name TEXT NOT NULL,
+    organization_name TEXT NOT NULL,
+    FOREIGN KEY (organization_name, project_name)
+        REFERENCES projects(organization_name, project_name)
+);
+CREATE INDEX IF NOT EXISTS idx_repositories_project
+    ON repositories(organization_name, project_name);
+
+-- users.csv: user_id, display_name, email
+-- Invariant 16: user_id is stable ADO ID, display_name/email are mutable labels
+CREATE TABLE IF NOT EXISTS users (
+    user_id TEXT PRIMARY KEY,
+    display_name TEXT NOT NULL,
+    email TEXT
+);
+
+-- pull_requests.csv: pull_request_uid, pull_request_id, organization_name, project_name,
+--                    repository_id, user_id, title, status, description,
+--                    creation_date, closed_date, cycle_time_minutes
+-- Invariant 14: pull_request_uid = {repository_id}-{pull_request_id}
+CREATE TABLE IF NOT EXISTS pull_requests (
+    pull_request_uid TEXT PRIMARY KEY,
+    pull_request_id INTEGER NOT NULL,
+    organization_name TEXT NOT NULL,
+    project_name TEXT NOT NULL,
+    repository_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    title TEXT NOT NULL,
+    status TEXT NOT NULL,
+    description TEXT,
+    creation_date TEXT NOT NULL,  -- ISO 8601
+    closed_date TEXT,             -- ISO 8601
+    cycle_time_minutes REAL,
+    raw_json TEXT,                -- Original ADO response for auditing
+    FOREIGN KEY (repository_id) REFERENCES repositories(repository_id),
+    FOREIGN KEY (user_id) REFERENCES users(user_id)
+);
+CREATE INDEX IF NOT EXISTS idx_pull_requests_closed_date
+    ON pull_requests(closed_date);
+CREATE INDEX IF NOT EXISTS idx_pull_requests_org_project
+    ON pull_requests(organization_name, project_name);
+
+-- reviewers.csv: pull_request_uid, user_id, vote, repository_id
+CREATE TABLE IF NOT EXISTS reviewers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    pull_request_uid TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    vote INTEGER NOT NULL,
+    repository_id TEXT NOT NULL,
+    FOREIGN KEY (pull_request_uid) REFERENCES pull_requests(pull_request_uid),
+    FOREIGN KEY (user_id) REFERENCES users(user_id),
+    UNIQUE(pull_request_uid, user_id)  -- One vote per reviewer per PR
+);
+CREATE INDEX IF NOT EXISTS idx_reviewers_pr ON reviewers(pull_request_uid);
+
+-- Schema version for future migrations
+CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER PRIMARY KEY,
+    applied_at TEXT NOT NULL
+);
+
+-- Insert initial schema version
+INSERT OR IGNORE INTO schema_version (version, applied_at)
+VALUES (1, datetime('now'));
+"""
+
+# CSV column order contract (NON-NEGOTIABLE per Invariants 1-4)
+CSV_SCHEMAS: dict[str, list[str]] = {
+    "organizations": ["organization_name"],
+    "projects": ["organization_name", "project_name"],
+    "repositories": [
+        "repository_id",
+        "repository_name",
+        "project_name",
+        "organization_name",
+    ],
+    "pull_requests": [
+        "pull_request_uid",
+        "pull_request_id",
+        "organization_name",
+        "project_name",
+        "repository_id",
+        "user_id",
+        "title",
+        "status",
+        "description",
+        "creation_date",
+        "closed_date",
+        "cycle_time_minutes",
+    ],
+    "users": ["user_id", "display_name", "email"],
+    "reviewers": ["pull_request_uid", "user_id", "vote", "repository_id"],
+}
+
+# Deterministic row ordering: primary key + tie-breaker (Adjustment 3)
+SORT_KEYS: dict[str, list[str]] = {
+    "organizations": ["organization_name"],
+    "projects": ["organization_name", "project_name"],
+    "repositories": ["repository_id"],
+    "pull_requests": ["pull_request_uid", "creation_date"],
+    "users": ["user_id"],
+    "reviewers": ["pull_request_uid", "user_id"],
+}

ado_git_repo_insights/persistence/repository.py

@@ -0,0 +1,376 @@
+"""Data access layer for ado-git-repo-insights.
+
+This module implements UPSERT operations and state tracking per Invariant 8
+(idempotent and convergent state updates).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ExtractionMetadata:
+    """Metadata about the last extraction for a project."""
+
+    organization_name: str
+    project_name: str
+    last_extraction_date: date
+    last_extraction_timestamp: datetime
+
+
+class PRRepository:
+    """Data access layer for Pull Request data.
+
+    Invariant 8: State updates must be idempotent and converge.
+    Invariant 14: Stable identifiers are required for UPSERT keys.
+    Invariant 15: All entities must be scoped to organization + project.
+    """
+
+    def __init__(self, db: DatabaseManager) -> None:
+        """Initialize the repository.
+
+        Args:
+            db: Database manager instance.
+        """
+        self.db = db
+
+    # --- Extraction Metadata ---
+
+    def get_last_extraction_date(self, organization: str, project: str) -> date | None:
+        """Get the last successful extraction date for a project.
+
+        Args:
+            organization: Organization name.
+            project: Project name.
+
+        Returns:
+            Last extraction date, or None if never extracted or metadata is corrupt.
+        """
+        cursor = self.db.execute(
+            """
+            SELECT last_extraction_date FROM extraction_metadata
+            WHERE organization_name = ? AND project_name = ?
+            """,
+            (organization, project),
+        )
+        row = cursor.fetchone()
+        if row:
+            date_value = row["last_extraction_date"]
+            # Handle NULL or empty string
+            if not date_value:
+                return None
+            # Handle corrupt date format gracefully (warn + fallback)
+            try:
+                return date.fromisoformat(date_value)
+            except (ValueError, TypeError) as e:
+                logger.warning(
+                    f"Invalid/corrupt extraction metadata date for "
+                    f"{organization}/{project}: '{date_value}' - {e}"
+                )
+                return None
+        return None
+
+    def update_extraction_metadata(
+        self, organization: str, project: str, extraction_date: date
+    ) -> None:
+        """Record successful extraction for the given date.
+
+        Args:
+            organization: Organization name.
+            project: Project name.
+            extraction_date: Date that was extracted.
+        """
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO extraction_metadata
+            (organization_name, project_name, last_extraction_date, last_extraction_timestamp)
+            VALUES (?, ?, ?, ?)
+            """,
+            (
+                organization,
+                project,
+                extraction_date.isoformat(),
+                datetime.now(timezone.utc).isoformat(),
+            ),
+        )
+        logger.debug(
+            f"Updated extraction metadata: {organization}/{project} = {extraction_date}"
+        )
+
+    # --- Organizations ---
+
+    def upsert_organization(self, organization_name: str) -> None:
+        """Insert or update an organization.
+
+        Args:
+            organization_name: Organization name.
+        """
+        self.db.execute(
+            "INSERT OR IGNORE INTO organizations (organization_name) VALUES (?)",
+            (organization_name,),
+        )
+
+    # --- Projects ---
+
+    def upsert_project(self, organization_name: str, project_name: str) -> None:
+        """Insert or update a project.
+
+        Args:
+            organization_name: Organization name.
+            project_name: Project name.
+        """
+        # Ensure organization exists first
+        self.upsert_organization(organization_name)
+
+        self.db.execute(
+            """
+            INSERT OR IGNORE INTO projects (organization_name, project_name)
+            VALUES (?, ?)
+            """,
+            (organization_name, project_name),
+        )
+
+    # --- Repositories ---
+
+    def upsert_repository(
+        self,
+        repository_id: str,
+        repository_name: str,
+        project_name: str,
+        organization_name: str,
+    ) -> None:
+        """Insert or update a repository.
+
+        Invariant 14: repository_id is the stable ADO ID.
+        Invariant 16: repository_name is a mutable label.
+
+        Args:
+            repository_id: Stable ADO repository ID.
+            repository_name: Current repository name.
+            project_name: Project name.
+            organization_name: Organization name.
+        """
+        # Ensure project exists first
+        self.upsert_project(organization_name, project_name)
+
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO repositories
+            (repository_id, repository_name, project_name, organization_name)
+            VALUES (?, ?, ?, ?)
+            """,
+            (repository_id, repository_name, project_name, organization_name),
+        )
+
+    # --- Users ---
+
+    def upsert_user(
+        self, user_id: str, display_name: str, email: str | None = None
+    ) -> None:
+        """Insert or update a user.
+
+        Invariant 16: user_id is stable, display_name/email are mutable.
+
+        Args:
+            user_id: Stable ADO user ID.
+            display_name: Current display name.
+            email: Current email (optional).
+        """
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO users (user_id, display_name, email)
+            VALUES (?, ?, ?)
+            """,
+            (user_id, display_name, email),
+        )
+
+    # --- Pull Requests ---
+
+    def upsert_pull_request(
+        self,
+        pull_request_uid: str,
+        pull_request_id: int,
+        organization_name: str,
+        project_name: str,
+        repository_id: str,
+        user_id: str,
+        title: str,
+        status: str,
+        description: str | None,
+        creation_date: str,
+        closed_date: str | None,
+        cycle_time_minutes: float | None,
+        raw_json: dict[str, Any] | None = None,
+    ) -> None:
+        """Insert or update a pull request.
+
+        Invariant 8: UPSERT semantics ensure idempotent updates.
+        Invariant 14: pull_request_uid = {repository_id}-{pull_request_id}.
+
+        Args:
+            pull_request_uid: Unique identifier (repo_id-pr_id).
+            pull_request_id: ADO PR ID.
+            organization_name: Organization name.
+            project_name: Project name.
+            repository_id: Repository ID.
+            user_id: Author user ID.
+            title: PR title.
+            status: PR status.
+            description: PR description.
+            creation_date: ISO 8601 creation date.
+            closed_date: ISO 8601 closed date.
+            cycle_time_minutes: Calculated cycle time.
+            raw_json: Original ADO API response for auditing.
+        """
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO pull_requests (
+                pull_request_uid, pull_request_id, organization_name, project_name,
+                repository_id, user_id, title, status, description,
+                creation_date, closed_date, cycle_time_minutes, raw_json
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                pull_request_uid,
+                pull_request_id,
+                organization_name,
+                project_name,
+                repository_id,
+                user_id,
+                title,
+                status,
+                description,
+                creation_date,
+                closed_date,
+                cycle_time_minutes,
+                json.dumps(raw_json) if raw_json else None,
+            ),
+        )
+
+    # --- Reviewers ---
+
+    def upsert_reviewer(
+        self,
+        pull_request_uid: str,
+        user_id: str,
+        vote: int,
+        repository_id: str,
+    ) -> None:
+        """Insert or update a reviewer.
+
+        Args:
+            pull_request_uid: PR unique identifier.
+            user_id: Reviewer user ID.
+            vote: Vote value.
+            repository_id: Repository ID.
+        """
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO reviewers
+            (pull_request_uid, user_id, vote, repository_id)
+            VALUES (?, ?, ?, ?)
+            """,
+            (pull_request_uid, user_id, vote, repository_id),
+        )
+
+    # --- Bulk Operations ---
+
+    def upsert_pr_with_related(
+        self,
+        pr_data: dict[str, Any],
+        organization_name: str,
+        project_name: str,
+    ) -> None:
+        """Insert or update a PR and all related entities.
+
+        This is the main entry point for processing a PR from the ADO API.
+        Handles repository, user, reviewers, and the PR itself.
+
+        Args:
+            pr_data: Raw PR data from ADO API.
+            organization_name: Organization name.
+            project_name: Project name.
+        """
+        from ..utils.datetime_utils import calculate_cycle_time_minutes
+
+        # Extract repository
+        repo = pr_data.get("repository", {})
+        repository_id = repo.get("id", "")
+        repository_name = repo.get("name", "")
+
+        self.upsert_repository(
+            repository_id=repository_id,
+            repository_name=repository_name,
+            project_name=project_name,
+            organization_name=organization_name,
+        )
+
+        # Extract author
+        created_by = pr_data.get("createdBy", {})
+        user_id = created_by.get("id", "")
+        display_name = created_by.get("displayName", "")
+        email = created_by.get("uniqueName")
+
+        self.upsert_user(
+            user_id=user_id,
+            display_name=display_name,
+            email=email,
+        )
+
+        # Build PR UID (Invariant 14)
+        pr_id = pr_data.get("pullRequestId", 0)
+        pull_request_uid = f"{repository_id}-{pr_id}"
+
+        # Calculate cycle time
+        creation_date = pr_data.get("creationDate", "")
+        closed_date = pr_data.get("closedDate")
+        cycle_time = calculate_cycle_time_minutes(creation_date, closed_date)
+
+        # Upsert PR
+        self.upsert_pull_request(
+            pull_request_uid=pull_request_uid,
+            pull_request_id=pr_id,
+            organization_name=organization_name,
+            project_name=project_name,
+            repository_id=repository_id,
+            user_id=user_id,
+            title=pr_data.get("title", ""),
+            status=pr_data.get("status", ""),
+            description=pr_data.get("description"),
+            creation_date=creation_date,
+            closed_date=closed_date,
+            cycle_time_minutes=cycle_time,
+            raw_json=pr_data,
+        )
+
+        # Upsert reviewers
+        for reviewer in pr_data.get("reviewers", []):
+            reviewer_id = reviewer.get("id", "")
+            reviewer_name = reviewer.get("displayName", "")
+            reviewer_email = reviewer.get("uniqueName")
+            vote = reviewer.get("vote", 0)
+
+            self.upsert_user(
+                user_id=reviewer_id,
+                display_name=reviewer_name,
+                email=reviewer_email,
+            )
+
+            self.upsert_reviewer(
+                pull_request_uid=pull_request_uid,
+                user_id=reviewer_id,
+                vote=vote,
+                repository_id=repository_id,
+            )
+
+        logger.debug(f"Upserted PR: {pull_request_uid}")

ado_git_repo_insights/transform/__init__.py

@@ -0,0 +1 @@
+"""Transform module for CSV generation."""

ado_git_repo_insights/transform/csv_generator.py

@@ -0,0 +1,132 @@
+"""CSV generator for PowerBI-compatible output.
+
+Generates CSVs that are:
+- Schema-compliant (exact columns, exact order - Invariants 1-4)
+- Deterministic (same DB → same bytes - Adjustment 3)
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pandas as pd
+
+from ..persistence.models import CSV_SCHEMAS, SORT_KEYS
+
+if TYPE_CHECKING:
+    from ..persistence.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+
+class CSVGenerationError(Exception):
+    """CSV generation failed."""
+
+
+class CSVGenerator:
+    """Generates PowerBI-compatible CSV files from SQLite.
+
+    Invariant 1: CSV schema is a hard contract.
+    Invariant 3: CSV output must be deterministic.
+    """
+
+    def __init__(self, db: DatabaseManager, output_dir: Path) -> None:
+        """Initialize the CSV generator.
+
+        Args:
+            db: Database manager instance.
+            output_dir: Directory for CSV output files.
+        """
+        self.db = db
+        self.output_dir = output_dir
+
+    def generate_all(self) -> dict[str, int]:
+        """Generate all CSV files.
+
+        Returns:
+            Dict mapping table names to row counts.
+
+        Raises:
+            CSVGenerationError: If generation fails.
+        """
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        results: dict[str, int] = {}
+
+        for table_name, columns in CSV_SCHEMAS.items():
+            try:
+                count = self._generate_table(table_name, columns)
+                results[table_name] = count
+                logger.info(f"Generated {table_name}.csv: {count} rows")
+            except Exception as e:
+                raise CSVGenerationError(
+                    f"Failed to generate {table_name}.csv: {e}"
+                ) from e
+
+        return results
+
+    def _generate_table(self, table_name: str, columns: list[str]) -> int:
+        """Generate a single CSV file.
+
+        Args:
+            table_name: Name of the table/CSV.
+            columns: Expected column order (contract).
+
+        Returns:
+            Number of rows written.
+        """
+        # Query the table
+        column_list = ", ".join(columns)
+        df = pd.read_sql_query(
+            f"SELECT {column_list} FROM {table_name}",  # noqa: S608
+            self.db.connection,
+        )
+
+        # Ensure column order matches contract exactly (Invariant 1)
+        df = df[columns]
+
+        # Deterministic row ordering (Adjustment 3)
+        sort_keys = SORT_KEYS.get(table_name, columns[:1])
+        df = df.sort_values(by=sort_keys, ascending=True)
+
+        # Write CSV with deterministic settings
+        output_path = self.output_dir / f"{table_name}.csv"
+        df.to_csv(
+            output_path,
+            index=False,
+            encoding="utf-8",
+            lineterminator="\n",  # Unix line endings for consistency
+            date_format="%Y-%m-%dT%H:%M:%S",  # Consistent datetime format
+        )
+
+        return len(df)
+
+    def validate_schemas(self) -> bool:
+        """Validate that generated CSVs match expected schemas.
+
+        Returns:
+            True if all schemas valid.
+
+        Raises:
+            CSVGenerationError: If any schema mismatch.
+        """
+        for table_name, expected_columns in CSV_SCHEMAS.items():
+            csv_path = self.output_dir / f"{table_name}.csv"
+
+            if not csv_path.exists():
+                raise CSVGenerationError(f"Missing CSV: {csv_path}")
+
+            df = pd.read_csv(csv_path, nrows=0)  # Just read headers
+            actual_columns = list(df.columns)
+
+            if actual_columns != expected_columns:
+                raise CSVGenerationError(
+                    f"Schema mismatch in {table_name}.csv:\n"
+                    f"  Expected: {expected_columns}\n"
+                    f"  Actual:   {actual_columns}"
+                )
+
+        logger.info("All CSV schemas validated successfully")
+        return True

ado_git_repo_insights/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities module for shared helper functions."""