ado-git-repo-insights 1.2.1__py3-none-any.whl

ado_git_repo_insights/__init__.py

@@ -0,0 +1,3 @@
+"""ado-git-repo-insights: Azure DevOps PR metrics extraction and CSV generation."""
+
+__version__ = "0.0.0"  # Managed by setuptools_scm

ado_git_repo_insights/cli.py

@@ -0,0 +1,354 @@
+"""CLI entry point for ado-git-repo-insights."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+from datetime import date
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from .config import ConfigurationError, load_config
+from .extractor.ado_client import ADOClient, ExtractionError
+from .extractor.pr_extractor import PRExtractor
+from .persistence.database import DatabaseError, DatabaseManager
+from .transform.csv_generator import CSVGenerationError, CSVGenerator
+from .utils.logging_config import LoggingConfig, setup_logging
+from .utils.run_summary import (
+    RunCounts,
+    RunSummary,
+    RunTimings,
+    create_minimal_summary,
+    get_git_sha,
+    get_tool_version,
+)
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+logger = logging.getLogger(__name__)
+
+
+def create_parser() -> argparse.ArgumentParser:  # pragma: no cover
+    """Create the argument parser for the CLI."""
+    parser = argparse.ArgumentParser(
+        prog="ado-insights",
+        description="Extract Azure DevOps PR metrics and generate PowerBI-compatible CSVs.",
+    )
+
+    # Global options
+    parser.add_argument(
+        "--log-format",
+        type=str,
+        choices=["console", "jsonl"],
+        default="console",
+        help="Log format: console (human-readable) or jsonl (structured)",
+    )
+    parser.add_argument(
+        "--artifacts-dir",
+        type=Path,
+        default=Path("run_artifacts"),
+        help="Directory for run artifacts (summary, logs)",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Extract command
+    extract_parser = subparsers.add_parser(
+        "extract",
+        help="Extract PR data from Azure DevOps",
+    )
+    extract_parser.add_argument(
+        "--organization",
+        type=str,
+        help="Azure DevOps organization name",
+    )
+    extract_parser.add_argument(
+        "--projects",
+        type=str,
+        help="Comma-separated list of project names",
+    )
+    extract_parser.add_argument(
+        "--pat",
+        type=str,
+        required=True,
+        help="Personal Access Token with Code (Read) scope",
+    )
+    extract_parser.add_argument(
+        "--config",
+        type=Path,
+        help="Path to config.yaml file",
+    )
+    extract_parser.add_argument(
+        "--database",
+        type=Path,
+        default=Path("ado-insights.sqlite"),
+        help="Path to SQLite database file",
+    )
+    extract_parser.add_argument(
+        "--start-date",
+        type=str,
+        help="Override start date (YYYY-MM-DD)",
+    )
+    extract_parser.add_argument(
+        "--end-date",
+        type=str,
+        help="Override end date (YYYY-MM-DD)",
+    )
+    extract_parser.add_argument(
+        "--backfill-days",
+        type=int,
+        help="Number of days to backfill for convergence",
+    )
+
+    # Generate CSV command
+    csv_parser = subparsers.add_parser(
+        "generate-csv",
+        help="Generate CSV files from SQLite database",
+    )
+    csv_parser.add_argument(
+        "--database",
+        type=Path,
+        required=True,
+        help="Path to SQLite database file",
+    )
+    csv_parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path("csv_output"),
+        help="Output directory for CSV files",
+    )
+
+    return parser
+
+
+def cmd_extract(args: Namespace) -> int:
+    """Execute the extract command."""
+    start_time = time.perf_counter()
+    timing = RunTimings()
+    counts = RunCounts()
+    warnings_list: list[str] = []
+    per_project_status: dict[str, str] = {}
+    first_fatal_error: str | None = None
+
+    try:
+        # Load and validate configuration
+        config = load_config(
+            config_path=args.config,
+            organization=args.organization,
+            projects=args.projects,
+            pat=args.pat,
+            database=args.database,
+            start_date=args.start_date,
+            end_date=args.end_date,
+            backfill_days=args.backfill_days,
+        )
+        config.log_summary()
+
+        # Connect to database
+        extract_start = time.perf_counter()
+        db = DatabaseManager(config.database)
+        db.connect()
+
+        try:
+            # Create ADO client
+            client = ADOClient(
+                organization=config.organization,
+                pat=config.pat,  # Invariant 19: PAT handled securely
+                config=config.api,
+            )
+
+            # Test connection
+            client.test_connection(config.projects[0])
+
+            # Run extraction
+            extractor = PRExtractor(client, db, config)
+            summary = extractor.extract_all(backfill_days=args.backfill_days)
+
+            # Collect timing
+            timing.extract_seconds = time.perf_counter() - extract_start
+
+            # Collect counts and warnings
+            counts.prs_fetched = summary.total_prs
+            if hasattr(summary, "warnings"):
+                warnings_list.extend(summary.warnings)
+
+            # Collect per-project status
+            for project_result in summary.projects:
+                status = "success" if project_result.success else "failed"
+                per_project_status[project_result.project] = status
+
+                # Capture first fatal error
+                if not project_result.success and first_fatal_error is None:
+                    first_fatal_error = (
+                        project_result.error
+                        or f"Extraction failed for project: {project_result.project}"
+                    )
+
+            # Fail-fast: any project failure = exit 1
+            if not summary.success:
+                logger.error("Extraction failed")
+                timing.total_seconds = time.perf_counter() - start_time
+
+                # Write failure summary
+                run_summary = RunSummary(
+                    tool_version=get_tool_version(),
+                    git_sha=get_git_sha(),
+                    organization=config.organization,
+                    projects=config.projects,
+                    date_range_start=str(config.date_range.start or date.today()),
+                    date_range_end=str(config.date_range.end or date.today()),
+                    counts=counts,
+                    timings=timing,
+                    warnings=warnings_list,
+                    final_status="failed",
+                    per_project_status=per_project_status,
+                    first_fatal_error=first_fatal_error,
+                )
+                run_summary.write(args.artifacts_dir / "run_summary.json")
+                run_summary.print_final_line()
+                run_summary.emit_ado_commands()
+                return 1
+
+            logger.info(f"Extraction complete: {summary.total_prs} PRs")
+            timing.total_seconds = time.perf_counter() - start_time
+
+            # Write success summary
+            run_summary = RunSummary(
+                tool_version=get_tool_version(),
+                git_sha=get_git_sha(),
+                organization=config.organization,
+                projects=config.projects,
+                date_range_start=str(config.date_range.start or date.today()),
+                date_range_end=str(config.date_range.end or date.today()),
+                counts=counts,
+                timings=timing,
+                warnings=warnings_list,
+                final_status="success",
+                per_project_status=per_project_status,
+                first_fatal_error=None,
+            )
+            run_summary.write(args.artifacts_dir / "run_summary.json")
+            run_summary.print_final_line()
+            run_summary.emit_ado_commands()
+            return 0
+
+        finally:
+            db.close()
+
+    except ConfigurationError as e:
+        logger.error(f"Configuration error: {e}")
+        # P2 Fix: Write minimal summary for caught errors
+        minimal_summary = create_minimal_summary(
+            f"Configuration error: {e}", args.artifacts_dir
+        )
+        minimal_summary.write(args.artifacts_dir / "run_summary.json")
+        return 1
+    except DatabaseError as e:
+        logger.error(f"Database error: {e}")
+        # P2 Fix: Write minimal summary for caught errors
+        minimal_summary = create_minimal_summary(
+            f"Database error: {e}", args.artifacts_dir
+        )
+        minimal_summary.write(args.artifacts_dir / "run_summary.json")
+        return 1
+    except ExtractionError as e:
+        logger.error(f"Extraction error: {e}")
+        # P2 Fix: Write minimal summary for caught errors
+        minimal_summary = create_minimal_summary(
+            f"Extraction error: {e}", args.artifacts_dir
+        )
+        minimal_summary.write(args.artifacts_dir / "run_summary.json")
+        return 1
+
+
+def cmd_generate_csv(args: Namespace) -> int:
+    """Execute the generate-csv command."""
+    logger.info("Generating CSV files...")
+    logger.info(f"Database: {args.database}")
+    logger.info(f"Output: {args.output}")
+
+    if not args.database.exists():
+        logger.error(f"Database not found: {args.database}")
+        return 1
+
+    try:
+        db = DatabaseManager(args.database)
+        db.connect()
+
+        try:
+            generator = CSVGenerator(db, args.output)
+            results = generator.generate_all()
+
+            # Validate schemas (Invariant 1)
+            generator.validate_schemas()
+
+            logger.info("CSV generation complete:")
+            for table, count in results.items():
+                logger.info(f"  {table}: {count} rows")
+
+            return 0
+
+        finally:
+            db.close()
+
+    except DatabaseError as e:
+        logger.error(f"Database error: {e}")
+        return 1
+    except CSVGenerationError as e:
+        logger.error(f"CSV generation error: {e}")
+        return 1
+
+
+def main() -> int:
+    """Main entry point for the CLI."""
+    parser = create_parser()
+    args = parser.parse_args()
+
+    # Setup logging early
+    log_config = LoggingConfig(
+        format=getattr(args, "log_format", "console"),
+        artifacts_dir=getattr(args, "artifacts_dir", Path("run_artifacts")),
+    )
+    setup_logging(log_config)
+
+    # Ensure artifacts directory exists
+    artifacts_dir = getattr(args, "artifacts_dir", Path("run_artifacts"))
+    artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+    summary_path = artifacts_dir / "run_summary.json"
+
+    try:
+        if args.command == "extract":
+            return cmd_extract(args)
+        elif args.command == "generate-csv":
+            return cmd_generate_csv(args)
+        else:
+            parser.print_help()
+            return 1
+    except KeyboardInterrupt:
+        logger.info("Operation cancelled by user")
+
+        # Write minimal failure summary if success summary doesn't exist
+        if not summary_path.exists():
+            minimal_summary = create_minimal_summary(
+                "Operation cancelled by user", artifacts_dir
+            )
+            minimal_summary.write(summary_path)
+
+        return 130
+    except Exception as e:
+        logger.exception(f"Unexpected error: {e}")
+
+        # Write minimal failure summary if success summary doesn't exist
+        if not summary_path.exists():
+            minimal_summary = create_minimal_summary(str(e), artifacts_dir)
+            minimal_summary.write(summary_path)
+
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ado_git_repo_insights/config.py

@@ -0,0 +1,186 @@
+"""Configuration loader for ado-git-repo-insights.
+
+Loads and validates configuration from YAML files or CLI arguments.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigurationError(Exception):
+    """Configuration validation error."""
+
+
+@dataclass
+class APIConfig:
+    """API configuration settings."""
+
+    base_url: str = "https://dev.azure.com"
+    version: str = "7.1-preview.1"
+    rate_limit_sleep_seconds: float = 0.5
+    max_retries: int = 3
+    retry_delay_seconds: float = 5.0
+    retry_backoff_multiplier: float = 2.0
+
+
+@dataclass
+class BackfillConfig:
+    """Backfill configuration settings (Adjustment 1)."""
+
+    enabled: bool = True
+    window_days: int = 60  # Default: 60 days (configurable 30-90)
+
+
+@dataclass
+class DateRangeConfig:
+    """Optional date range override."""
+
+    start: date | None = None
+    end: date | None = None
+
+
+@dataclass
+class Config:
+    """Main configuration for ado-git-repo-insights."""
+
+    organization: str
+    projects: list[str]
+    pat: str  # Will be masked in logs
+    database: Path = field(default_factory=lambda: Path("ado-insights.sqlite"))
+    api: APIConfig = field(default_factory=APIConfig)
+    backfill: BackfillConfig = field(default_factory=BackfillConfig)
+    date_range: DateRangeConfig = field(default_factory=DateRangeConfig)
+
+    def __post_init__(self) -> None:
+        """Validate configuration after initialization."""
+        if not self.organization:
+            raise ConfigurationError("organization is required")
+        if not self.projects:
+            raise ConfigurationError("At least one project is required")
+        if not self.pat:
+            raise ConfigurationError("PAT is required")
+
+    def __repr__(self) -> str:
+        """Repr with masked PAT (Invariant 19: Never expose secrets)."""
+        return (
+            f"Config(organization={self.organization!r}, "
+            f"projects={self.projects!r}, "
+            f"pat='********', "  # Masked
+            f"database={self.database!r}, "
+            f"api={self.api!r}, "
+            f"backfill={self.backfill!r}, "
+            f"date_range={self.date_range!r})"
+        )
+
+    def log_summary(self) -> None:
+        """Log configuration summary (with PAT masked)."""
+        logger.info(f"Organization: {self.organization}")
+        logger.info(f"Projects: {', '.join(self.projects)}")
+        logger.info(f"Database: {self.database}")
+        logger.info(f"PAT: {'*' * 8}...{'*' * 4}")  # Invariant 19: Never log PAT
+        if self.date_range.start or self.date_range.end:
+            logger.info(f"Date range: {self.date_range.start} → {self.date_range.end}")
+        if self.backfill.enabled:
+            logger.info(f"Backfill: {self.backfill.window_days} days")
+
+
+def load_config(
+    config_path: Path | None = None,
+    organization: str | None = None,
+    projects: str | None = None,
+    pat: str | None = None,
+    database: Path | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    backfill_days: int | None = None,
+) -> Config:
+    """Load configuration from file and/or CLI arguments.
+
+    CLI arguments override file values.
+
+    Args:
+        config_path: Path to config.yaml file.
+        organization: Organization name (CLI override).
+        projects: Comma-separated project names (CLI override).
+        pat: Personal Access Token (CLI override).
+        database: Database path (CLI override).
+        start_date: Start date YYYY-MM-DD (CLI override).
+        end_date: End date YYYY-MM-DD (CLI override).
+        backfill_days: Backfill window in days (CLI override).
+
+    Returns:
+        Validated Config instance.
+
+    Raises:
+        ConfigurationError: If configuration is invalid.
+    """
+    # Start with defaults
+    config_data: dict[str, Any] = {}
+
+    # Load from file if provided
+    if config_path and config_path.exists():
+        logger.info(f"Loading configuration from {config_path}")
+        with config_path.open() as f:
+            config_data = yaml.safe_load(f) or {}
+
+    # Apply CLI overrides
+    if organization:
+        config_data["organization"] = organization
+    if projects:
+        config_data["projects"] = [p.strip() for p in projects.split(",")]
+    if pat:
+        config_data["pat"] = pat
+    elif not config_data.get("pat"):
+        # Try environment variable
+        config_data["pat"] = os.environ.get("ADO_PAT", "")
+
+    # Build API config
+    api_data = config_data.get("api", {})
+    api_config = APIConfig(
+        base_url=api_data.get("base_url", "https://dev.azure.com"),
+        version=api_data.get("version", "7.1-preview.1"),
+        rate_limit_sleep_seconds=api_data.get("rate_limit_sleep_seconds", 0.5),
+        max_retries=api_data.get("max_retries", 3),
+        retry_delay_seconds=api_data.get("retry_delay_seconds", 5.0),
+        retry_backoff_multiplier=api_data.get("retry_backoff_multiplier", 2.0),
+    )
+
+    # Build backfill config
+    backfill_data = config_data.get("backfill", {})
+    backfill_config = BackfillConfig(
+        enabled=backfill_data.get("enabled", True),
+        window_days=backfill_days or backfill_data.get("window_days", 60),
+    )
+
+    # Build date range config
+    date_range = DateRangeConfig()
+    if start_date:
+        date_range.start = date.fromisoformat(start_date)
+    elif config_data.get("date_range", {}).get("start"):
+        date_range.start = date.fromisoformat(config_data["date_range"]["start"])
+
+    if end_date:
+        date_range.end = date.fromisoformat(end_date)
+    elif config_data.get("date_range", {}).get("end"):
+        date_range.end = date.fromisoformat(config_data["date_range"]["end"])
+
+    # Build main config
+    return Config(
+        organization=config_data.get("organization", ""),
+        projects=config_data.get("projects", []),
+        pat=config_data.get("pat", ""),
+        database=database or Path(config_data.get("database", "ado-insights.sqlite")),
+        api=api_config,
+        backfill=backfill_config,
+        date_range=date_range,
+    )

ado_git_repo_insights/extractor/__init__.py

@@ -0,0 +1 @@
+"""Extractor module for Azure DevOps API interactions."""