ci-time-tracker 0.2.0__py3-none-any.whl

ci_time_tracker/__init__.py

@@ -0,0 +1,3 @@
+"""ci-time-tracker: A CLI tool for analyzing CI/CD pipeline configurations and build logs."""
+
+__version__ = "0.2.0"

ci_time_tracker/__main__.py

@@ -0,0 +1,12 @@
+"""Module entry point for ci-time-tracker.
+
+Enables execution via: python -m ci_time_tracker
+"""
+
+import sys
+
+from ci_time_tracker.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())
+

ci_time_tracker/analyzer.py

@@ -0,0 +1,444 @@
+"""Analyzer module for ci-time-tracker.
+
+This module provides functionality to analyze CI/CD pipeline configurations
+and build logs, computing statistics, detecting duration regressions, and identifying
+flaky steps.
+"""
+
+import re
+from collections import Counter, defaultdict
+from statistics import median
+from typing import Any
+
+from ci_time_tracker.detection import (
+    DEFAULT_MIN_SAMPLES,
+    DEFAULT_REGRESSION_THRESHOLD,
+    detect_flaky_by_commit,
+    detect_regression,
+)
+from ci_time_tracker.models import (
+    AnalysisResult,
+    BuildLog,
+    PipelineConfig,
+    PipelineStep,
+    StepStatistics,
+)
+from ci_time_tracker.pricing import PricingTable, attributed_cost, job_cost
+
+
+def compute_percentiles(durations: list[float]) -> dict[str, float | None]:
+    """Compute p50, p90, p95, p99 percentiles for a list of durations.
+    
+    Uses the nearest-rank method for percentile calculation, where pN is
+    the smallest value such that at least N% of the data is less than or
+    equal to that value.
+    
+    Args:
+        durations: List of duration values in seconds. Must be non-empty.
+        
+    Returns:
+        Dictionary with keys 'p50', 'p90', 'p95', 'p99' and their computed values.
+        Returns None values if durations list is empty.
+    """
+    if not durations:
+        return {"p50": None, "p90": None, "p95": None, "p99": None}
+    
+    sorted_durations = sorted(durations)
+    n = len(sorted_durations)
+    
+    def percentile(p: float) -> float:
+        """Calculate percentile using nearest-rank method."""
+        if n == 1:
+            return sorted_durations[0]
+        # Calculate rank (1-indexed position)
+        rank = (p / 100.0) * n
+        # Use ceiling to get the index (convert to 0-indexed)
+        index = int(rank)
+        if index >= n:
+            index = n - 1
+        elif index < 1:
+            index = 0
+        else:
+            index = index - 1 if rank == int(rank) else index
+        # Clamp to valid range
+        index = max(0, min(index, n - 1))
+        return sorted_durations[index]
+    
+    return {
+        "p50": percentile(50),
+        "p90": percentile(90),
+        "p95": percentile(95),
+        "p99": percentile(99),
+    }
+
+
+def analyze_config(
+    config: PipelineConfig, 
+    estimates: dict[str, float] | None = None
+) -> AnalysisResult:
+    """Analyze pipeline configuration and return summary.
+    
+    Processes a parsed pipeline configuration to create an analysis result
+    containing all steps with their estimated durations (if provided).
+    
+    Args:
+        config: Parsed pipeline configuration
+        estimates: Optional dictionary mapping step names to estimated
+                   durations in seconds
+                   
+    Returns:
+        AnalysisResult in config mode with steps and total estimated duration
+    """
+    estimates = estimates or {}
+    
+    # Associate estimates with steps
+    steps_with_estimates: list[PipelineStep] = []
+    total_estimated = 0.0
+    has_estimates = False
+    
+    for step in config.steps:
+        # Look up estimate by step name
+        estimated_duration = estimates.get(step.name)
+        
+        # Create new step with estimate if found
+        step_with_estimate = PipelineStep(
+            name=step.name,
+            stage=step.stage,
+            command=step.command,
+            estimated_duration=estimated_duration,
+        )
+        steps_with_estimates.append(step_with_estimate)
+        
+        if estimated_duration is not None:
+            total_estimated += estimated_duration
+            has_estimates = True
+    
+    return AnalysisResult(
+        mode="config",
+        steps=steps_with_estimates,
+        total_estimated_duration=total_estimated if has_estimates else None,
+        regressed_steps=[],
+        flaky_steps=[],
+        metadata={
+            "provider": config.provider,
+            "stages": config.stages,
+            "step_count": len(steps_with_estimates),
+            **config.metadata,
+        },
+    )
+
+
+# Matrix parameters as GitHub appends them to a job name: "build (3.12, windows-latest)"
+_MATRIX_PARAMS_PATTERN = re.compile(r'\s*\([^()]*\)')
+
+
+def normalize_matrix_name(name: str) -> str:
+    """Strip matrix parameters from a step's job prefix.
+    
+    GitHub names every leg of a matrix separately, so "Run tests" becomes
+    "build (3.12, windows-latest) / Run tests", one step per leg. Grouping
+    the legs answers how much a step costs across the whole matrix, at the
+    price of mixing runners with different rates.
+    
+    Only the job prefix is stripped, so parentheses in a step's own name
+    are left alone.
+    """
+    job, separator, step = name.partition(" / ")
+    if not separator:
+        return _MATRIX_PARAMS_PATTERN.sub("", name).strip()
+    
+    return f"{_MATRIX_PARAMS_PATTERN.sub('', job).strip()}{separator}{step}"
+
+
+def _chronological(logs: list[BuildLog]) -> list[BuildLog]:
+    """Order builds oldest first when every build has a comparable timestamp.
+
+    Otherwise the given order is kept: log files and API results already come
+    in a meaningful order, and aware and naive timestamps cannot be compared.
+    """
+    timestamps = [log.timestamp for log in logs]
+
+    if any(timestamp is None for timestamp in timestamps):
+        return list(logs)
+    if len({timestamp.tzinfo is None for timestamp in timestamps}) > 1:
+        return list(logs)
+
+    return sorted(logs, key=lambda log: log.timestamp)
+
+
+def analyze_logs(
+    logs: list[BuildLog],
+    pricing: PricingTable | None = None,
+    group_matrix: bool = False,
+    regression_threshold: float = DEFAULT_REGRESSION_THRESHOLD,
+    min_samples: int = DEFAULT_MIN_SAMPLES,
+) -> AnalysisResult:
+    """Analyze multiple build logs and compute statistics.
+
+    Aggregates step execution data across builds to compute duration
+    percentiles and consumed time, detect duration regressions, identify
+    flaky steps, and estimate cost where the runner's rate is known.
+
+    Flaky detection uses same-commit evidence whenever builds carry a commit
+    SHA and falls back to the failure-rate heuristic otherwise; the method
+    used is recorded in the metadata.
+
+    Args:
+        logs: List of parsed build logs
+        pricing: Optional runner pricing table; defaults to the built-in rates
+        group_matrix: Aggregate the legs of a matrix job into one step
+        regression_threshold: Minimum relative duration increase to report,
+            e.g. 0.25 for +25%
+        min_samples: Minimum builds on each side of a regression
+
+    Returns:
+        AnalysisResult in log mode with step statistics and detected issues
+    """
+    pricing = pricing or PricingTable()
+
+    if not logs:
+        return AnalysisResult(
+            mode="log",
+            steps=[],
+            total_estimated_duration=None,
+            regressed_steps=[],
+            flaky_steps=[],
+            metadata={"build_count": 0},
+        )
+
+    logs = _chronological(logs)
+
+    def reported_name(name: str) -> str:
+        return normalize_matrix_name(name) if group_matrix else name
+
+    # Aggregate step data by name
+    step_data: dict[str, dict] = {}
+
+    for build_index, log in enumerate(logs):
+        for step in log.steps:
+            name = reported_name(step.name)
+
+            if name not in step_data:
+                step_data[name] = {
+                    "durations": [],
+                    "build_durations": defaultdict(list),
+                    "success_count": 0,
+                    "failure_count": 0,
+                    "execution_count": 0,
+                    "retry_count": 0,
+                    "runners": Counter(),
+                    "cost": 0.0,
+                    "priced_executions": 0,
+                }
+
+            data = step_data[name]
+            data["execution_count"] += 1
+
+            if step.duration_seconds is not None:
+                data["durations"].append(step.duration_seconds)
+                data["build_durations"][build_index].append(step.duration_seconds)
+
+            if step.status == "success":
+                data["success_count"] += 1
+            elif step.status == "failure":
+                data["failure_count"] += 1
+
+            if step.is_retry:
+                data["retry_count"] += 1
+
+            if step.runner:
+                data["runners"][step.runner] += 1
+
+            cost = attributed_cost(step.duration_seconds, pricing.price_for(step.runner))
+            if cost is not None:
+                data["cost"] += cost
+                data["priced_executions"] += 1
+
+    # Build step statistics
+    step_stats: list[StepStatistics] = []
+
+    for name, data in step_data.items():
+        percentiles = compute_percentiles(data["durations"])
+
+        execution_count = data["execution_count"]
+        failure_count = data["failure_count"]
+        failure_rate = failure_count / execution_count if execution_count > 0 else 0.0
+
+        durations = data["durations"]
+        total_duration = sum(durations)
+        most_common_runner = data["runners"].most_common(1)
+
+        stats = StepStatistics(
+            name=name,
+            execution_count=execution_count,
+            success_count=data["success_count"],
+            failure_count=failure_count,
+            durations=durations,
+            p50=percentiles["p50"],
+            p90=percentiles["p90"],
+            p95=percentiles["p95"],
+            p99=percentiles["p99"],
+            failure_rate=failure_rate,
+            total_duration=total_duration,
+            mean_duration=total_duration / len(durations) if durations else None,
+            runner=most_common_runner[0][0] if most_common_runner else None,
+            estimated_cost_usd=data["cost"] if data["priced_executions"] else None,
+        )
+        step_stats.append(stats)
+
+    # Flaky steps: same-commit evidence when builds carry a commit, otherwise
+    # the failure-rate heuristic, which cannot tell flakiness from breakage
+    if any(log.head_sha for log in logs):
+        flaky_method = "same-commit"
+        flaky_evidence = detect_flaky_by_commit(logs, reported_name)
+    else:
+        flaky_method = "failure-rate"
+        flaky_evidence = {
+            name: {"method": "failure-rate"} for name in detect_flaky_steps(step_stats)
+        }
+
+    # Regressions, from one sample per build (the median across matrix legs
+    # when they are grouped), in chronological order
+    regression_checked = 0
+
+    for stats in step_stats:
+        build_durations = step_data[stats.name]["build_durations"]
+        samples = [
+            (median(values), logs[build_index])
+            for build_index, values in sorted(build_durations.items())
+        ]
+
+        if len(samples) >= 2 * min_samples:
+            regression_checked += 1
+
+        stats.regression = detect_regression(samples, regression_threshold, min_samples)
+        stats.is_regression = stats.regression is not None
+        stats.flaky_evidence = flaky_evidence.get(stats.name)
+        stats.is_flaky = stats.flaky_evidence is not None
+
+    metadata: dict[str, Any] = {
+        "build_count": len(logs),
+        "step_count": len(step_stats),
+        "total_execution_seconds": sum(stats.total_duration for stats in step_stats),
+    }
+    
+    # How long a build takes end to end; superseded attempts are left out so
+    # that a re-run build is not counted twice
+    build_durations = [
+        log.total_duration
+        for log in logs
+        if log.total_duration is not None and not log.metadata.get("superseded_attempt")
+    ]
+    if build_durations:
+        build_percentiles = compute_percentiles(build_durations)
+        metadata["build_p50_seconds"] = build_percentiles["p50"]
+        metadata["build_p90_seconds"] = build_percentiles["p90"]
+    
+    if group_matrix:
+        metadata["grouping"] = "matrix legs merged"
+
+    superseded = sum(1 for log in logs if log.metadata.get("superseded_attempt"))
+    if superseded:
+        metadata["superseded_attempts"] = superseded
+
+    metadata["flaky_detection"] = flaky_method
+    metadata["regression_checked_steps"] = regression_checked
+    metadata["regression_min_builds"] = 2 * min_samples
+    metadata.update(aggregate_job_metrics(logs, pricing))
+
+    return AnalysisResult(
+        mode="log",
+        steps=step_stats,
+        total_estimated_duration=None,
+        regressed_steps=[stats.name for stats in step_stats if stats.is_regression],
+        flaky_steps=[stats.name for stats in step_stats if stats.is_flaky],
+        metadata=metadata,
+    )
+
+
+def aggregate_job_metrics(
+    logs: list[BuildLog],
+    pricing: PricingTable | None = None,
+) -> dict[str, Any]:
+    """Aggregate job-level queue time and billed cost across builds.
+    
+    Only providers that report jobs (the API ingestion) carry this data, so
+    text logs yield an empty dict. Cost uses GitHub's per-job rounding, which
+    makes it slightly higher than the sum of the steps' attributed costs.
+    
+    Args:
+        logs: List of build logs whose metadata may carry a "jobs" list
+        pricing: Optional runner pricing table
+        
+    Returns:
+        Metadata entries for queue time and cost, empty when no jobs are known
+    """
+    pricing = pricing or PricingTable()
+    
+    queue_times: list[float] = []
+    billed_cost = 0.0
+    priced_jobs = 0
+    unpriced_runners: set[str] = set()
+    job_count = 0
+    
+    for log in logs:
+        for job in log.metadata.get("jobs") or []:
+            job_count += 1
+            
+            queued = job.get("queued_seconds")
+            if queued is not None and queued >= 0:
+                queue_times.append(float(queued))
+            
+            price = pricing.price_for(job.get("runner"))
+            if price is None:
+                if job.get("runner"):
+                    unpriced_runners.add(str(job["runner"]))
+                continue
+            
+            cost = job_cost(job.get("duration_seconds"), price)
+            if cost is not None:
+                billed_cost += cost
+                priced_jobs += 1
+    
+    if not job_count:
+        return {}
+    
+    metrics: dict[str, Any] = {"job_count": job_count}
+    
+    if queue_times:
+        percentiles = compute_percentiles(queue_times)
+        metrics["total_queue_seconds"] = sum(queue_times)
+        metrics["queue_p50_seconds"] = percentiles["p50"]
+        metrics["queue_p90_seconds"] = percentiles["p90"]
+    
+    if priced_jobs:
+        metrics["estimated_cost_usd"] = round(billed_cost, 4)
+        metrics["priced_jobs"] = priced_jobs
+        metrics["pricing_source"] = pricing.source
+    
+    if unpriced_runners:
+        metrics["unpriced_runners"] = sorted(unpriced_runners)
+    
+    return metrics
+
+
+def detect_flaky_steps(stats: list[StepStatistics]) -> list[str]:
+    """Identify steps with failure rate between 5% and 95%.
+    
+    A step is flagged as flaky if its failure rate is strictly between
+    5% and 95% (exclusive), indicating intermittent failures.
+    
+    Args:
+        stats: List of step statistics with failure counts
+        
+    Returns:
+        List of step names that are flagged as flaky
+    """
+    flaky_steps: list[str] = []
+    
+    for step in stats:
+        # Flaky if failure rate is strictly between 5% and 95%
+        if 0.05 < step.failure_rate < 0.95:
+            flaky_steps.append(step.name)
+    
+    return flaky_steps

ci_time_tracker/cache.py

@@ -0,0 +1,113 @@
+"""Local cache for CI provider API responses.
+
+Listing workflow runs is cheap, but every run costs an extra request for its
+jobs. Those payloads no longer change once a run has completed, so they are
+stored in a small SQLite database between invocations.
+"""
+
+import json
+import os
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS run_jobs (
+    repo TEXT NOT NULL,
+    run_id INTEGER NOT NULL,
+    run_attempt INTEGER NOT NULL,
+    fetched_at TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    PRIMARY KEY (repo, run_id, run_attempt)
+);
+"""
+
+
+def default_cache_path() -> Path:
+    """Return the cache database location for this platform.
+
+    Overridable with the CI_TIME_TRACKER_CACHE environment variable.
+    """
+    override = os.environ.get("CI_TIME_TRACKER_CACHE")
+    if override:
+        return Path(override)
+
+    if os.name == "nt":
+        base = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local")
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache")
+
+    return base / "ci-time-tracker" / "runs.db"
+
+
+class RunCache:
+    """Stores the jobs payload of completed runs, keyed by run and attempt."""
+
+    def __init__(self, path: str | Path | None = None):
+        self.path = Path(path) if path is not None else default_cache_path()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._connection = sqlite3.connect(str(self.path))
+        self._connection.executescript(_SCHEMA)
+        self._connection.commit()
+
+    def get_jobs(
+        self,
+        repo: str,
+        run_id: int | str,
+        run_attempt: int = 1,
+    ) -> list[dict[str, Any]] | None:
+        """Return the cached jobs of a run, or None when not cached."""
+        row = self._connection.execute(
+            "SELECT payload FROM run_jobs WHERE repo = ? AND run_id = ? AND run_attempt = ?",
+            (repo, int(run_id), int(run_attempt)),
+        ).fetchone()
+
+        if row is None:
+            return None
+
+        try:
+            return json.loads(row[0])
+        except json.JSONDecodeError:
+            # A corrupted row should not break the run; refetch instead.
+            return None
+
+    def store_jobs(
+        self,
+        repo: str,
+        run_id: int | str,
+        run_attempt: int,
+        jobs: list[dict[str, Any]],
+    ) -> None:
+        """Cache the jobs payload of a completed run."""
+        self._connection.execute(
+            "INSERT OR REPLACE INTO run_jobs "
+            "(repo, run_id, run_attempt, fetched_at, payload) VALUES (?, ?, ?, ?, ?)",
+            (
+                repo,
+                int(run_id),
+                int(run_attempt),
+                datetime.now(timezone.utc).isoformat(),
+                json.dumps(jobs),
+            ),
+        )
+        self._connection.commit()
+
+    def count(self, repo: str | None = None) -> int:
+        """Number of cached runs, optionally limited to one repository."""
+        if repo is None:
+            query, params = "SELECT COUNT(*) FROM run_jobs", ()
+        else:
+            query, params = "SELECT COUNT(*) FROM run_jobs WHERE repo = ?", (repo,)
+
+        return int(self._connection.execute(query, params).fetchone()[0])
+
+    def close(self) -> None:
+        """Close the underlying database connection."""
+        self._connection.close()
+
+    def __enter__(self) -> "RunCache":
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()