airbyte-internal-ops 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

airbyte_ops_mcp/regression_tests/ci_output.py

@@ -171,7 +171,7 @@ def _get_github_artifacts_url() -> str | None:
     return f"{run_url}#artifacts"
 
 
-def generate_regression_report(
+def generate_action_test_comparison_report(
     target_image: str,
     control_image: str,
     command: str,
@@ -179,16 +179,17 @@ def generate_regression_report(
     control_result: dict[str, Any],
     output_dir: Path,
 ) -> Path:
-    """Generate a markdown regression test report modeled on legacy HTML reports.
+    """Generate a markdown comparison report for a single action (command).
 
     This creates a comprehensive report with context, message counts comparison,
-    and record counts per stream (for read commands). The structure mirrors the
-    legacy connector_live_tests HTML report.
+    and record counts per stream (for read commands). The report starts with an
+    L2 header containing the command name, making it easy to consolidate multiple
+    command reports into a single document.
 
     Args:
         target_image: The target (new version) connector image.
         control_image: The control (baseline version) connector image.
-        command: The Airbyte command that was run.
+        command: The Airbyte command that was run (e.g., "spec", "check", "discover", "read").
         target_result: Results dict from running target connector.
         control_result: Results dict from running control connector.
         output_dir: Directory to write the report to.
@@ -204,51 +205,22 @@ def generate_regression_report(
     target_record_counts = target_result.get("record_counts_per_stream", {})
     control_record_counts = control_result.get("record_counts_per_stream", {})
 
-    run_id = os.getenv("GITHUB_RUN_ID", "")
-    artifact_name = (
-        f"regression-test-artifacts-{run_id}" if run_id else "regression-test-artifacts"
-    )
-
+    # Extract version tags for the summary table
     target_version = (
         target_image.rsplit(":", 1)[-1] if ":" in target_image else "unknown"
     )
     control_version = (
         control_image.rsplit(":", 1)[-1] if ":" in control_image else "unknown"
     )
-    connector_name = (
-        target_image.rsplit(":", 1)[0] if ":" in target_image else target_image
-    )
-
-    run_url = _get_github_run_url()
-    artifacts_url = _get_github_artifacts_url()
 
+    # Start with L2 header containing the command name (no L1 header)
+    # This allows multiple command reports to be concatenated into a single document
+    # Note: Context block (connector, versions, workflow links) is added at the workflow level
     lines: list[str] = [
-        "# Regression Test Report",
-        "",
-        "## Context",
+        f"## `{command.upper()}` Test Results",
         "",
-        f"- **Test Date:** {datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
-        f"- **Connector:** `{connector_name}`",
-        f"- **Control Version:** `{control_version}`",
-        f"- **Target Version:** `{target_version}`",
-        f"- **Command:** `{command.upper()}`",
     ]
 
-    if run_url:
-        lines.append(f"- **Workflow Run:** [View Execution]({run_url})")
-    if artifacts_url:
-        lines.append(f"- **Artifacts:** [Download `{artifact_name}`]({artifacts_url})")
-    else:
-        lines.append(f"- **Artifacts:** `{artifact_name}`")
-
-    lines.extend(
-        [
-            "",
-            "## Summary",
-            "",
-        ]
-    )
-
     if regression_detected:
         if target_result["success"] and not control_result["success"]:
             lines.append("**Result:** Target succeeded, control failed (improvement)")
@@ -261,20 +233,24 @@ def generate_regression_report(
     else:
         lines.append("**Result:** Both versions failed")
 
+    # Use emojis for better scanability
+    control_emoji = "✅" if control_result["success"] else "❌"
+    target_emoji = "✅" if target_result["success"] else "❌"
+
     lines.extend(
         [
             "",
-            "| Version | Exit Code | Success |",
-            "|---------|-----------|---------|",
-            f"| Control ({control_version}) | {control_result['exit_code']} | {control_result['success']} |",
-            f"| Target ({target_version}) | {target_result['exit_code']} | {target_result['success']} |",
+            "| Version | Exit Code | Result |",
+            "|---------|-----------|--------|",
+            f"| Control (`{control_version}`) | {control_result['exit_code']} | {control_emoji} |",
+            f"| Target (`{target_version}`) | {target_result['exit_code']} | {target_emoji} |",
             "",
         ]
     )
 
     lines.extend(
         [
-            "## Command Execution Metrics",
+            "### Command Execution Metrics",
             "",
         ]
     )
@@ -282,7 +258,7 @@ def generate_regression_report(
     if target_counts or control_counts:
         lines.extend(
             [
-                "### Message Types",
+                "#### Message Types",
                 "",
                 "| Type | Control | Target | Delta |",
                 "|------|---------|--------|-------|",
@@ -294,14 +270,14 @@ def generate_regression_report(
             target_count = target_counts.get(msg_type, 0)
             delta = target_count - control_count
             lines.append(
-                f"| {msg_type} | {control_count} | {target_count} | {_format_delta(delta)} |"
+                f"| `{msg_type}` | {control_count} | {target_count} | {_format_delta(delta)} |"
             )
         lines.append("")
 
     if target_record_counts or control_record_counts:
         lines.extend(
             [
-                "### Record Count per Stream",
+                "#### Record Count per Stream",
                 "",
                 "| Stream | Control | Target | Delta |",
                 "|--------|---------|--------|-------|",
@@ -330,35 +306,144 @@ def generate_regression_report(
     if control_http or target_http:
         lines.extend(
             [
-                "### HTTP Metrics",
+                "#### HTTP Metrics",
                 "",
-                "| Version | Flow Count | Duplicate Flows |",
-                "|---------|------------|-----------------|",
-                f"| Control | {control_http.get('flow_count', 0)} | {control_http.get('duplicate_flow_count', 0)} |",
-                f"| Target | {target_http.get('flow_count', 0)} | {target_http.get('duplicate_flow_count', 0)} |",
+                "| Version | Flow Count | Duplicate Flows | Cache Hit Ratio |",
+                "|---------|------------|-----------------|-----------------|",
+                f"| Control | {control_http.get('flow_count', 0)} | {control_http.get('duplicate_flow_count', 0)} | {control_http.get('cache_hit_ratio', 'N/A')} |",
+                f"| Target | {target_http.get('flow_count', 0)} | {target_http.get('duplicate_flow_count', 0)} | {target_http.get('cache_hit_ratio', 'N/A')} |",
                 "",
             ]
         )
 
+    # Note: Execution Details section removed as redundant with Summary table
+
+    report_content = "\n".join(lines)
+    report_path = output_dir / "report.md"
+    report_path.write_text(report_content)
+
+    return report_path
+
+
+# Backwards-compatible alias for the old function name
+generate_regression_report = generate_action_test_comparison_report
+
+
+def generate_single_version_report(
+    connector_image: str,
+    command: str,
+    result: dict[str, Any],
+    output_dir: Path,
+) -> Path:
+    """Generate a markdown report for a single-version regression test.
+
+    This creates a report with message counts and record counts per stream for a single
+    connector run. The report starts with an L2 header containing the command name,
+    making it easy to consolidate multiple command reports.
+
+    Args:
+        connector_image: The connector image that was tested.
+        command: The Airbyte command that was run (e.g., "spec", "check", "discover", "read").
+        result: Results dict from running the connector.
+        output_dir: Directory to write the report to.
+
+    Returns:
+        Path to the generated report.md file.
+    """
+    message_counts = result.get("message_counts", {})
+    record_counts = result.get("record_counts_per_stream", {})
+
+    run_id = os.getenv("GITHUB_RUN_ID", "")
+    artifact_name = (
+        f"regression-test-artifacts-{command}-{run_id}"
+        if run_id
+        else f"regression-test-artifacts-{command}"
+    )
+
+    version = (
+        connector_image.rsplit(":", 1)[-1] if ":" in connector_image else "unknown"
+    )
+    connector_name = (
+        connector_image.rsplit(":", 1)[0] if ":" in connector_image else connector_image
+    )
+
+    run_url = _get_github_run_url()
+    artifacts_url = _get_github_artifacts_url()
+
+    # Get tester identity from environment (GitHub Actions sets GITHUB_ACTOR)
+    tester = os.getenv("GITHUB_ACTOR") or os.getenv("USER") or "unknown"
+
+    # Start with L2 header containing the command name (no L1 header)
+    lines: list[str] = [
+        f"## `{command.upper()}` Test Results",
+        "",
+        "### Context",
+        "",
+        f"- **Test Date:** {datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
+        f"- **Tester:** `{tester}`",
+        f"- **Connector:** `{connector_name}`",
+        f"- **Version:** `{version}`",
+        f"- **Command:** `{command.upper()}`",
+    ]
+
+    if run_url:
+        lines.append(f"- **Workflow Run:** [View Execution]({run_url})")
+    if artifacts_url:
+        lines.append(f"- **Artifacts:** [Download `{artifact_name}`]({artifacts_url})")
+    else:
+        lines.append(f"- **Artifacts:** `{artifact_name}`")
+
     lines.extend(
         [
-            "## Execution Details",
             "",
-            "### Control",
+            "### Summary",
+            "",
+            f"**Result:** {'PASS' if result['success'] else 'FAIL'}",
             "",
-            f"- **Image:** `{control_image}`",
-            f"- **Exit Code:** {control_result['exit_code']}",
-            f"- **Success:** {control_result['success']}",
-            f"- **Stdout:** `{control_result.get('stdout_file', 'N/A')}`",
-            f"- **Stderr:** `{control_result.get('stderr_file', 'N/A')}`",
+            f"- **Exit Code:** {result['exit_code']}",
+            f"- **Success:** {result['success']}",
             "",
-            "### Target",
+        ]
+    )
+
+    if message_counts:
+        lines.extend(
+            [
+                "### Message Types",
+                "",
+                "| Type | Count |",
+                "|------|-------|",
+            ]
+        )
+        for msg_type in sorted(message_counts.keys()):
+            count = message_counts[msg_type]
+            lines.append(f"| `{msg_type}` | {count} |")
+        lines.append("")
+
+    if record_counts:
+        lines.extend(
+            [
+                "### Record Count per Stream",
+                "",
+                "| Stream | Count |",
+                "|--------|-------|",
+            ]
+        )
+        total = 0
+        for stream in sorted(record_counts.keys()):
+            count = record_counts[stream]
+            total += count
+            lines.append(f"| {stream} | {count} |")
+        lines.append(f"| **Total** | **{total}** |")
+        lines.append("")
+
+    lines.extend(
+        [
+            "### Execution Details",
             "",
-            f"- **Image:** `{target_image}`",
-            f"- **Exit Code:** {target_result['exit_code']}",
-            f"- **Success:** {target_result['success']}",
-            f"- **Stdout:** `{target_result.get('stdout_file', 'N/A')}`",
-            f"- **Stderr:** `{target_result.get('stderr_file', 'N/A')}`",
+            f"- **Image:** `{connector_image}`",
+            f"- **Stdout:** `{result.get('stdout_file', 'N/A')}`",
+            f"- **Stderr:** `{result.get('stderr_file', 'N/A')}`",
             "",
         ]
     )
@@ -373,9 +458,6 @@ def generate_regression_report(
 def get_report_summary(report_path: Path) -> str:
     """Get a brief summary pointing to the full report.
 
-    Args:
-        report_path: Path to the full report.md file.
-
     Returns:
         Brief markdown summary for GITHUB_STEP_SUMMARY.
     """
@@ -393,7 +475,5 @@ def get_report_summary(report_path: Path) -> str:
 
     return f"""## Regression Test Report
 
-Full report available in the **Regression Test Report** check or in artifact {artifact_link}.
-
-See the Checks tab for the complete report with message counts and execution details.
+Full report available in artifact {artifact_link}.
 """

airbyte_ops_mcp/regression_tests/http_metrics.py

@@ -47,11 +47,21 @@ class HttpMetrics:
     flow_count: int
     duplicate_flow_count: int
     unique_urls: list[str]
+    cache_hits_count: int = 0
+
+    @property
+    def cache_hit_ratio(self) -> str:
+        """Calculate cache hit ratio as a percentage string."""
+        if self.flow_count == 0:
+            return "N/A"
+        return f"{(self.cache_hits_count / self.flow_count) * 100:.2f}%"
 
     @classmethod
     def empty(cls) -> HttpMetrics:
         """Create empty metrics when HTTP capture is unavailable."""
-        return cls(flow_count=0, duplicate_flow_count=0, unique_urls=[])
+        return cls(
+            flow_count=0, duplicate_flow_count=0, unique_urls=[], cache_hits_count=0
+        )
 
 
 @dataclass
@@ -285,17 +295,22 @@ def parse_http_dump(dump_file_path: Path) -> HttpMetrics:
     unique_urls = list(set(all_urls))
     duplicate_count = len(all_urls) - len(unique_urls)
 
+    # Cache hits are interpreted as duplicate requests to the same URL
+    # (requests that could potentially be served from cache)
+    cache_hits = duplicate_count
+
     return HttpMetrics(
         flow_count=len(flows),
         duplicate_flow_count=duplicate_count,
         unique_urls=sorted(unique_urls),
+        cache_hits_count=cache_hits,
     )
 
 
 def compute_http_metrics_comparison(
     control_metrics: HttpMetrics,
     target_metrics: HttpMetrics,
-) -> dict[str, dict[str, int | str] | int]:
+) -> dict[str, dict[str, int | str] | int | str]:
     """Compute HTTP metrics comparison between control and target.
 
     This produces output in the same format as the legacy
@@ -312,10 +327,14 @@ def compute_http_metrics_comparison(
         "control": {
             "flow_count": control_metrics.flow_count,
             "duplicate_flow_count": control_metrics.duplicate_flow_count,
+            "cache_hits_count": control_metrics.cache_hits_count,
+            "cache_hit_ratio": control_metrics.cache_hit_ratio,
         },
         "target": {
             "flow_count": target_metrics.flow_count,
             "duplicate_flow_count": target_metrics.duplicate_flow_count,
+            "cache_hits_count": target_metrics.cache_hits_count,
+            "cache_hit_ratio": target_metrics.cache_hit_ratio,
         },
         "difference": target_metrics.flow_count - control_metrics.flow_count,
     }

airbyte_ops_mcp/regression_tests/models.py

@@ -256,4 +256,10 @@ class ExecutionResult:
             file_path = airbyte_messages_dir / f"{type_name}.jsonl"
             file_path.write_text("\n".join(messages))
 
+        # Save configured catalog (input) if available
+        if self.configured_catalog is not None:
+            catalog_path = output_dir / "configured_catalog.json"
+            catalog_path.write_text(self.configured_catalog.json(indent=2))
+            self.logger.info(f"Saved configured catalog to {catalog_path}")
+
         self.logger.info(f"Artifacts saved to {output_dir}")

airbyte_ops_mcp/telemetry.py

@@ -0,0 +1,162 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Telemetry module for tracking usage analytics.
+
+This module provides utilities for tracking usage of various Airbyte operations
+using Segment analytics. The tracking is optional and can be disabled via
+environment variables.
+
+Based on the legacy connector_live_tests/commons/segment_tracking.py implementation.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+try:
+    from segment import analytics  # type: ignore[import-untyped]
+
+    SEGMENT_AVAILABLE = True
+except ImportError:
+    analytics = None  # type: ignore[assignment]
+    SEGMENT_AVAILABLE = False
+
+logger = logging.getLogger(__name__)
+
+# Environment variable to disable tracking
+DISABLE_TRACKING_ENV_VAR = "AIRBYTE_DISABLE_TELEMETRY"
+# Legacy env var for backward compatibility
+LEGACY_DISABLE_TRACKING_ENV_VAR = "REGRESSION_TEST_DISABLE_TRACKING"
+# Environment variable to enable debug mode
+DEBUG_SEGMENT_ENV_VAR = "DEBUG_SEGMENT"
+
+# Segment write key environment variable name
+# The write key can be provided via environment variable or uses the default
+# public key for the Airbyte analytics project. Segment write keys are designed
+# to be embedded in client-side code for analytics tracking.
+SEGMENT_WRITE_KEY_ENV_VAR = "SEGMENT_WRITE_KEY"
+_DEFAULT_SEGMENT_WRITE_KEY = "hnWfMdEtXNKBjvmJ258F72wShsLmcsZ8"
+
+
+def _is_tracking_enabled() -> bool:
+    """Check if tracking is enabled based on environment variables."""
+    if os.getenv(DISABLE_TRACKING_ENV_VAR) is not None:
+        return False
+    return os.getenv(LEGACY_DISABLE_TRACKING_ENV_VAR) is None
+
+
+def _on_error(error: Exception, items: Any) -> None:
+    """Handle Segment tracking errors."""
+    logger.warning("An error occurred in Segment Tracking", exc_info=error)
+
+
+def _initialize_analytics() -> bool:
+    """Initialize Segment analytics if available and enabled.
+
+    Returns:
+        True if analytics was initialized successfully, False otherwise.
+    """
+    if not SEGMENT_AVAILABLE:
+        logger.debug("Segment analytics not available (package not installed)")
+        return False
+
+    if not _is_tracking_enabled():
+        logger.debug("Telemetry tracking is disabled via environment variable")
+        return False
+
+    # Use environment variable if set, otherwise use default public key
+    write_key = os.getenv(SEGMENT_WRITE_KEY_ENV_VAR, _DEFAULT_SEGMENT_WRITE_KEY)
+    analytics.write_key = write_key
+    analytics.send = True
+    analytics.debug = os.getenv(DEBUG_SEGMENT_ENV_VAR) is not None
+    analytics.on_error = _on_error
+    return True
+
+
+def track_regression_test(
+    user_id: str | None,
+    connector_image: str,
+    command: str,
+    target_version: str,
+    control_version: str | None = None,
+    additional_properties: dict[str, Any] | None = None,
+) -> None:
+    """Track a regression test execution.
+
+    Args:
+        user_id: The user ID to associate with the event. If None, uses "airbyte-ci".
+        connector_image: The connector image being tested.
+        command: The Airbyte command being run (spec, check, discover, read).
+        target_version: The target connector version being tested.
+        control_version: The control connector version (for comparison mode).
+        additional_properties: Additional properties to include in the event.
+    """
+    if not _initialize_analytics():
+        return
+
+    if not user_id:
+        user_id = "airbyte-ci"
+
+    analytics.identify(user_id)
+
+    properties: dict[str, Any] = {
+        "connector_image": connector_image,
+        "command": command,
+        "target_version": target_version,
+    }
+
+    if control_version:
+        properties["control_version"] = control_version
+        properties["test_mode"] = "comparison"
+    else:
+        properties["test_mode"] = "single_version"
+
+    if additional_properties:
+        properties.update(additional_properties)
+
+    try:
+        from importlib.metadata import version
+
+        properties["package_version"] = version("airbyte-ops-mcp")
+    except Exception:
+        properties["package_version"] = "unknown"
+
+    analytics.track(user_id, "regression_test_start", properties)
+    logger.debug(f"Tracked regression_test_start event for user {user_id}")
+
+
+def track_event(
+    user_id: str | None,
+    event_name: str,
+    properties: dict[str, Any] | None = None,
+) -> None:
+    """Track a generic event.
+
+    This is a general-purpose tracking function for events that don't fit
+    into the more specific tracking functions.
+
+    Args:
+        user_id: The user ID to associate with the event. If None, uses "airbyte-ci".
+        event_name: The name of the event to track.
+        properties: Properties to include in the event.
+    """
+    if not _initialize_analytics():
+        return
+
+    if not user_id:
+        user_id = "airbyte-ci"
+
+    analytics.identify(user_id)
+
+    event_properties = properties or {}
+
+    try:
+        from importlib.metadata import version
+
+        event_properties["package_version"] = version("airbyte-ops-mcp")
+    except Exception:
+        event_properties["package_version"] = "unknown"
+
+    analytics.track(user_id, event_name, event_properties)
+    logger.debug(f"Tracked {event_name} event for user {user_id}")

airbyte_ops_mcp/_legacy/airbyte_ci/connector_live_tests/.gitignore

@@ -1 +0,0 @@
-regression_tests_artifacts