airbyte-internal-ops 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: airbyte-internal-ops
3
- Version: 0.1.8
3
+ Version: 0.1.10
4
4
  Summary: MCP and API interfaces that let the agents do the admin work
5
5
  Author-email: Aaron Steers <aj@airbyte.io>
6
6
  Keywords: admin,airbyte,api,mcp
@@ -1,6 +1,6 @@
1
1
  airbyte_ops_mcp/__init__.py,sha256=HhzURuYr29_UIdMrnWYaZB8ENr_kFkBdm4uqeiIW3Vw,760
2
2
  airbyte_ops_mcp/_annotations.py,sha256=MO-SBDnbykxxHDESG7d8rviZZ4WlZgJKv0a8eBqcEzQ,1757
3
- airbyte_ops_mcp/constants.py,sha256=419-AlRfwbbxeEEV9lhmXhpTUjsSdzJpfcuL_MZZtXM,1982
3
+ airbyte_ops_mcp/constants.py,sha256=col6-5BUWuIYhbtKmlvSRR8URBoSNExoz94cn4_kujI,2333
4
4
  airbyte_ops_mcp/gcp_auth.py,sha256=5k-k145ZoYhHLjyDES8nrA8f8BBihRI0ykrdD1IcfOs,3599
5
5
  airbyte_ops_mcp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
6
  airbyte_ops_mcp/_legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -207,7 +207,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/checks/documentation/templates/s
207
207
  airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/checks/documentation/templates/template.md.j2,sha256=bYxLfOP3GDwQC9Q3PiVbxsL5GS0p5b10k1dwzfOrm1M,1962
208
208
  airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
209
209
  airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/templates/qa_checks.md.j2,sha256=v3NyYUAYkqRiYAFDQ2NgpgYK6PIDhDKrqfRgpX8454s,1946
210
- airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md,sha256=9uGj0cTaGawoD-uMOT8ImT19XXozWz6-r_aFRjZhN8Y,3567
210
+ airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md,sha256=CQtTgVzHH3UdgRCohA9GyNhxIiYxVZmSwhMKWnfbocE,3569
211
211
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/package-lock.json,sha256=uazyPocrveknTDfwu4DlcaDQ5HwkPNf1Q_vDNpGMp0s,2233
212
212
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/package.json,sha256=_JMByXIhM9iYRKlLqheCVWXWswfwDzuXc5-A1VRUdUM,307
213
213
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/transform.py,sha256=cx-u9y0aYKs0M7rL47I62WQfcx6uIyzbL2wWv5GERNM,1939
@@ -282,7 +282,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/constants.py,sha256=bSXsx-RQ
282
282
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/docker_hub.py,sha256=K-6-QjTiSEPeJUnVJIMyYfTtu6uFoh0vLX579b04LSs,5528
283
283
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/gcs_upload.py,sha256=_r1nh54U6QFglFiPWBpxrSylE6Guu0Gjk7Bq5NKK4EE,26506
284
284
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry.py,sha256=3okT639PCZGL0p6tlWJoTYy-xI2igeAj6DerxkMC1_Q,15130
285
- airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_entry.py,sha256=sVq0fgAnTIihROn9Iy5OHMswT-fF2JDTPsRo-bNF0yA,26417
285
+ airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_entry.py,sha256=QT-UopRiFGJpabvoya-B3F32qrlB2qluOiNKj56Pv4g,26489
286
286
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_report.py,sha256=dp13BxBiXG7DgxhzfupMm-sTFdjUVn9TyYrvo6KShU0,12600
287
287
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/sentry.py,sha256=e1O6_FHrHvqNNQdb5TkXs5Bhv7_hZ8fMiOWJr8H5tkI,1806
288
288
  airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/spec_cache.py,sha256=rcsJtu6B8_bmfXFv9U-IVjsVpPDJWLJYdH8e_XbT7yk,4482
@@ -351,12 +351,12 @@ airbyte_ops_mcp/cli/__init__.py,sha256=XpL7FyVfgabfBF2JR7u7NwJ2krlYqjd_OwLcWf-Xc
351
351
  airbyte_ops_mcp/cli/_base.py,sha256=I8tWnyQf0ks4r3J8N8h-5GZxyn37T-55KsbuHnxYlcg,415
352
352
  airbyte_ops_mcp/cli/_shared.py,sha256=jg-xMyGzTCGPqKd8VTfE_3kGPIyO_3Kx5sQbG4rPc0Y,1311
353
353
  airbyte_ops_mcp/cli/app.py,sha256=SEdBpqFUG2O8zGV5ifwptxrLGFph_dLr66-MX9d69gQ,789
354
- airbyte_ops_mcp/cli/cloud.py,sha256=BMFYs5bTEgdOhxwzBrtSyYMKaHhXnMM_SGzK2hFDPBY,32076
354
+ airbyte_ops_mcp/cli/cloud.py,sha256=semAuPqruWfe7JiVmtbELZgcpeqtoIn4LGD0kEvIS30,38981
355
355
  airbyte_ops_mcp/cli/gh.py,sha256=91b1AxFXvHQCFyXhrrym-756ZjnMCqvxFdmwCtma1zI,2046
356
- airbyte_ops_mcp/cli/registry.py,sha256=OpON1p4_A-G-FSfIpr6UlKYTjcj_zyiprKOu7qxwuhc,5787
356
+ airbyte_ops_mcp/cli/registry.py,sha256=-yiLJWSslV_qGi6ImXZYfXOJSE4oJBO7yICkyA_RiUo,5792
357
357
  airbyte_ops_mcp/cli/repo.py,sha256=G1hoQpH0XYhUH3FFOsia9xabGB0LP9o3XcwBuqvFVo0,16331
358
358
  airbyte_ops_mcp/cloud_admin/__init__.py,sha256=cqE96Q10Kp6elhH9DAi6TVsIwSUy3sooDLLrxTaktGk,816
359
- airbyte_ops_mcp/cloud_admin/api_client.py,sha256=4vZv1J4S2Q8ETl6gIB20X1X6KHTVV-bx__b2Ax8oqyc,17358
359
+ airbyte_ops_mcp/cloud_admin/api_client.py,sha256=6PovHDwOzo4fxSyk6viwvnXjCRIiC4uPZo0pGMx0Bdk,17359
360
360
  airbyte_ops_mcp/cloud_admin/auth.py,sha256=j45pRR8fg6CLwVdn7Uu5KW_kTz_CjRP6ZJGUzqHj_Dk,2558
361
361
  airbyte_ops_mcp/cloud_admin/connection_config.py,sha256=UtbIwuB7CA3WJr9oYRwlKDsjciqd_9ewWdml2f8DuXw,4887
362
362
  airbyte_ops_mcp/cloud_admin/models.py,sha256=YZ3FbEW-tZa50khKTTl4Bzvy_LsGyyQd6qcpXo62jls,2670
@@ -389,25 +389,25 @@ airbyte_ops_mcp/mcp/_mcp_utils.py,sha256=nhztHcoc-_ASPpJfoDBjxjjqEvQM6_QIrhp7F2U
389
389
  airbyte_ops_mcp/mcp/cloud_connector_versions.py,sha256=XxaS6WBP0sJPRwT7TTPhVH2PzhPqVWMNU5fVdWdxLLk,10361
390
390
  airbyte_ops_mcp/mcp/connector_analysis.py,sha256=OC4KrOSkMkKPkOisWnSv96BDDE5TQYHq-Jxa2vtjJpo,298
391
391
  airbyte_ops_mcp/mcp/connector_qa.py,sha256=aImpqdnqBPDrz10BS0owsV4kuIU2XdalzgbaGZsbOL0,258
392
- airbyte_ops_mcp/mcp/github.py,sha256=keB5_LUs6OULXr4Ukg-gzJfeDiC_QvBXSh56yTD9kSQ,7625
392
+ airbyte_ops_mcp/mcp/github.py,sha256=opmVWRwjNs_jeWejv8wHOtb-3J09hOlqxg98GCzmFLo,7627
393
393
  airbyte_ops_mcp/mcp/github_repo_ops.py,sha256=PiERpt8abo20Gz4CfXhrDNlVM4o4FOt5sweZJND2a0s,5314
394
394
  airbyte_ops_mcp/mcp/live_tests.py,sha256=eHfgNkEcY1OKzYiJmkxwOluLPiFHIdP4nm_H1H0MXDg,17940
395
395
  airbyte_ops_mcp/mcp/metadata.py,sha256=fwGW97WknR5lfKcQnFtK6dU87aA6TmLj1NkKyqDAV9g,270
396
- airbyte_ops_mcp/mcp/prerelease.py,sha256=RTEq9uguaREeZ9BqC2In3abdGuqg2IUZZ43KNoYj66Y,9444
397
- airbyte_ops_mcp/mcp/prod_db_queries.py,sha256=RV909zZX156WKXzwtu68Sd071tmaiJaCJz9wNHg3j0I,12399
396
+ airbyte_ops_mcp/mcp/prerelease.py,sha256=08G6ogRqEauQyDxFLirUVaYeU3exAJd-DJn_8fXCNXg,9450
397
+ airbyte_ops_mcp/mcp/prod_db_queries.py,sha256=RkBVISfkbwML3grWONxYsULRnFEYdqDZVBZIyo6W8xE,14311
398
398
  airbyte_ops_mcp/mcp/prompts.py,sha256=mJld9mdPECXYZffWXGSvNs4Xevx3rxqUGNlzGKVC2_s,1599
399
399
  airbyte_ops_mcp/mcp/registry.py,sha256=PW-VYUj42qx2pQ_apUkVaoUFq7VgB9zEU7-aGrkSCCw,290
400
400
  airbyte_ops_mcp/mcp/server.py,sha256=7zi91xioVTx1q-bEleekZH2c2JnbzDQt_6zxdEwnLbg,2958
401
401
  airbyte_ops_mcp/mcp/server_info.py,sha256=Yi4B1auW64QZGBDas5mro_vwTjvrP785TFNSBP7GhRg,2361
402
402
  airbyte_ops_mcp/prod_db_access/__init__.py,sha256=5pxouMPY1beyWlB0UwPnbaLTKTHqU6X82rbbgKY2vYU,1069
403
- airbyte_ops_mcp/prod_db_access/db_engine.py,sha256=ia1KcuQOXi3Qhy_MnxYmccCBJ4rAt_d4nVDjcyzje6o,4289
403
+ airbyte_ops_mcp/prod_db_access/db_engine.py,sha256=11xNZTk4I8SKYhsnmE7-LVrkJXN4dCRbBeD1_hj3f-s,9027
404
404
  airbyte_ops_mcp/prod_db_access/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
405
- airbyte_ops_mcp/prod_db_access/queries.py,sha256=jUGJErgDmaKf7mswgVEGin8bjsUFAxegSAOB47XzT9k,8724
406
- airbyte_ops_mcp/prod_db_access/sql.py,sha256=zHPucNuMlfxz3aU8vYo1ziiGk0lIncG9XmblEoRDd4c,12725
405
+ airbyte_ops_mcp/prod_db_access/queries.py,sha256=q7PcI15EGh6jFS9MVB_gZt1a56YvrZV5hnwa5lgU2q0,10844
406
+ airbyte_ops_mcp/prod_db_access/sql.py,sha256=tWQAwMk8DzG8HpLIYglljlReI2oeYulQPsV31ocUJSw,16251
407
407
  airbyte_ops_mcp/registry/__init__.py,sha256=iEaPlt9GrnlaLbc__98TguNeZG8wuQu7S-_2QkhHcbA,858
408
408
  airbyte_ops_mcp/registry/models.py,sha256=B4L4TKr52wo0xs0CqvCBrpowqjShzVnZ5eTr2-EyhNs,2346
409
409
  airbyte_ops_mcp/registry/publish.py,sha256=VoPxsM2_0zJ829orzCRN-kjgcJtuBNyXgW4I9J680ro,12717
410
- airbyte_internal_ops-0.1.8.dist-info/METADATA,sha256=BSNP1wsy5PbIFtyq6RbvCGPHgY3C8vC5UNsAzjqZAOk,5282
411
- airbyte_internal_ops-0.1.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
412
- airbyte_internal_ops-0.1.8.dist-info/entry_points.txt,sha256=eUgJ9xIy9PlR-CgRbqRMsh1NVp6jz08v9bul9vCYlU4,111
413
- airbyte_internal_ops-0.1.8.dist-info/RECORD,,
410
+ airbyte_internal_ops-0.1.10.dist-info/METADATA,sha256=I3WAz4JM1tgyGMcFaV5ElRwvwYjlRLR9kXKzifetd1Q,5283
411
+ airbyte_internal_ops-0.1.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
412
+ airbyte_internal_ops-0.1.10.dist-info/entry_points.txt,sha256=eUgJ9xIy9PlR-CgRbqRMsh1NVp6jz08v9bul9vCYlU4,111
413
+ airbyte_internal_ops-0.1.10.dist-info/RECORD,,
@@ -75,7 +75,7 @@ This will copy the specified connector version to your development bucket. This
75
75
  _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
76
76
 
77
77
  ```bash
78
- TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry run poe copy-connector-from-prod
78
+ TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe copy-connector-from-prod
79
79
  ```
80
80
 
81
81
  ### Promote Connector Version to Latest
@@ -87,5 +87,5 @@ _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage
87
87
  _⚠️ Warning: Its important to know that this will remove ANY existing files in the latest folder that are not in the versioned folder as it calls `gsutil rsync` with `-d` enabled._
88
88
 
89
89
  ```bash
90
- TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry run poe promote-connector-to-latest
90
+ TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe promote-connector-to-latest
91
91
  ```
@@ -526,7 +526,7 @@ def generate_and_persist_registry_entry(
526
526
  bucket_name (str): The name of the GCS bucket.
527
527
  repo_metadata_file_path (pathlib.Path): The path to the spec file.
528
528
  registry_type (str): The registry type.
529
- docker_image_tag (str): The docker image tag associated with this release. Typically a semver string (e.g. '1.2.3'), possibly with a suffix (e.g. '1.2.3-dev.abcde12345')
529
+ docker_image_tag (str): The docker image tag associated with this release. Typically a semver string (e.g. '1.2.3'), possibly with a suffix (e.g. '1.2.3-preview.abcde12')
530
530
  is_prerelease (bool): Whether this is a prerelease, or a main release.
531
531
  """
532
532
  # Read the repo metadata dict to bootstrap ourselves. We need the docker repository,
@@ -536,7 +536,7 @@ def generate_and_persist_registry_entry(
536
536
 
537
537
  try:
538
538
  # Now that we have the docker repo, read the appropriate versioned metadata from GCS.
539
- # This metadata will differ in a few fields (e.g. in prerelease mode, dockerImageTag will contain the actual prerelease tag `1.2.3-dev.abcde12345`),
539
+ # This metadata will differ in a few fields (e.g. in prerelease mode, dockerImageTag will contain the actual prerelease tag `1.2.3-preview.abcde12`),
540
540
  # so we'll treat this as the source of truth (ish. See below for how we handle the registryOverrides field.)
541
541
  gcs_client = get_gcs_storage_client(gcs_creds=os.environ.get("GCS_CREDENTIALS"))
542
542
  bucket = gcs_client.bucket(bucket_name)
@@ -645,6 +645,7 @@ def generate_and_persist_registry_entry(
645
645
  if (
646
646
  "-rc" not in metadata_dict["data"]["dockerImageTag"]
647
647
  and "-dev" not in metadata_dict["data"]["dockerImageTag"]
648
+ and "-preview" not in metadata_dict["data"]["dockerImageTag"]
648
649
  ) and not metadata_dict["data"]["registryOverrides"][registry_type]["enabled"]:
649
650
  logger.info(
650
651
  f"{registry_type} is not enabled: deleting existing {registry_type} registry entry for {metadata_dict['data']['dockerRepository']} at latest path."
@@ -13,6 +13,12 @@ Commands:
13
13
  from __future__ import annotations
14
14
 
15
15
  import json
16
+ import os
17
+ import shutil
18
+ import signal
19
+ import socket
20
+ import subprocess
21
+ import time
16
22
  from pathlib import Path
17
23
  from typing import Annotated, Literal
18
24
 
@@ -30,6 +36,12 @@ from airbyte_ops_mcp.cli._shared import (
30
36
  print_success,
31
37
  )
32
38
  from airbyte_ops_mcp.cloud_admin.connection_config import fetch_connection_config
39
+ from airbyte_ops_mcp.constants import (
40
+ CLOUD_SQL_INSTANCE,
41
+ CLOUD_SQL_PROXY_PID_FILE,
42
+ DEFAULT_CLOUD_SQL_PROXY_PORT,
43
+ ENV_GCP_PROD_DB_ACCESS_CREDENTIALS,
44
+ )
33
45
  from airbyte_ops_mcp.live_tests.ci_output import (
34
46
  generate_regression_report,
35
47
  get_report_summary,
@@ -75,6 +87,184 @@ connector_app = App(
75
87
  )
76
88
  cloud_app.command(connector_app)
77
89
 
90
+ # Create the db sub-app under cloud
91
+ db_app = App(name="db", help="Database operations for Airbyte Cloud Prod DB Replica.")
92
+ cloud_app.command(db_app)
93
+
94
+
95
+ @db_app.command(name="start-proxy")
96
+ def start_proxy(
97
+ port: Annotated[
98
+ int,
99
+ Parameter(help="Port for the Cloud SQL Proxy to listen on."),
100
+ ] = DEFAULT_CLOUD_SQL_PROXY_PORT,
101
+ daemon: Annotated[
102
+ bool,
103
+ Parameter(
104
+ help="Run as daemon in background (default). Use --no-daemon for foreground."
105
+ ),
106
+ ] = True,
107
+ ) -> None:
108
+ """Start the Cloud SQL Proxy for database access.
109
+
110
+ This command starts the Cloud SQL Auth Proxy to enable connections to the
111
+ Airbyte Cloud Prod DB Replica. The proxy is required for database query tools.
112
+
113
+ By default, runs as a daemon (background process). Use --no-daemon to run in
114
+ foreground mode where you can see logs and stop with Ctrl+C.
115
+
116
+ Credentials are read from the GCP_PROD_DB_ACCESS_CREDENTIALS environment variable,
117
+ which should contain the service account JSON credentials.
118
+
119
+ After starting the proxy, set these environment variables to use database tools:
120
+ export USE_CLOUD_SQL_PROXY=1
121
+ export DB_PORT={port}
122
+
123
+ Example:
124
+ airbyte-ops cloud db start-proxy
125
+ airbyte-ops cloud db start-proxy --port 15432
126
+ airbyte-ops cloud db start-proxy --no-daemon
127
+ """
128
+ # Check if proxy is already running on the requested port (idempotency)
129
+ try:
130
+ with socket.create_connection(("127.0.0.1", port), timeout=0.5):
131
+ # Something is already listening on this port
132
+ pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
133
+ pid_info = ""
134
+ if pid_file.exists():
135
+ pid_info = f" (PID: {pid_file.read_text().strip()})"
136
+ print_success(
137
+ f"Cloud SQL Proxy is already running on port {port}{pid_info}"
138
+ )
139
+ print_success("")
140
+ print_success("To use database tools, set these environment variables:")
141
+ print_success(" export USE_CLOUD_SQL_PROXY=1")
142
+ print_success(f" export DB_PORT={port}")
143
+ return
144
+ except (OSError, TimeoutError, ConnectionRefusedError):
145
+ pass # Port not in use, proceed with starting proxy
146
+
147
+ # Check if cloud-sql-proxy is installed
148
+ proxy_path = shutil.which("cloud-sql-proxy")
149
+ if not proxy_path:
150
+ exit_with_error(
151
+ "cloud-sql-proxy not found in PATH. "
152
+ "Install it from: https://cloud.google.com/sql/docs/mysql/sql-proxy"
153
+ )
154
+
155
+ # Get credentials from environment
156
+ creds_json = os.getenv(ENV_GCP_PROD_DB_ACCESS_CREDENTIALS)
157
+ if not creds_json:
158
+ exit_with_error(
159
+ f"{ENV_GCP_PROD_DB_ACCESS_CREDENTIALS} environment variable is not set. "
160
+ "This should contain the GCP service account JSON credentials."
161
+ )
162
+
163
+ # Build the command using --json-credentials to avoid writing to disk
164
+ cmd = [
165
+ proxy_path,
166
+ CLOUD_SQL_INSTANCE,
167
+ f"--port={port}",
168
+ f"--json-credentials={creds_json}",
169
+ ]
170
+
171
+ print_success(f"Starting Cloud SQL Proxy on port {port}...")
172
+ print_success(f"Instance: {CLOUD_SQL_INSTANCE}")
173
+ print_success("")
174
+ print_success("To use database tools, set these environment variables:")
175
+ print_success(" export USE_CLOUD_SQL_PROXY=1")
176
+ print_success(f" export DB_PORT={port}")
177
+ print_success("")
178
+
179
+ if daemon:
180
+ # Run in background (daemon mode) with log file for diagnostics
181
+ log_file_path = Path("/tmp/airbyte-cloud-sql-proxy.log")
182
+ log_file = log_file_path.open("ab")
183
+ process = subprocess.Popen(
184
+ cmd,
185
+ stdout=subprocess.DEVNULL,
186
+ stderr=log_file,
187
+ start_new_session=True,
188
+ )
189
+
190
+ # Brief wait to verify the process started successfully
191
+ time.sleep(0.5)
192
+ if process.poll() is not None:
193
+ # Process exited immediately - read any error output
194
+ log_file.close()
195
+ error_output = ""
196
+ if log_file_path.exists():
197
+ error_output = log_file_path.read_text()[-1000:] # Last 1000 chars
198
+ exit_with_error(
199
+ f"Cloud SQL Proxy failed to start (exit code: {process.returncode}).\n"
200
+ f"Check logs at {log_file_path}\n"
201
+ f"Recent output: {error_output}"
202
+ )
203
+
204
+ # Write PID to file for stop-proxy command
205
+ pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
206
+ pid_file.write_text(str(process.pid))
207
+ print_success(f"Cloud SQL Proxy started as daemon (PID: {process.pid})")
208
+ print_success(f"Logs: {log_file_path}")
209
+ print_success("To stop: airbyte-ops cloud db stop-proxy")
210
+ else:
211
+ # Run in foreground - replace current process
212
+ # Signals (Ctrl+C) will be handled directly by the cloud-sql-proxy process
213
+ print_success("Running in foreground. Press Ctrl+C to stop the proxy.")
214
+ print_success("")
215
+ os.execv(proxy_path, cmd)
216
+
217
+
218
+ @db_app.command(name="stop-proxy")
219
+ def stop_proxy() -> None:
220
+ """Stop the Cloud SQL Proxy daemon.
221
+
222
+ This command stops a Cloud SQL Proxy that was started with 'start-proxy'.
223
+ It reads the PID from the PID file and sends a SIGTERM signal to stop the process.
224
+
225
+ Example:
226
+ airbyte-ops cloud db stop-proxy
227
+ """
228
+ pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
229
+
230
+ if not pid_file.exists():
231
+ exit_with_error(
232
+ f"PID file not found at {CLOUD_SQL_PROXY_PID_FILE}. "
233
+ "No Cloud SQL Proxy daemon appears to be running."
234
+ )
235
+
236
+ pid_str = pid_file.read_text().strip()
237
+ if not pid_str.isdigit():
238
+ pid_file.unlink()
239
+ exit_with_error(f"Invalid PID in {CLOUD_SQL_PROXY_PID_FILE}: {pid_str}")
240
+
241
+ pid = int(pid_str)
242
+
243
+ # Check if process is still running
244
+ try:
245
+ os.kill(pid, 0) # Signal 0 just checks if process exists
246
+ except ProcessLookupError:
247
+ pid_file.unlink()
248
+ print_success(
249
+ f"Cloud SQL Proxy (PID: {pid}) is not running. Cleaned up PID file."
250
+ )
251
+ return
252
+ except PermissionError:
253
+ exit_with_error(f"Permission denied to check process {pid}.")
254
+
255
+ # Send SIGTERM to stop the process
256
+ try:
257
+ os.kill(pid, signal.SIGTERM)
258
+ print_success(f"Sent SIGTERM to Cloud SQL Proxy (PID: {pid}).")
259
+ except ProcessLookupError:
260
+ print_success(f"Cloud SQL Proxy (PID: {pid}) already stopped.")
261
+ except PermissionError:
262
+ exit_with_error(f"Permission denied to stop process {pid}.")
263
+
264
+ # Clean up PID file
265
+ pid_file.unlink(missing_ok=True)
266
+ print_success("Cloud SQL Proxy stopped.")
267
+
78
268
 
79
269
  @connector_app.command(name="get-version-info")
80
270
  def get_version_info(
@@ -117,7 +307,7 @@ def set_version_override(
117
307
  version: Annotated[
118
308
  str,
119
309
  Parameter(
120
- help="The semver version string to pin to (e.g., '2.1.5-dev.abc1234567')."
310
+ help="The semver version string to pin to (e.g., '2.1.5-preview.abc1234')."
121
311
  ),
122
312
  ],
123
313
  reason: Annotated[
@@ -63,7 +63,7 @@ def publish_prerelease(
63
63
  """Publish a connector prerelease to the Airbyte registry.
64
64
 
65
65
  Triggers the publish-connectors-prerelease workflow in the airbytehq/airbyte
66
- repository. Pre-release versions are tagged with format: {version}-dev.{git-sha}
66
+ repository. Pre-release versions are tagged with format: {version}-preview.{git-sha}
67
67
 
68
68
  Requires GITHUB_CONNECTOR_PUBLISHING_PAT or GITHUB_TOKEN environment variable
69
69
  with 'actions:write' permission.
@@ -162,7 +162,7 @@ def inspect_image(
162
162
  ],
163
163
  tag: Annotated[
164
164
  str,
165
- Parameter(help="Image tag (e.g., '2.1.5-dev.abc1234567')."),
165
+ Parameter(help="Image tag (e.g., '2.1.5-preview.abc1234')."),
166
166
  ],
167
167
  ) -> None:
168
168
  """Check if a Docker image exists on DockerHub.
@@ -160,7 +160,7 @@ def resolve_connector_version_id(
160
160
  Args:
161
161
  actor_definition_id: The actor definition ID
162
162
  connector_type: Either "source" or "destination"
163
- version: The version string (e.g., "0.1.47-dev.abe7cb4ddb")
163
+ version: The version string (e.g., "0.1.47-preview.abe7cb4")
164
164
  api_root: The API root URL
165
165
  client_id: The Airbyte Cloud client ID
166
166
  client_secret: The Airbyte Cloud client secret
@@ -27,6 +27,15 @@ EXPECTED_ADMIN_EMAIL_DOMAIN = "@airbyte.io"
27
27
  GCP_PROJECT_NAME = "prod-ab-cloud-proj"
28
28
  """The GCP project name for Airbyte Cloud production."""
29
29
 
30
+ CLOUD_SQL_INSTANCE = "prod-ab-cloud-proj:us-west3:prod-pgsql-replica"
31
+ """The Cloud SQL instance connection name for the Prod DB Replica."""
32
+
33
+ DEFAULT_CLOUD_SQL_PROXY_PORT = 15432
34
+ """Default port for Cloud SQL Proxy connections."""
35
+
36
+ CLOUD_SQL_PROXY_PID_FILE = "/tmp/airbyte-cloud-sql-proxy.pid"
37
+ """PID file for tracking the Cloud SQL Proxy process."""
38
+
30
39
  CLOUD_REGISTRY_URL = (
31
40
  "https://connectors.airbyte.com/files/registries/v0/cloud_registry.json"
32
41
  )
@@ -204,7 +204,7 @@ def _check_dockerhub_image(
204
204
 
205
205
  Args:
206
206
  image: Docker image name (e.g., "airbyte/source-github")
207
- tag: Image tag (e.g., "2.1.5-dev.abc1234567")
207
+ tag: Image tag (e.g., "2.1.5-preview.abc1234")
208
208
 
209
209
  Returns:
210
210
  Tag data dictionary if found, None if not found.
@@ -232,7 +232,7 @@ def get_docker_image_info(
232
232
  ],
233
233
  tag: Annotated[
234
234
  str,
235
- Field(description="Image tag (e.g., '2.1.5-dev.abc1234567')"),
235
+ Field(description="Image tag (e.g., '2.1.5-preview.abc1234')"),
236
236
  ],
237
237
  ) -> DockerImageInfo:
238
238
  """Check if a Docker image exists on DockerHub.
@@ -111,7 +111,7 @@ def _get_pr_head_info(
111
111
  return PRHeadInfo(
112
112
  ref=pr_data["head"]["ref"],
113
113
  sha=sha,
114
- short_sha=sha[:10],
114
+ short_sha=sha[:7],
115
115
  )
116
116
 
117
117
 
@@ -235,7 +235,7 @@ def publish_connector_to_airbyte_registry(
235
235
  publish-connectors-prerelease workflow in the airbytehq/airbyte repository,
236
236
  which publishes a pre-release version of the specified connector from the PR branch.
237
237
 
238
- Pre-release versions are tagged with the format: {version}-dev.{10-char-git-sha}
238
+ Pre-release versions are tagged with the format: {version}-preview.{7-char-git-sha}
239
239
  These versions are available for version pinning via the scoped_configuration API.
240
240
 
241
241
  Requires GITHUB_CONNECTOR_PUBLISHING_PAT or GITHUB_TOKEN environment variable
@@ -290,7 +290,7 @@ def publish_connector_to_airbyte_registry(
290
290
  docker_image = data.get("dockerRepository")
291
291
  base_version = data.get("dockerImageTag")
292
292
  if base_version:
293
- docker_image_tag = f"{base_version}-dev.{head_info.short_sha}"
293
+ docker_image_tag = f"{base_version}-preview.{head_info.short_sha}"
294
294
 
295
295
  return PrereleaseWorkflowResult(
296
296
  success=True,
@@ -20,6 +20,7 @@ from airbyte_ops_mcp.prod_db_access.queries import (
20
20
  query_connections_by_connector,
21
21
  query_connector_versions,
22
22
  query_dataplanes_list,
23
+ query_failed_sync_attempts_for_version,
23
24
  query_new_connector_releases,
24
25
  query_sync_results_for_version,
25
26
  query_workspace_info,
@@ -245,6 +246,52 @@ def query_prod_connector_version_sync_results(
245
246
  )
246
247
 
247
248
 
249
+ @mcp_tool(
250
+ read_only=True,
251
+ idempotent=True,
252
+ )
253
+ def query_prod_failed_sync_attempts_for_version(
254
+ connector_version_id: Annotated[
255
+ str,
256
+ Field(description="Connector version UUID to find failed sync attempts for"),
257
+ ],
258
+ days: Annotated[
259
+ int,
260
+ Field(description="Number of days to look back (default: 7)", default=7),
261
+ ] = 7,
262
+ limit: Annotated[
263
+ int,
264
+ Field(description="Maximum number of results (default: 100)", default=100),
265
+ ] = 100,
266
+ ) -> list[dict[str, Any]]:
267
+ """List failed sync attempts with failure details for actors pinned to a connector version.
268
+
269
+ Returns failed attempt records for connections using actors pinned to the specified
270
+ version. Includes failure_summary from the attempts table for debugging.
271
+
272
+ Key fields:
273
+ - latest_job_attempt_status: Final job status after all retries ('succeeded' means
274
+ the job eventually succeeded despite this failed attempt)
275
+ - failed_attempt_number: Which attempt this was (0-indexed)
276
+ - failure_summary: JSON containing failure details including failureType and messages
277
+
278
+ Note: May return multiple rows per job (one per failed attempt). Results ordered by
279
+ job_updated_at DESC, then failed_attempt_number DESC.
280
+
281
+ Returns list of dicts with keys: job_id, connection_id, latest_job_attempt_status,
282
+ job_started_at, job_updated_at, connection_name, actor_id, actor_name,
283
+ actor_definition_id, pin_origin_type, pin_origin, workspace_id, workspace_name,
284
+ organization_id, dataplane_group_id, dataplane_name, failed_attempt_id,
285
+ failed_attempt_number, failed_attempt_status, failed_attempt_created_at,
286
+ failed_attempt_ended_at, failure_summary, processing_task_queue
287
+ """
288
+ return query_failed_sync_attempts_for_version(
289
+ connector_version_id,
290
+ days=days,
291
+ limit=limit,
292
+ )
293
+
294
+
248
295
  @mcp_tool(
249
296
  read_only=True,
250
297
  idempotent=True,
@@ -11,7 +11,9 @@ from __future__ import annotations
11
11
 
12
12
  import json
13
13
  import os
14
- import traceback
14
+ import shutil
15
+ import socket
16
+ import subprocess
15
17
  from typing import Any, Callable
16
18
 
17
19
  import sqlalchemy
@@ -21,9 +23,127 @@ from google.cloud.sql.connector.enums import IPTypes
21
23
 
22
24
  from airbyte_ops_mcp.constants import (
23
25
  CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID,
26
+ DEFAULT_CLOUD_SQL_PROXY_PORT,
24
27
  )
25
28
 
26
29
  PG_DRIVER = "pg8000"
30
+ PROXY_CHECK_TIMEOUT = 0.5 # seconds
31
+ DIRECT_CONNECTION_TIMEOUT = 5 # seconds - timeout for direct VPC/Tailscale connections
32
+
33
+
34
+ class CloudSqlProxyNotRunningError(Exception):
35
+ """Raised when proxy mode is enabled but the Cloud SQL Proxy is not running."""
36
+
37
+ pass
38
+
39
+
40
+ class VpnNotConnectedError(Exception):
41
+ """Raised when direct connection mode requires VPN but it's not connected."""
42
+
43
+ pass
44
+
45
+
46
+ def _is_tailscale_connected() -> bool:
47
+ """Check if Tailscale VPN is likely connected.
48
+
49
+ This is a best-effort check that works on Linux and macOS.
50
+ Returns True if Tailscale appears to be connected, False otherwise.
51
+
52
+ Detection methods:
53
+ 1. Check for tailscale0 network interface (Linux)
54
+ 2. Run 'tailscale status --json' and check backend state (cross-platform)
55
+ """
56
+ # Method 1: Check for tailscale0 interface (Linux)
57
+ try:
58
+ interfaces = [name for _, name in socket.if_nameindex()]
59
+ if "tailscale0" in interfaces:
60
+ return True
61
+ except (OSError, AttributeError):
62
+ pass # if_nameindex not available on this platform
63
+
64
+ # Method 2: Check tailscale CLI status
65
+ tailscale_path = shutil.which("tailscale")
66
+ if tailscale_path:
67
+ try:
68
+ result = subprocess.run(
69
+ [tailscale_path, "status", "--json"],
70
+ capture_output=True,
71
+ text=True,
72
+ timeout=2,
73
+ )
74
+ if result.returncode == 0:
75
+ import json as json_module
76
+
77
+ status = json_module.loads(result.stdout)
78
+ # BackendState "Running" indicates connected
79
+ return status.get("BackendState") == "Running"
80
+ except (subprocess.TimeoutExpired, subprocess.SubprocessError, ValueError):
81
+ pass
82
+
83
+ return False
84
+
85
+
86
+ def _check_vpn_or_proxy_available() -> None:
87
+ """Check if either VPN or proxy is available for database access.
88
+
89
+ This function checks if the environment is properly configured for
90
+ database access. It fails fast with a helpful error message if neither
91
+ Tailscale VPN nor the Cloud SQL Proxy appears to be available.
92
+
93
+ Raises:
94
+ VpnNotConnectedError: If no VPN or proxy is detected
95
+ """
96
+ # If proxy mode is explicitly enabled, don't check VPN
97
+ if os.getenv("CI") or os.getenv("USE_CLOUD_SQL_PROXY"):
98
+ return
99
+
100
+ # Check if Tailscale is connected
101
+ if _is_tailscale_connected():
102
+ return
103
+
104
+ # Neither proxy mode nor Tailscale detected
105
+ raise VpnNotConnectedError(
106
+ "No VPN or proxy detected for database access.\n\n"
107
+ "To connect to the Airbyte Cloud Prod DB Replica, you need either:\n\n"
108
+ "1. Tailscale VPN connected (for direct VPC access)\n"
109
+ " - Install Tailscale: https://tailscale.com/download\n"
110
+ " - Connect to the Airbyte network\n\n"
111
+ "2. Cloud SQL Proxy running locally\n"
112
+ " - Start the proxy:\n"
113
+ " airbyte-ops cloud db start-proxy\n"
114
+ " uvx --from=airbyte-internal-ops airbyte-ops cloud db start-proxy\n"
115
+ " - Set env vars: export USE_CLOUD_SQL_PROXY=1 DB_PORT=15432\n"
116
+ )
117
+
118
+
119
+ def _check_proxy_is_running(host: str, port: int) -> None:
120
+ """Check if the Cloud SQL Proxy is running and accepting connections.
121
+
122
+ This performs a quick socket connection check to fail fast if the proxy
123
+ is not running, rather than waiting for a long connection timeout.
124
+
125
+ Args:
126
+ host: The host to connect to (typically 127.0.0.1)
127
+ port: The port to connect to
128
+
129
+ Raises:
130
+ CloudSqlProxyNotRunningError: If the proxy is not accepting connections
131
+ """
132
+ try:
133
+ with socket.create_connection((host, port), timeout=PROXY_CHECK_TIMEOUT):
134
+ pass # Connection successful, proxy is running
135
+ except (OSError, TimeoutError, ConnectionRefusedError) as e:
136
+ raise CloudSqlProxyNotRunningError(
137
+ f"Cloud SQL Proxy is not running on {host}:{port}. "
138
+ f"Proxy mode is enabled (CI or USE_CLOUD_SQL_PROXY env var is set), "
139
+ f"but nothing is listening on the expected port.\n\n"
140
+ f"To start the proxy, run:\n"
141
+ f" airbyte-ops cloud db start-proxy --port {port}\n"
142
+ f" uvx --from=airbyte-internal-ops airbyte-ops cloud db start-proxy --port {port}\n\n"
143
+ f"Or unset USE_CLOUD_SQL_PROXY to use direct VPC connection.\n\n"
144
+ f"Original error: {e}"
145
+ ) from e
146
+
27
147
 
28
148
  # Lazy-initialized to avoid import-time GCP auth
29
149
  _connector: Connector | None = None
@@ -81,16 +201,20 @@ def get_pool(
81
201
  """Get a SQLAlchemy connection pool for the Airbyte Cloud database.
82
202
 
83
203
  This function supports two connection modes:
84
- 1. Direct connection via Cloud SQL Python Connector (default, requires VPC access)
204
+ 1. Direct connection via Cloud SQL Python Connector (default, requires VPC/Tailscale)
85
205
  2. Connection via Cloud SQL Auth Proxy (when CI or USE_CLOUD_SQL_PROXY env var is set)
86
206
 
87
207
  For proxy mode, start the proxy with:
88
- cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --port=<port>
208
+ airbyte-ops cloud db start-proxy
89
209
 
90
210
  Environment variables:
91
211
  CI: If set, uses proxy connection mode
92
212
  USE_CLOUD_SQL_PROXY: If set, uses proxy connection mode
93
- DB_PORT: Port for proxy connection (default: 5432)
213
+ DB_PORT: Port for proxy connection (default: 15432)
214
+
215
+ Raises:
216
+ VpnNotConnectedError: If direct mode is used but no VPN/proxy is detected
217
+ CloudSqlProxyNotRunningError: If proxy mode is enabled but the proxy is not running
94
218
 
95
219
  Args:
96
220
  gsm_client: GCP Secret Manager client for retrieving credentials
@@ -98,6 +222,9 @@ def get_pool(
98
222
  Returns:
99
223
  SQLAlchemy Engine connected to the Prod DB Replica
100
224
  """
225
+ # Fail fast if no VPN or proxy is available
226
+ _check_vpn_or_proxy_available()
227
+
101
228
  pg_connection_details = json.loads(
102
229
  _get_secret_value(
103
230
  gsm_client, CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID
@@ -106,21 +233,21 @@ def get_pool(
106
233
 
107
234
  if os.getenv("CI") or os.getenv("USE_CLOUD_SQL_PROXY"):
108
235
  # Connect via Cloud SQL Auth Proxy, running on localhost
109
- # Port can be configured via DB_PORT env var (default: 5432)
236
+ # Port can be configured via DB_PORT env var (default: DEFAULT_CLOUD_SQL_PROXY_PORT)
110
237
  host = "127.0.0.1"
111
- port = os.getenv("DB_PORT", "5432")
112
- try:
113
- return sqlalchemy.create_engine(
114
- f"postgresql+{PG_DRIVER}://{pg_connection_details['pg_user']}:{pg_connection_details['pg_password']}@{host}:{port}/{pg_connection_details['database_name']}",
115
- )
116
- except Exception as e:
117
- raise AssertionError(
118
- f"sqlalchemy.create_engine exception; could not connect to the proxy at {host}:{port}. "
119
- f"Error: {traceback.format_exception(e)}"
120
- ) from e
238
+ port = int(os.getenv("DB_PORT", str(DEFAULT_CLOUD_SQL_PROXY_PORT)))
239
+
240
+ # Fail fast if proxy is not running
241
+ _check_proxy_is_running(host, port)
242
+
243
+ return sqlalchemy.create_engine(
244
+ f"postgresql+{PG_DRIVER}://{pg_connection_details['pg_user']}:{pg_connection_details['pg_password']}@{host}:{port}/{pg_connection_details['database_name']}",
245
+ )
121
246
 
122
- # Default: Connect via Cloud SQL Python Connector (requires VPC access)
247
+ # Default: Connect via Cloud SQL Python Connector (requires VPC/Tailscale access)
248
+ # Use a timeout to fail faster if the connection can't be established
123
249
  return sqlalchemy.create_engine(
124
250
  f"postgresql+{PG_DRIVER}://",
125
251
  creator=get_database_creator(pg_connection_details),
252
+ connect_args={"timeout": DIRECT_CONNECTION_TIMEOUT},
126
253
  )
@@ -21,8 +21,10 @@ from airbyte_ops_mcp.prod_db_access.db_engine import get_pool
21
21
  from airbyte_ops_mcp.prod_db_access.sql import (
22
22
  SELECT_ACTORS_PINNED_TO_VERSION,
23
23
  SELECT_CONNECTIONS_BY_CONNECTOR,
24
+ SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG,
24
25
  SELECT_CONNECTOR_VERSIONS,
25
26
  SELECT_DATAPLANES_LIST,
27
+ SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION,
26
28
  SELECT_NEW_CONNECTOR_RELEASES,
27
29
  SELECT_ORG_WORKSPACES,
28
30
  SELECT_SUCCESSFUL_SYNCS_FOR_VERSION,
@@ -85,14 +87,28 @@ def query_connections_by_connector(
85
87
  Returns:
86
88
  List of connection records with workspace and dataplane info
87
89
  """
90
+ # Use separate queries to avoid pg8000 NULL parameter type issues
91
+ # pg8000 cannot determine the type of NULL parameters in patterns like
92
+ # "(:param IS NULL OR column = :param)"
93
+ if organization_id is None:
94
+ return _run_sql_query(
95
+ SELECT_CONNECTIONS_BY_CONNECTOR,
96
+ parameters={
97
+ "connector_definition_id": connector_definition_id,
98
+ "limit": limit,
99
+ },
100
+ query_name="SELECT_CONNECTIONS_BY_CONNECTOR",
101
+ gsm_client=gsm_client,
102
+ )
103
+
88
104
  return _run_sql_query(
89
- SELECT_CONNECTIONS_BY_CONNECTOR,
105
+ SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG,
90
106
  parameters={
91
107
  "connector_definition_id": connector_definition_id,
92
108
  "organization_id": organization_id,
93
109
  "limit": limit,
94
110
  },
95
- query_name="SELECT_CONNECTIONS_BY_CONNECTOR",
111
+ query_name="SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG",
96
112
  gsm_client=gsm_client,
97
113
  )
98
114
 
@@ -209,6 +225,44 @@ def query_sync_results_for_version(
209
225
  )
210
226
 
211
227
 
228
+ def query_failed_sync_attempts_for_version(
229
+ connector_version_id: str,
230
+ days: int = 7,
231
+ limit: int = 100,
232
+ *,
233
+ gsm_client: secretmanager.SecretManagerServiceClient | None = None,
234
+ ) -> list[dict[str, Any]]:
235
+ """Query failed sync job results with attempt details for actors pinned to a version.
236
+
237
+ This query joins to the attempts table to include failure_summary and other
238
+ attempt-level details useful for debugging. Date filters are applied to both
239
+ jobs and attempts tables to optimize join performance.
240
+
241
+ Note: This may return multiple rows per job (one per attempt). Results are
242
+ ordered by job_updated_at DESC, then attempt_number DESC.
243
+
244
+ Args:
245
+ connector_version_id: Connector version UUID to filter by
246
+ days: Number of days to look back (default: 7)
247
+ limit: Maximum number of results (default: 100)
248
+ gsm_client: GCP Secret Manager client. If None, a new client will be instantiated.
249
+
250
+ Returns:
251
+ List of failed sync job results with attempt details including failure_summary
252
+ """
253
+ cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
254
+ return _run_sql_query(
255
+ SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION,
256
+ parameters={
257
+ "actor_definition_version_id": connector_version_id,
258
+ "cutoff_date": cutoff_date,
259
+ "limit": limit,
260
+ },
261
+ query_name="SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION",
262
+ gsm_client=gsm_client,
263
+ )
264
+
265
+
212
266
  def query_dataplanes_list(
213
267
  *,
214
268
  gsm_client: secretmanager.SecretManagerServiceClient | None = None,
@@ -74,7 +74,9 @@ import sqlalchemy
74
74
  # Connection Queries
75
75
  # =============================================================================
76
76
 
77
- # Query connections by connector type, optionally filtered by organization
77
+ # Query connections by connector type (no organization filter)
78
+ # Note: pg8000 cannot determine the type of NULL parameters in patterns like
79
+ # "(:param IS NULL OR column = :param)", so we use separate queries instead
78
80
  SELECT_CONNECTIONS_BY_CONNECTOR = sqlalchemy.text(
79
81
  """
80
82
  SELECT
@@ -97,7 +99,34 @@ SELECT_CONNECTIONS_BY_CONNECTOR = sqlalchemy.text(
97
99
  ON workspace.dataplane_group_id = dataplane_group.id
98
100
  WHERE
99
101
  source_actor.actor_definition_id = :connector_definition_id
100
- AND (:organization_id IS NULL OR workspace.organization_id = :organization_id)
102
+ LIMIT :limit
103
+ """
104
+ )
105
+
106
+ # Query connections by connector type, filtered by organization
107
+ SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG = sqlalchemy.text(
108
+ """
109
+ SELECT
110
+ connection.id AS connection_id,
111
+ connection.name AS connection_name,
112
+ connection.source_id,
113
+ workspace.id AS workspace_id,
114
+ workspace.name AS workspace_name,
115
+ workspace.organization_id,
116
+ workspace.dataplane_group_id,
117
+ dataplane_group.name AS dataplane_name,
118
+ source_actor.actor_definition_id AS source_definition_id,
119
+ source_actor.name AS source_name
120
+ FROM connection
121
+ JOIN actor AS source_actor
122
+ ON connection.source_id = source_actor.id
123
+ JOIN workspace
124
+ ON source_actor.workspace_id = workspace.id
125
+ LEFT JOIN dataplane_group
126
+ ON workspace.dataplane_group_id = dataplane_group.id
127
+ WHERE
128
+ source_actor.actor_definition_id = :connector_definition_id
129
+ AND workspace.organization_id = :organization_id
101
130
  LIMIT :limit
102
131
  """
103
132
  )
@@ -276,6 +305,66 @@ SELECT_SUCCESSFUL_SYNCS_FOR_VERSION = sqlalchemy.text(
276
305
  """
277
306
  )
278
307
 
308
+ # Get failed attempt results for actors pinned to a specific connector definition VERSION ID
309
+ # Includes attempt details (failure_summary, etc.) for research/debugging
310
+ # Filters on attempts.status = 'failed' to capture all failed attempts, including those
311
+ # from jobs that eventually succeeded via retry. The latest_job_attempt_status field
312
+ # indicates whether the job eventually succeeded or remained failed.
313
+ # Query starts from attempts table to leverage indexed columns (ended_at, status)
314
+ # Note: attempts.ended_at and attempts.status are indexed (btree)
315
+ SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION = sqlalchemy.text(
316
+ """
317
+ SELECT
318
+ jobs.id AS job_id,
319
+ jobs.scope AS connection_id,
320
+ jobs.status AS latest_job_attempt_status,
321
+ jobs.started_at AS job_started_at,
322
+ jobs.updated_at AS job_updated_at,
323
+ connection.name AS connection_name,
324
+ actor.id AS actor_id,
325
+ actor.name AS actor_name,
326
+ actor.actor_definition_id,
327
+ scoped_configuration.origin_type AS pin_origin_type,
328
+ scoped_configuration.origin AS pin_origin,
329
+ workspace.id AS workspace_id,
330
+ workspace.name AS workspace_name,
331
+ workspace.organization_id,
332
+ workspace.dataplane_group_id,
333
+ dataplane_group.name AS dataplane_name,
334
+ attempts.id AS failed_attempt_id,
335
+ attempts.attempt_number AS failed_attempt_number,
336
+ attempts.status AS failed_attempt_status,
337
+ attempts.created_at AS failed_attempt_created_at,
338
+ attempts.ended_at AS failed_attempt_ended_at,
339
+ attempts.failure_summary,
340
+ attempts.processing_task_queue
341
+ FROM attempts
342
+ JOIN jobs
343
+ ON jobs.id = attempts.job_id
344
+ AND jobs.config_type = 'sync'
345
+ AND jobs.updated_at >= :cutoff_date
346
+ JOIN connection
347
+ ON jobs.scope = connection.id::text
348
+ JOIN actor
349
+ ON connection.source_id = actor.id
350
+ JOIN scoped_configuration
351
+ ON scoped_configuration.scope_id = actor.id
352
+ AND scoped_configuration.key = 'connector_version'
353
+ AND scoped_configuration.scope_type = 'actor'
354
+ AND scoped_configuration.value = :actor_definition_version_id
355
+ JOIN workspace
356
+ ON actor.workspace_id = workspace.id
357
+ LEFT JOIN dataplane_group
358
+ ON workspace.dataplane_group_id = dataplane_group.id
359
+ WHERE
360
+ attempts.ended_at >= :cutoff_date
361
+ AND attempts.status = 'failed'
362
+ ORDER BY
363
+ attempts.ended_at DESC
364
+ LIMIT :limit
365
+ """
366
+ )
367
+
279
368
  # =============================================================================
280
369
  # Dataplane and Workspace Queries
281
370
  # =============================================================================