airbyte-internal-ops 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_internal_ops-0.1.8.dist-info → airbyte_internal_ops-0.1.10.dist-info}/METADATA +1 -1
- {airbyte_internal_ops-0.1.8.dist-info → airbyte_internal_ops-0.1.10.dist-info}/RECORD +16 -16
- airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md +2 -2
- airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_entry.py +3 -2
- airbyte_ops_mcp/cli/cloud.py +191 -1
- airbyte_ops_mcp/cli/registry.py +2 -2
- airbyte_ops_mcp/cloud_admin/api_client.py +1 -1
- airbyte_ops_mcp/constants.py +9 -0
- airbyte_ops_mcp/mcp/github.py +2 -2
- airbyte_ops_mcp/mcp/prerelease.py +3 -3
- airbyte_ops_mcp/mcp/prod_db_queries.py +47 -0
- airbyte_ops_mcp/prod_db_access/db_engine.py +143 -16
- airbyte_ops_mcp/prod_db_access/queries.py +56 -2
- airbyte_ops_mcp/prod_db_access/sql.py +91 -2
- {airbyte_internal_ops-0.1.8.dist-info → airbyte_internal_ops-0.1.10.dist-info}/WHEEL +0 -0
- {airbyte_internal_ops-0.1.8.dist-info → airbyte_internal_ops-0.1.10.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
airbyte_ops_mcp/__init__.py,sha256=HhzURuYr29_UIdMrnWYaZB8ENr_kFkBdm4uqeiIW3Vw,760
|
|
2
2
|
airbyte_ops_mcp/_annotations.py,sha256=MO-SBDnbykxxHDESG7d8rviZZ4WlZgJKv0a8eBqcEzQ,1757
|
|
3
|
-
airbyte_ops_mcp/constants.py,sha256=
|
|
3
|
+
airbyte_ops_mcp/constants.py,sha256=col6-5BUWuIYhbtKmlvSRR8URBoSNExoz94cn4_kujI,2333
|
|
4
4
|
airbyte_ops_mcp/gcp_auth.py,sha256=5k-k145ZoYhHLjyDES8nrA8f8BBihRI0ykrdD1IcfOs,3599
|
|
5
5
|
airbyte_ops_mcp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
airbyte_ops_mcp/_legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -207,7 +207,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/checks/documentation/templates/s
|
|
|
207
207
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/checks/documentation/templates/template.md.j2,sha256=bYxLfOP3GDwQC9Q3PiVbxsL5GS0p5b10k1dwzfOrm1M,1962
|
|
208
208
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
209
209
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/templates/qa_checks.md.j2,sha256=v3NyYUAYkqRiYAFDQ2NgpgYK6PIDhDKrqfRgpX8454s,1946
|
|
210
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md,sha256=
|
|
210
|
+
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/README.md,sha256=CQtTgVzHH3UdgRCohA9GyNhxIiYxVZmSwhMKWnfbocE,3569
|
|
211
211
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/package-lock.json,sha256=uazyPocrveknTDfwu4DlcaDQ5HwkPNf1Q_vDNpGMp0s,2233
|
|
212
212
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/package.json,sha256=_JMByXIhM9iYRKlLqheCVWXWswfwDzuXc5-A1VRUdUM,307
|
|
213
213
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_models/transform.py,sha256=cx-u9y0aYKs0M7rL47I62WQfcx6uIyzbL2wWv5GERNM,1939
|
|
@@ -282,7 +282,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/constants.py,sha256=bSXsx-RQ
|
|
|
282
282
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/docker_hub.py,sha256=K-6-QjTiSEPeJUnVJIMyYfTtu6uFoh0vLX579b04LSs,5528
|
|
283
283
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/gcs_upload.py,sha256=_r1nh54U6QFglFiPWBpxrSylE6Guu0Gjk7Bq5NKK4EE,26506
|
|
284
284
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry.py,sha256=3okT639PCZGL0p6tlWJoTYy-xI2igeAj6DerxkMC1_Q,15130
|
|
285
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_entry.py,sha256=
|
|
285
|
+
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_entry.py,sha256=QT-UopRiFGJpabvoya-B3F32qrlB2qluOiNKj56Pv4g,26489
|
|
286
286
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/registry_report.py,sha256=dp13BxBiXG7DgxhzfupMm-sTFdjUVn9TyYrvo6KShU0,12600
|
|
287
287
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/sentry.py,sha256=e1O6_FHrHvqNNQdb5TkXs5Bhv7_hZ8fMiOWJr8H5tkI,1806
|
|
288
288
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/spec_cache.py,sha256=rcsJtu6B8_bmfXFv9U-IVjsVpPDJWLJYdH8e_XbT7yk,4482
|
|
@@ -351,12 +351,12 @@ airbyte_ops_mcp/cli/__init__.py,sha256=XpL7FyVfgabfBF2JR7u7NwJ2krlYqjd_OwLcWf-Xc
|
|
|
351
351
|
airbyte_ops_mcp/cli/_base.py,sha256=I8tWnyQf0ks4r3J8N8h-5GZxyn37T-55KsbuHnxYlcg,415
|
|
352
352
|
airbyte_ops_mcp/cli/_shared.py,sha256=jg-xMyGzTCGPqKd8VTfE_3kGPIyO_3Kx5sQbG4rPc0Y,1311
|
|
353
353
|
airbyte_ops_mcp/cli/app.py,sha256=SEdBpqFUG2O8zGV5ifwptxrLGFph_dLr66-MX9d69gQ,789
|
|
354
|
-
airbyte_ops_mcp/cli/cloud.py,sha256=
|
|
354
|
+
airbyte_ops_mcp/cli/cloud.py,sha256=semAuPqruWfe7JiVmtbELZgcpeqtoIn4LGD0kEvIS30,38981
|
|
355
355
|
airbyte_ops_mcp/cli/gh.py,sha256=91b1AxFXvHQCFyXhrrym-756ZjnMCqvxFdmwCtma1zI,2046
|
|
356
|
-
airbyte_ops_mcp/cli/registry.py,sha256
|
|
356
|
+
airbyte_ops_mcp/cli/registry.py,sha256=-yiLJWSslV_qGi6ImXZYfXOJSE4oJBO7yICkyA_RiUo,5792
|
|
357
357
|
airbyte_ops_mcp/cli/repo.py,sha256=G1hoQpH0XYhUH3FFOsia9xabGB0LP9o3XcwBuqvFVo0,16331
|
|
358
358
|
airbyte_ops_mcp/cloud_admin/__init__.py,sha256=cqE96Q10Kp6elhH9DAi6TVsIwSUy3sooDLLrxTaktGk,816
|
|
359
|
-
airbyte_ops_mcp/cloud_admin/api_client.py,sha256=
|
|
359
|
+
airbyte_ops_mcp/cloud_admin/api_client.py,sha256=6PovHDwOzo4fxSyk6viwvnXjCRIiC4uPZo0pGMx0Bdk,17359
|
|
360
360
|
airbyte_ops_mcp/cloud_admin/auth.py,sha256=j45pRR8fg6CLwVdn7Uu5KW_kTz_CjRP6ZJGUzqHj_Dk,2558
|
|
361
361
|
airbyte_ops_mcp/cloud_admin/connection_config.py,sha256=UtbIwuB7CA3WJr9oYRwlKDsjciqd_9ewWdml2f8DuXw,4887
|
|
362
362
|
airbyte_ops_mcp/cloud_admin/models.py,sha256=YZ3FbEW-tZa50khKTTl4Bzvy_LsGyyQd6qcpXo62jls,2670
|
|
@@ -389,25 +389,25 @@ airbyte_ops_mcp/mcp/_mcp_utils.py,sha256=nhztHcoc-_ASPpJfoDBjxjjqEvQM6_QIrhp7F2U
|
|
|
389
389
|
airbyte_ops_mcp/mcp/cloud_connector_versions.py,sha256=XxaS6WBP0sJPRwT7TTPhVH2PzhPqVWMNU5fVdWdxLLk,10361
|
|
390
390
|
airbyte_ops_mcp/mcp/connector_analysis.py,sha256=OC4KrOSkMkKPkOisWnSv96BDDE5TQYHq-Jxa2vtjJpo,298
|
|
391
391
|
airbyte_ops_mcp/mcp/connector_qa.py,sha256=aImpqdnqBPDrz10BS0owsV4kuIU2XdalzgbaGZsbOL0,258
|
|
392
|
-
airbyte_ops_mcp/mcp/github.py,sha256=
|
|
392
|
+
airbyte_ops_mcp/mcp/github.py,sha256=opmVWRwjNs_jeWejv8wHOtb-3J09hOlqxg98GCzmFLo,7627
|
|
393
393
|
airbyte_ops_mcp/mcp/github_repo_ops.py,sha256=PiERpt8abo20Gz4CfXhrDNlVM4o4FOt5sweZJND2a0s,5314
|
|
394
394
|
airbyte_ops_mcp/mcp/live_tests.py,sha256=eHfgNkEcY1OKzYiJmkxwOluLPiFHIdP4nm_H1H0MXDg,17940
|
|
395
395
|
airbyte_ops_mcp/mcp/metadata.py,sha256=fwGW97WknR5lfKcQnFtK6dU87aA6TmLj1NkKyqDAV9g,270
|
|
396
|
-
airbyte_ops_mcp/mcp/prerelease.py,sha256=
|
|
397
|
-
airbyte_ops_mcp/mcp/prod_db_queries.py,sha256=
|
|
396
|
+
airbyte_ops_mcp/mcp/prerelease.py,sha256=08G6ogRqEauQyDxFLirUVaYeU3exAJd-DJn_8fXCNXg,9450
|
|
397
|
+
airbyte_ops_mcp/mcp/prod_db_queries.py,sha256=RkBVISfkbwML3grWONxYsULRnFEYdqDZVBZIyo6W8xE,14311
|
|
398
398
|
airbyte_ops_mcp/mcp/prompts.py,sha256=mJld9mdPECXYZffWXGSvNs4Xevx3rxqUGNlzGKVC2_s,1599
|
|
399
399
|
airbyte_ops_mcp/mcp/registry.py,sha256=PW-VYUj42qx2pQ_apUkVaoUFq7VgB9zEU7-aGrkSCCw,290
|
|
400
400
|
airbyte_ops_mcp/mcp/server.py,sha256=7zi91xioVTx1q-bEleekZH2c2JnbzDQt_6zxdEwnLbg,2958
|
|
401
401
|
airbyte_ops_mcp/mcp/server_info.py,sha256=Yi4B1auW64QZGBDas5mro_vwTjvrP785TFNSBP7GhRg,2361
|
|
402
402
|
airbyte_ops_mcp/prod_db_access/__init__.py,sha256=5pxouMPY1beyWlB0UwPnbaLTKTHqU6X82rbbgKY2vYU,1069
|
|
403
|
-
airbyte_ops_mcp/prod_db_access/db_engine.py,sha256=
|
|
403
|
+
airbyte_ops_mcp/prod_db_access/db_engine.py,sha256=11xNZTk4I8SKYhsnmE7-LVrkJXN4dCRbBeD1_hj3f-s,9027
|
|
404
404
|
airbyte_ops_mcp/prod_db_access/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
405
|
-
airbyte_ops_mcp/prod_db_access/queries.py,sha256=
|
|
406
|
-
airbyte_ops_mcp/prod_db_access/sql.py,sha256=
|
|
405
|
+
airbyte_ops_mcp/prod_db_access/queries.py,sha256=q7PcI15EGh6jFS9MVB_gZt1a56YvrZV5hnwa5lgU2q0,10844
|
|
406
|
+
airbyte_ops_mcp/prod_db_access/sql.py,sha256=tWQAwMk8DzG8HpLIYglljlReI2oeYulQPsV31ocUJSw,16251
|
|
407
407
|
airbyte_ops_mcp/registry/__init__.py,sha256=iEaPlt9GrnlaLbc__98TguNeZG8wuQu7S-_2QkhHcbA,858
|
|
408
408
|
airbyte_ops_mcp/registry/models.py,sha256=B4L4TKr52wo0xs0CqvCBrpowqjShzVnZ5eTr2-EyhNs,2346
|
|
409
409
|
airbyte_ops_mcp/registry/publish.py,sha256=VoPxsM2_0zJ829orzCRN-kjgcJtuBNyXgW4I9J680ro,12717
|
|
410
|
-
airbyte_internal_ops-0.1.
|
|
411
|
-
airbyte_internal_ops-0.1.
|
|
412
|
-
airbyte_internal_ops-0.1.
|
|
413
|
-
airbyte_internal_ops-0.1.
|
|
410
|
+
airbyte_internal_ops-0.1.10.dist-info/METADATA,sha256=I3WAz4JM1tgyGMcFaV5ElRwvwYjlRLR9kXKzifetd1Q,5283
|
|
411
|
+
airbyte_internal_ops-0.1.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
412
|
+
airbyte_internal_ops-0.1.10.dist-info/entry_points.txt,sha256=eUgJ9xIy9PlR-CgRbqRMsh1NVp6jz08v9bul9vCYlU4,111
|
|
413
|
+
airbyte_internal_ops-0.1.10.dist-info/RECORD,,
|
|
@@ -75,7 +75,7 @@ This will copy the specified connector version to your development bucket. This
|
|
|
75
75
|
_💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
|
|
76
76
|
|
|
77
77
|
```bash
|
|
78
|
-
TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-
|
|
78
|
+
TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe copy-connector-from-prod
|
|
79
79
|
```
|
|
80
80
|
|
|
81
81
|
### Promote Connector Version to Latest
|
|
@@ -87,5 +87,5 @@ _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage
|
|
|
87
87
|
_⚠️ Warning: Its important to know that this will remove ANY existing files in the latest folder that are not in the versioned folder as it calls `gsutil rsync` with `-d` enabled._
|
|
88
88
|
|
|
89
89
|
```bash
|
|
90
|
-
TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-
|
|
90
|
+
TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-preview.ea013c8" poetry run poe promote-connector-to-latest
|
|
91
91
|
```
|
|
@@ -526,7 +526,7 @@ def generate_and_persist_registry_entry(
|
|
|
526
526
|
bucket_name (str): The name of the GCS bucket.
|
|
527
527
|
repo_metadata_file_path (pathlib.Path): The path to the spec file.
|
|
528
528
|
registry_type (str): The registry type.
|
|
529
|
-
docker_image_tag (str): The docker image tag associated with this release. Typically a semver string (e.g. '1.2.3'), possibly with a suffix (e.g. '1.2.3-
|
|
529
|
+
docker_image_tag (str): The docker image tag associated with this release. Typically a semver string (e.g. '1.2.3'), possibly with a suffix (e.g. '1.2.3-preview.abcde12')
|
|
530
530
|
is_prerelease (bool): Whether this is a prerelease, or a main release.
|
|
531
531
|
"""
|
|
532
532
|
# Read the repo metadata dict to bootstrap ourselves. We need the docker repository,
|
|
@@ -536,7 +536,7 @@ def generate_and_persist_registry_entry(
|
|
|
536
536
|
|
|
537
537
|
try:
|
|
538
538
|
# Now that we have the docker repo, read the appropriate versioned metadata from GCS.
|
|
539
|
-
# This metadata will differ in a few fields (e.g. in prerelease mode, dockerImageTag will contain the actual prerelease tag `1.2.3-
|
|
539
|
+
# This metadata will differ in a few fields (e.g. in prerelease mode, dockerImageTag will contain the actual prerelease tag `1.2.3-preview.abcde12`),
|
|
540
540
|
# so we'll treat this as the source of truth (ish. See below for how we handle the registryOverrides field.)
|
|
541
541
|
gcs_client = get_gcs_storage_client(gcs_creds=os.environ.get("GCS_CREDENTIALS"))
|
|
542
542
|
bucket = gcs_client.bucket(bucket_name)
|
|
@@ -645,6 +645,7 @@ def generate_and_persist_registry_entry(
|
|
|
645
645
|
if (
|
|
646
646
|
"-rc" not in metadata_dict["data"]["dockerImageTag"]
|
|
647
647
|
and "-dev" not in metadata_dict["data"]["dockerImageTag"]
|
|
648
|
+
and "-preview" not in metadata_dict["data"]["dockerImageTag"]
|
|
648
649
|
) and not metadata_dict["data"]["registryOverrides"][registry_type]["enabled"]:
|
|
649
650
|
logger.info(
|
|
650
651
|
f"{registry_type} is not enabled: deleting existing {registry_type} registry entry for {metadata_dict['data']['dockerRepository']} at latest path."
|
airbyte_ops_mcp/cli/cloud.py
CHANGED
|
@@ -13,6 +13,12 @@ Commands:
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
import json
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import signal
|
|
19
|
+
import socket
|
|
20
|
+
import subprocess
|
|
21
|
+
import time
|
|
16
22
|
from pathlib import Path
|
|
17
23
|
from typing import Annotated, Literal
|
|
18
24
|
|
|
@@ -30,6 +36,12 @@ from airbyte_ops_mcp.cli._shared import (
|
|
|
30
36
|
print_success,
|
|
31
37
|
)
|
|
32
38
|
from airbyte_ops_mcp.cloud_admin.connection_config import fetch_connection_config
|
|
39
|
+
from airbyte_ops_mcp.constants import (
|
|
40
|
+
CLOUD_SQL_INSTANCE,
|
|
41
|
+
CLOUD_SQL_PROXY_PID_FILE,
|
|
42
|
+
DEFAULT_CLOUD_SQL_PROXY_PORT,
|
|
43
|
+
ENV_GCP_PROD_DB_ACCESS_CREDENTIALS,
|
|
44
|
+
)
|
|
33
45
|
from airbyte_ops_mcp.live_tests.ci_output import (
|
|
34
46
|
generate_regression_report,
|
|
35
47
|
get_report_summary,
|
|
@@ -75,6 +87,184 @@ connector_app = App(
|
|
|
75
87
|
)
|
|
76
88
|
cloud_app.command(connector_app)
|
|
77
89
|
|
|
90
|
+
# Create the db sub-app under cloud
|
|
91
|
+
db_app = App(name="db", help="Database operations for Airbyte Cloud Prod DB Replica.")
|
|
92
|
+
cloud_app.command(db_app)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@db_app.command(name="start-proxy")
|
|
96
|
+
def start_proxy(
|
|
97
|
+
port: Annotated[
|
|
98
|
+
int,
|
|
99
|
+
Parameter(help="Port for the Cloud SQL Proxy to listen on."),
|
|
100
|
+
] = DEFAULT_CLOUD_SQL_PROXY_PORT,
|
|
101
|
+
daemon: Annotated[
|
|
102
|
+
bool,
|
|
103
|
+
Parameter(
|
|
104
|
+
help="Run as daemon in background (default). Use --no-daemon for foreground."
|
|
105
|
+
),
|
|
106
|
+
] = True,
|
|
107
|
+
) -> None:
|
|
108
|
+
"""Start the Cloud SQL Proxy for database access.
|
|
109
|
+
|
|
110
|
+
This command starts the Cloud SQL Auth Proxy to enable connections to the
|
|
111
|
+
Airbyte Cloud Prod DB Replica. The proxy is required for database query tools.
|
|
112
|
+
|
|
113
|
+
By default, runs as a daemon (background process). Use --no-daemon to run in
|
|
114
|
+
foreground mode where you can see logs and stop with Ctrl+C.
|
|
115
|
+
|
|
116
|
+
Credentials are read from the GCP_PROD_DB_ACCESS_CREDENTIALS environment variable,
|
|
117
|
+
which should contain the service account JSON credentials.
|
|
118
|
+
|
|
119
|
+
After starting the proxy, set these environment variables to use database tools:
|
|
120
|
+
export USE_CLOUD_SQL_PROXY=1
|
|
121
|
+
export DB_PORT={port}
|
|
122
|
+
|
|
123
|
+
Example:
|
|
124
|
+
airbyte-ops cloud db start-proxy
|
|
125
|
+
airbyte-ops cloud db start-proxy --port 15432
|
|
126
|
+
airbyte-ops cloud db start-proxy --no-daemon
|
|
127
|
+
"""
|
|
128
|
+
# Check if proxy is already running on the requested port (idempotency)
|
|
129
|
+
try:
|
|
130
|
+
with socket.create_connection(("127.0.0.1", port), timeout=0.5):
|
|
131
|
+
# Something is already listening on this port
|
|
132
|
+
pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
|
|
133
|
+
pid_info = ""
|
|
134
|
+
if pid_file.exists():
|
|
135
|
+
pid_info = f" (PID: {pid_file.read_text().strip()})"
|
|
136
|
+
print_success(
|
|
137
|
+
f"Cloud SQL Proxy is already running on port {port}{pid_info}"
|
|
138
|
+
)
|
|
139
|
+
print_success("")
|
|
140
|
+
print_success("To use database tools, set these environment variables:")
|
|
141
|
+
print_success(" export USE_CLOUD_SQL_PROXY=1")
|
|
142
|
+
print_success(f" export DB_PORT={port}")
|
|
143
|
+
return
|
|
144
|
+
except (OSError, TimeoutError, ConnectionRefusedError):
|
|
145
|
+
pass # Port not in use, proceed with starting proxy
|
|
146
|
+
|
|
147
|
+
# Check if cloud-sql-proxy is installed
|
|
148
|
+
proxy_path = shutil.which("cloud-sql-proxy")
|
|
149
|
+
if not proxy_path:
|
|
150
|
+
exit_with_error(
|
|
151
|
+
"cloud-sql-proxy not found in PATH. "
|
|
152
|
+
"Install it from: https://cloud.google.com/sql/docs/mysql/sql-proxy"
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
# Get credentials from environment
|
|
156
|
+
creds_json = os.getenv(ENV_GCP_PROD_DB_ACCESS_CREDENTIALS)
|
|
157
|
+
if not creds_json:
|
|
158
|
+
exit_with_error(
|
|
159
|
+
f"{ENV_GCP_PROD_DB_ACCESS_CREDENTIALS} environment variable is not set. "
|
|
160
|
+
"This should contain the GCP service account JSON credentials."
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
# Build the command using --json-credentials to avoid writing to disk
|
|
164
|
+
cmd = [
|
|
165
|
+
proxy_path,
|
|
166
|
+
CLOUD_SQL_INSTANCE,
|
|
167
|
+
f"--port={port}",
|
|
168
|
+
f"--json-credentials={creds_json}",
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
print_success(f"Starting Cloud SQL Proxy on port {port}...")
|
|
172
|
+
print_success(f"Instance: {CLOUD_SQL_INSTANCE}")
|
|
173
|
+
print_success("")
|
|
174
|
+
print_success("To use database tools, set these environment variables:")
|
|
175
|
+
print_success(" export USE_CLOUD_SQL_PROXY=1")
|
|
176
|
+
print_success(f" export DB_PORT={port}")
|
|
177
|
+
print_success("")
|
|
178
|
+
|
|
179
|
+
if daemon:
|
|
180
|
+
# Run in background (daemon mode) with log file for diagnostics
|
|
181
|
+
log_file_path = Path("/tmp/airbyte-cloud-sql-proxy.log")
|
|
182
|
+
log_file = log_file_path.open("ab")
|
|
183
|
+
process = subprocess.Popen(
|
|
184
|
+
cmd,
|
|
185
|
+
stdout=subprocess.DEVNULL,
|
|
186
|
+
stderr=log_file,
|
|
187
|
+
start_new_session=True,
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
# Brief wait to verify the process started successfully
|
|
191
|
+
time.sleep(0.5)
|
|
192
|
+
if process.poll() is not None:
|
|
193
|
+
# Process exited immediately - read any error output
|
|
194
|
+
log_file.close()
|
|
195
|
+
error_output = ""
|
|
196
|
+
if log_file_path.exists():
|
|
197
|
+
error_output = log_file_path.read_text()[-1000:] # Last 1000 chars
|
|
198
|
+
exit_with_error(
|
|
199
|
+
f"Cloud SQL Proxy failed to start (exit code: {process.returncode}).\n"
|
|
200
|
+
f"Check logs at {log_file_path}\n"
|
|
201
|
+
f"Recent output: {error_output}"
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
# Write PID to file for stop-proxy command
|
|
205
|
+
pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
|
|
206
|
+
pid_file.write_text(str(process.pid))
|
|
207
|
+
print_success(f"Cloud SQL Proxy started as daemon (PID: {process.pid})")
|
|
208
|
+
print_success(f"Logs: {log_file_path}")
|
|
209
|
+
print_success("To stop: airbyte-ops cloud db stop-proxy")
|
|
210
|
+
else:
|
|
211
|
+
# Run in foreground - replace current process
|
|
212
|
+
# Signals (Ctrl+C) will be handled directly by the cloud-sql-proxy process
|
|
213
|
+
print_success("Running in foreground. Press Ctrl+C to stop the proxy.")
|
|
214
|
+
print_success("")
|
|
215
|
+
os.execv(proxy_path, cmd)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@db_app.command(name="stop-proxy")
|
|
219
|
+
def stop_proxy() -> None:
|
|
220
|
+
"""Stop the Cloud SQL Proxy daemon.
|
|
221
|
+
|
|
222
|
+
This command stops a Cloud SQL Proxy that was started with 'start-proxy'.
|
|
223
|
+
It reads the PID from the PID file and sends a SIGTERM signal to stop the process.
|
|
224
|
+
|
|
225
|
+
Example:
|
|
226
|
+
airbyte-ops cloud db stop-proxy
|
|
227
|
+
"""
|
|
228
|
+
pid_file = Path(CLOUD_SQL_PROXY_PID_FILE)
|
|
229
|
+
|
|
230
|
+
if not pid_file.exists():
|
|
231
|
+
exit_with_error(
|
|
232
|
+
f"PID file not found at {CLOUD_SQL_PROXY_PID_FILE}. "
|
|
233
|
+
"No Cloud SQL Proxy daemon appears to be running."
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
pid_str = pid_file.read_text().strip()
|
|
237
|
+
if not pid_str.isdigit():
|
|
238
|
+
pid_file.unlink()
|
|
239
|
+
exit_with_error(f"Invalid PID in {CLOUD_SQL_PROXY_PID_FILE}: {pid_str}")
|
|
240
|
+
|
|
241
|
+
pid = int(pid_str)
|
|
242
|
+
|
|
243
|
+
# Check if process is still running
|
|
244
|
+
try:
|
|
245
|
+
os.kill(pid, 0) # Signal 0 just checks if process exists
|
|
246
|
+
except ProcessLookupError:
|
|
247
|
+
pid_file.unlink()
|
|
248
|
+
print_success(
|
|
249
|
+
f"Cloud SQL Proxy (PID: {pid}) is not running. Cleaned up PID file."
|
|
250
|
+
)
|
|
251
|
+
return
|
|
252
|
+
except PermissionError:
|
|
253
|
+
exit_with_error(f"Permission denied to check process {pid}.")
|
|
254
|
+
|
|
255
|
+
# Send SIGTERM to stop the process
|
|
256
|
+
try:
|
|
257
|
+
os.kill(pid, signal.SIGTERM)
|
|
258
|
+
print_success(f"Sent SIGTERM to Cloud SQL Proxy (PID: {pid}).")
|
|
259
|
+
except ProcessLookupError:
|
|
260
|
+
print_success(f"Cloud SQL Proxy (PID: {pid}) already stopped.")
|
|
261
|
+
except PermissionError:
|
|
262
|
+
exit_with_error(f"Permission denied to stop process {pid}.")
|
|
263
|
+
|
|
264
|
+
# Clean up PID file
|
|
265
|
+
pid_file.unlink(missing_ok=True)
|
|
266
|
+
print_success("Cloud SQL Proxy stopped.")
|
|
267
|
+
|
|
78
268
|
|
|
79
269
|
@connector_app.command(name="get-version-info")
|
|
80
270
|
def get_version_info(
|
|
@@ -117,7 +307,7 @@ def set_version_override(
|
|
|
117
307
|
version: Annotated[
|
|
118
308
|
str,
|
|
119
309
|
Parameter(
|
|
120
|
-
help="The semver version string to pin to (e.g., '2.1.5-
|
|
310
|
+
help="The semver version string to pin to (e.g., '2.1.5-preview.abc1234')."
|
|
121
311
|
),
|
|
122
312
|
],
|
|
123
313
|
reason: Annotated[
|
airbyte_ops_mcp/cli/registry.py
CHANGED
|
@@ -63,7 +63,7 @@ def publish_prerelease(
|
|
|
63
63
|
"""Publish a connector prerelease to the Airbyte registry.
|
|
64
64
|
|
|
65
65
|
Triggers the publish-connectors-prerelease workflow in the airbytehq/airbyte
|
|
66
|
-
repository. Pre-release versions are tagged with format: {version}-
|
|
66
|
+
repository. Pre-release versions are tagged with format: {version}-preview.{git-sha}
|
|
67
67
|
|
|
68
68
|
Requires GITHUB_CONNECTOR_PUBLISHING_PAT or GITHUB_TOKEN environment variable
|
|
69
69
|
with 'actions:write' permission.
|
|
@@ -162,7 +162,7 @@ def inspect_image(
|
|
|
162
162
|
],
|
|
163
163
|
tag: Annotated[
|
|
164
164
|
str,
|
|
165
|
-
Parameter(help="Image tag (e.g., '2.1.5-
|
|
165
|
+
Parameter(help="Image tag (e.g., '2.1.5-preview.abc1234')."),
|
|
166
166
|
],
|
|
167
167
|
) -> None:
|
|
168
168
|
"""Check if a Docker image exists on DockerHub.
|
|
@@ -160,7 +160,7 @@ def resolve_connector_version_id(
|
|
|
160
160
|
Args:
|
|
161
161
|
actor_definition_id: The actor definition ID
|
|
162
162
|
connector_type: Either "source" or "destination"
|
|
163
|
-
version: The version string (e.g., "0.1.47-
|
|
163
|
+
version: The version string (e.g., "0.1.47-preview.abe7cb4")
|
|
164
164
|
api_root: The API root URL
|
|
165
165
|
client_id: The Airbyte Cloud client ID
|
|
166
166
|
client_secret: The Airbyte Cloud client secret
|
airbyte_ops_mcp/constants.py
CHANGED
|
@@ -27,6 +27,15 @@ EXPECTED_ADMIN_EMAIL_DOMAIN = "@airbyte.io"
|
|
|
27
27
|
GCP_PROJECT_NAME = "prod-ab-cloud-proj"
|
|
28
28
|
"""The GCP project name for Airbyte Cloud production."""
|
|
29
29
|
|
|
30
|
+
CLOUD_SQL_INSTANCE = "prod-ab-cloud-proj:us-west3:prod-pgsql-replica"
|
|
31
|
+
"""The Cloud SQL instance connection name for the Prod DB Replica."""
|
|
32
|
+
|
|
33
|
+
DEFAULT_CLOUD_SQL_PROXY_PORT = 15432
|
|
34
|
+
"""Default port for Cloud SQL Proxy connections."""
|
|
35
|
+
|
|
36
|
+
CLOUD_SQL_PROXY_PID_FILE = "/tmp/airbyte-cloud-sql-proxy.pid"
|
|
37
|
+
"""PID file for tracking the Cloud SQL Proxy process."""
|
|
38
|
+
|
|
30
39
|
CLOUD_REGISTRY_URL = (
|
|
31
40
|
"https://connectors.airbyte.com/files/registries/v0/cloud_registry.json"
|
|
32
41
|
)
|
airbyte_ops_mcp/mcp/github.py
CHANGED
|
@@ -204,7 +204,7 @@ def _check_dockerhub_image(
|
|
|
204
204
|
|
|
205
205
|
Args:
|
|
206
206
|
image: Docker image name (e.g., "airbyte/source-github")
|
|
207
|
-
tag: Image tag (e.g., "2.1.5-
|
|
207
|
+
tag: Image tag (e.g., "2.1.5-preview.abc1234")
|
|
208
208
|
|
|
209
209
|
Returns:
|
|
210
210
|
Tag data dictionary if found, None if not found.
|
|
@@ -232,7 +232,7 @@ def get_docker_image_info(
|
|
|
232
232
|
],
|
|
233
233
|
tag: Annotated[
|
|
234
234
|
str,
|
|
235
|
-
Field(description="Image tag (e.g., '2.1.5-
|
|
235
|
+
Field(description="Image tag (e.g., '2.1.5-preview.abc1234')"),
|
|
236
236
|
],
|
|
237
237
|
) -> DockerImageInfo:
|
|
238
238
|
"""Check if a Docker image exists on DockerHub.
|
|
@@ -111,7 +111,7 @@ def _get_pr_head_info(
|
|
|
111
111
|
return PRHeadInfo(
|
|
112
112
|
ref=pr_data["head"]["ref"],
|
|
113
113
|
sha=sha,
|
|
114
|
-
short_sha=sha[:
|
|
114
|
+
short_sha=sha[:7],
|
|
115
115
|
)
|
|
116
116
|
|
|
117
117
|
|
|
@@ -235,7 +235,7 @@ def publish_connector_to_airbyte_registry(
|
|
|
235
235
|
publish-connectors-prerelease workflow in the airbytehq/airbyte repository,
|
|
236
236
|
which publishes a pre-release version of the specified connector from the PR branch.
|
|
237
237
|
|
|
238
|
-
Pre-release versions are tagged with the format: {version}-
|
|
238
|
+
Pre-release versions are tagged with the format: {version}-preview.{7-char-git-sha}
|
|
239
239
|
These versions are available for version pinning via the scoped_configuration API.
|
|
240
240
|
|
|
241
241
|
Requires GITHUB_CONNECTOR_PUBLISHING_PAT or GITHUB_TOKEN environment variable
|
|
@@ -290,7 +290,7 @@ def publish_connector_to_airbyte_registry(
|
|
|
290
290
|
docker_image = data.get("dockerRepository")
|
|
291
291
|
base_version = data.get("dockerImageTag")
|
|
292
292
|
if base_version:
|
|
293
|
-
docker_image_tag = f"{base_version}-
|
|
293
|
+
docker_image_tag = f"{base_version}-preview.{head_info.short_sha}"
|
|
294
294
|
|
|
295
295
|
return PrereleaseWorkflowResult(
|
|
296
296
|
success=True,
|
|
@@ -20,6 +20,7 @@ from airbyte_ops_mcp.prod_db_access.queries import (
|
|
|
20
20
|
query_connections_by_connector,
|
|
21
21
|
query_connector_versions,
|
|
22
22
|
query_dataplanes_list,
|
|
23
|
+
query_failed_sync_attempts_for_version,
|
|
23
24
|
query_new_connector_releases,
|
|
24
25
|
query_sync_results_for_version,
|
|
25
26
|
query_workspace_info,
|
|
@@ -245,6 +246,52 @@ def query_prod_connector_version_sync_results(
|
|
|
245
246
|
)
|
|
246
247
|
|
|
247
248
|
|
|
249
|
+
@mcp_tool(
|
|
250
|
+
read_only=True,
|
|
251
|
+
idempotent=True,
|
|
252
|
+
)
|
|
253
|
+
def query_prod_failed_sync_attempts_for_version(
|
|
254
|
+
connector_version_id: Annotated[
|
|
255
|
+
str,
|
|
256
|
+
Field(description="Connector version UUID to find failed sync attempts for"),
|
|
257
|
+
],
|
|
258
|
+
days: Annotated[
|
|
259
|
+
int,
|
|
260
|
+
Field(description="Number of days to look back (default: 7)", default=7),
|
|
261
|
+
] = 7,
|
|
262
|
+
limit: Annotated[
|
|
263
|
+
int,
|
|
264
|
+
Field(description="Maximum number of results (default: 100)", default=100),
|
|
265
|
+
] = 100,
|
|
266
|
+
) -> list[dict[str, Any]]:
|
|
267
|
+
"""List failed sync attempts with failure details for actors pinned to a connector version.
|
|
268
|
+
|
|
269
|
+
Returns failed attempt records for connections using actors pinned to the specified
|
|
270
|
+
version. Includes failure_summary from the attempts table for debugging.
|
|
271
|
+
|
|
272
|
+
Key fields:
|
|
273
|
+
- latest_job_attempt_status: Final job status after all retries ('succeeded' means
|
|
274
|
+
the job eventually succeeded despite this failed attempt)
|
|
275
|
+
- failed_attempt_number: Which attempt this was (0-indexed)
|
|
276
|
+
- failure_summary: JSON containing failure details including failureType and messages
|
|
277
|
+
|
|
278
|
+
Note: May return multiple rows per job (one per failed attempt). Results ordered by
|
|
279
|
+
job_updated_at DESC, then failed_attempt_number DESC.
|
|
280
|
+
|
|
281
|
+
Returns list of dicts with keys: job_id, connection_id, latest_job_attempt_status,
|
|
282
|
+
job_started_at, job_updated_at, connection_name, actor_id, actor_name,
|
|
283
|
+
actor_definition_id, pin_origin_type, pin_origin, workspace_id, workspace_name,
|
|
284
|
+
organization_id, dataplane_group_id, dataplane_name, failed_attempt_id,
|
|
285
|
+
failed_attempt_number, failed_attempt_status, failed_attempt_created_at,
|
|
286
|
+
failed_attempt_ended_at, failure_summary, processing_task_queue
|
|
287
|
+
"""
|
|
288
|
+
return query_failed_sync_attempts_for_version(
|
|
289
|
+
connector_version_id,
|
|
290
|
+
days=days,
|
|
291
|
+
limit=limit,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
248
295
|
@mcp_tool(
|
|
249
296
|
read_only=True,
|
|
250
297
|
idempotent=True,
|
|
@@ -11,7 +11,9 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
13
|
import os
|
|
14
|
-
import
|
|
14
|
+
import shutil
|
|
15
|
+
import socket
|
|
16
|
+
import subprocess
|
|
15
17
|
from typing import Any, Callable
|
|
16
18
|
|
|
17
19
|
import sqlalchemy
|
|
@@ -21,9 +23,127 @@ from google.cloud.sql.connector.enums import IPTypes
|
|
|
21
23
|
|
|
22
24
|
from airbyte_ops_mcp.constants import (
|
|
23
25
|
CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID,
|
|
26
|
+
DEFAULT_CLOUD_SQL_PROXY_PORT,
|
|
24
27
|
)
|
|
25
28
|
|
|
26
29
|
PG_DRIVER = "pg8000"
|
|
30
|
+
PROXY_CHECK_TIMEOUT = 0.5 # seconds
|
|
31
|
+
DIRECT_CONNECTION_TIMEOUT = 5 # seconds - timeout for direct VPC/Tailscale connections
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CloudSqlProxyNotRunningError(Exception):
|
|
35
|
+
"""Raised when proxy mode is enabled but the Cloud SQL Proxy is not running."""
|
|
36
|
+
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class VpnNotConnectedError(Exception):
|
|
41
|
+
"""Raised when direct connection mode requires VPN but it's not connected."""
|
|
42
|
+
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _is_tailscale_connected() -> bool:
|
|
47
|
+
"""Check if Tailscale VPN is likely connected.
|
|
48
|
+
|
|
49
|
+
This is a best-effort check that works on Linux and macOS.
|
|
50
|
+
Returns True if Tailscale appears to be connected, False otherwise.
|
|
51
|
+
|
|
52
|
+
Detection methods:
|
|
53
|
+
1. Check for tailscale0 network interface (Linux)
|
|
54
|
+
2. Run 'tailscale status --json' and check backend state (cross-platform)
|
|
55
|
+
"""
|
|
56
|
+
# Method 1: Check for tailscale0 interface (Linux)
|
|
57
|
+
try:
|
|
58
|
+
interfaces = [name for _, name in socket.if_nameindex()]
|
|
59
|
+
if "tailscale0" in interfaces:
|
|
60
|
+
return True
|
|
61
|
+
except (OSError, AttributeError):
|
|
62
|
+
pass # if_nameindex not available on this platform
|
|
63
|
+
|
|
64
|
+
# Method 2: Check tailscale CLI status
|
|
65
|
+
tailscale_path = shutil.which("tailscale")
|
|
66
|
+
if tailscale_path:
|
|
67
|
+
try:
|
|
68
|
+
result = subprocess.run(
|
|
69
|
+
[tailscale_path, "status", "--json"],
|
|
70
|
+
capture_output=True,
|
|
71
|
+
text=True,
|
|
72
|
+
timeout=2,
|
|
73
|
+
)
|
|
74
|
+
if result.returncode == 0:
|
|
75
|
+
import json as json_module
|
|
76
|
+
|
|
77
|
+
status = json_module.loads(result.stdout)
|
|
78
|
+
# BackendState "Running" indicates connected
|
|
79
|
+
return status.get("BackendState") == "Running"
|
|
80
|
+
except (subprocess.TimeoutExpired, subprocess.SubprocessError, ValueError):
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _check_vpn_or_proxy_available() -> None:
|
|
87
|
+
"""Check if either VPN or proxy is available for database access.
|
|
88
|
+
|
|
89
|
+
This function checks if the environment is properly configured for
|
|
90
|
+
database access. It fails fast with a helpful error message if neither
|
|
91
|
+
Tailscale VPN nor the Cloud SQL Proxy appears to be available.
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
VpnNotConnectedError: If no VPN or proxy is detected
|
|
95
|
+
"""
|
|
96
|
+
# If proxy mode is explicitly enabled, don't check VPN
|
|
97
|
+
if os.getenv("CI") or os.getenv("USE_CLOUD_SQL_PROXY"):
|
|
98
|
+
return
|
|
99
|
+
|
|
100
|
+
# Check if Tailscale is connected
|
|
101
|
+
if _is_tailscale_connected():
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
# Neither proxy mode nor Tailscale detected
|
|
105
|
+
raise VpnNotConnectedError(
|
|
106
|
+
"No VPN or proxy detected for database access.\n\n"
|
|
107
|
+
"To connect to the Airbyte Cloud Prod DB Replica, you need either:\n\n"
|
|
108
|
+
"1. Tailscale VPN connected (for direct VPC access)\n"
|
|
109
|
+
" - Install Tailscale: https://tailscale.com/download\n"
|
|
110
|
+
" - Connect to the Airbyte network\n\n"
|
|
111
|
+
"2. Cloud SQL Proxy running locally\n"
|
|
112
|
+
" - Start the proxy:\n"
|
|
113
|
+
" airbyte-ops cloud db start-proxy\n"
|
|
114
|
+
" uvx --from=airbyte-internal-ops airbyte-ops cloud db start-proxy\n"
|
|
115
|
+
" - Set env vars: export USE_CLOUD_SQL_PROXY=1 DB_PORT=15432\n"
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _check_proxy_is_running(host: str, port: int) -> None:
|
|
120
|
+
"""Check if the Cloud SQL Proxy is running and accepting connections.
|
|
121
|
+
|
|
122
|
+
This performs a quick socket connection check to fail fast if the proxy
|
|
123
|
+
is not running, rather than waiting for a long connection timeout.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
host: The host to connect to (typically 127.0.0.1)
|
|
127
|
+
port: The port to connect to
|
|
128
|
+
|
|
129
|
+
Raises:
|
|
130
|
+
CloudSqlProxyNotRunningError: If the proxy is not accepting connections
|
|
131
|
+
"""
|
|
132
|
+
try:
|
|
133
|
+
with socket.create_connection((host, port), timeout=PROXY_CHECK_TIMEOUT):
|
|
134
|
+
pass # Connection successful, proxy is running
|
|
135
|
+
except (OSError, TimeoutError, ConnectionRefusedError) as e:
|
|
136
|
+
raise CloudSqlProxyNotRunningError(
|
|
137
|
+
f"Cloud SQL Proxy is not running on {host}:{port}. "
|
|
138
|
+
f"Proxy mode is enabled (CI or USE_CLOUD_SQL_PROXY env var is set), "
|
|
139
|
+
f"but nothing is listening on the expected port.\n\n"
|
|
140
|
+
f"To start the proxy, run:\n"
|
|
141
|
+
f" airbyte-ops cloud db start-proxy --port {port}\n"
|
|
142
|
+
f" uvx --from=airbyte-internal-ops airbyte-ops cloud db start-proxy --port {port}\n\n"
|
|
143
|
+
f"Or unset USE_CLOUD_SQL_PROXY to use direct VPC connection.\n\n"
|
|
144
|
+
f"Original error: {e}"
|
|
145
|
+
) from e
|
|
146
|
+
|
|
27
147
|
|
|
28
148
|
# Lazy-initialized to avoid import-time GCP auth
|
|
29
149
|
_connector: Connector | None = None
|
|
@@ -81,16 +201,20 @@ def get_pool(
|
|
|
81
201
|
"""Get a SQLAlchemy connection pool for the Airbyte Cloud database.
|
|
82
202
|
|
|
83
203
|
This function supports two connection modes:
|
|
84
|
-
1. Direct connection via Cloud SQL Python Connector (default, requires VPC
|
|
204
|
+
1. Direct connection via Cloud SQL Python Connector (default, requires VPC/Tailscale)
|
|
85
205
|
2. Connection via Cloud SQL Auth Proxy (when CI or USE_CLOUD_SQL_PROXY env var is set)
|
|
86
206
|
|
|
87
207
|
For proxy mode, start the proxy with:
|
|
88
|
-
|
|
208
|
+
airbyte-ops cloud db start-proxy
|
|
89
209
|
|
|
90
210
|
Environment variables:
|
|
91
211
|
CI: If set, uses proxy connection mode
|
|
92
212
|
USE_CLOUD_SQL_PROXY: If set, uses proxy connection mode
|
|
93
|
-
DB_PORT: Port for proxy connection (default:
|
|
213
|
+
DB_PORT: Port for proxy connection (default: 15432)
|
|
214
|
+
|
|
215
|
+
Raises:
|
|
216
|
+
VpnNotConnectedError: If direct mode is used but no VPN/proxy is detected
|
|
217
|
+
CloudSqlProxyNotRunningError: If proxy mode is enabled but the proxy is not running
|
|
94
218
|
|
|
95
219
|
Args:
|
|
96
220
|
gsm_client: GCP Secret Manager client for retrieving credentials
|
|
@@ -98,6 +222,9 @@ def get_pool(
|
|
|
98
222
|
Returns:
|
|
99
223
|
SQLAlchemy Engine connected to the Prod DB Replica
|
|
100
224
|
"""
|
|
225
|
+
# Fail fast if no VPN or proxy is available
|
|
226
|
+
_check_vpn_or_proxy_available()
|
|
227
|
+
|
|
101
228
|
pg_connection_details = json.loads(
|
|
102
229
|
_get_secret_value(
|
|
103
230
|
gsm_client, CONNECTION_RETRIEVER_PG_CONNECTION_DETAILS_SECRET_ID
|
|
@@ -106,21 +233,21 @@ def get_pool(
|
|
|
106
233
|
|
|
107
234
|
if os.getenv("CI") or os.getenv("USE_CLOUD_SQL_PROXY"):
|
|
108
235
|
# Connect via Cloud SQL Auth Proxy, running on localhost
|
|
109
|
-
# Port can be configured via DB_PORT env var (default:
|
|
236
|
+
# Port can be configured via DB_PORT env var (default: DEFAULT_CLOUD_SQL_PROXY_PORT)
|
|
110
237
|
host = "127.0.0.1"
|
|
111
|
-
port = os.getenv("DB_PORT",
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
f"Error: {traceback.format_exception(e)}"
|
|
120
|
-
) from e
|
|
238
|
+
port = int(os.getenv("DB_PORT", str(DEFAULT_CLOUD_SQL_PROXY_PORT)))
|
|
239
|
+
|
|
240
|
+
# Fail fast if proxy is not running
|
|
241
|
+
_check_proxy_is_running(host, port)
|
|
242
|
+
|
|
243
|
+
return sqlalchemy.create_engine(
|
|
244
|
+
f"postgresql+{PG_DRIVER}://{pg_connection_details['pg_user']}:{pg_connection_details['pg_password']}@{host}:{port}/{pg_connection_details['database_name']}",
|
|
245
|
+
)
|
|
121
246
|
|
|
122
|
-
# Default: Connect via Cloud SQL Python Connector (requires VPC access)
|
|
247
|
+
# Default: Connect via Cloud SQL Python Connector (requires VPC/Tailscale access)
|
|
248
|
+
# Use a timeout to fail faster if the connection can't be established
|
|
123
249
|
return sqlalchemy.create_engine(
|
|
124
250
|
f"postgresql+{PG_DRIVER}://",
|
|
125
251
|
creator=get_database_creator(pg_connection_details),
|
|
252
|
+
connect_args={"timeout": DIRECT_CONNECTION_TIMEOUT},
|
|
126
253
|
)
|
|
@@ -21,8 +21,10 @@ from airbyte_ops_mcp.prod_db_access.db_engine import get_pool
|
|
|
21
21
|
from airbyte_ops_mcp.prod_db_access.sql import (
|
|
22
22
|
SELECT_ACTORS_PINNED_TO_VERSION,
|
|
23
23
|
SELECT_CONNECTIONS_BY_CONNECTOR,
|
|
24
|
+
SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG,
|
|
24
25
|
SELECT_CONNECTOR_VERSIONS,
|
|
25
26
|
SELECT_DATAPLANES_LIST,
|
|
27
|
+
SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION,
|
|
26
28
|
SELECT_NEW_CONNECTOR_RELEASES,
|
|
27
29
|
SELECT_ORG_WORKSPACES,
|
|
28
30
|
SELECT_SUCCESSFUL_SYNCS_FOR_VERSION,
|
|
@@ -85,14 +87,28 @@ def query_connections_by_connector(
|
|
|
85
87
|
Returns:
|
|
86
88
|
List of connection records with workspace and dataplane info
|
|
87
89
|
"""
|
|
90
|
+
# Use separate queries to avoid pg8000 NULL parameter type issues
|
|
91
|
+
# pg8000 cannot determine the type of NULL parameters in patterns like
|
|
92
|
+
# "(:param IS NULL OR column = :param)"
|
|
93
|
+
if organization_id is None:
|
|
94
|
+
return _run_sql_query(
|
|
95
|
+
SELECT_CONNECTIONS_BY_CONNECTOR,
|
|
96
|
+
parameters={
|
|
97
|
+
"connector_definition_id": connector_definition_id,
|
|
98
|
+
"limit": limit,
|
|
99
|
+
},
|
|
100
|
+
query_name="SELECT_CONNECTIONS_BY_CONNECTOR",
|
|
101
|
+
gsm_client=gsm_client,
|
|
102
|
+
)
|
|
103
|
+
|
|
88
104
|
return _run_sql_query(
|
|
89
|
-
|
|
105
|
+
SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG,
|
|
90
106
|
parameters={
|
|
91
107
|
"connector_definition_id": connector_definition_id,
|
|
92
108
|
"organization_id": organization_id,
|
|
93
109
|
"limit": limit,
|
|
94
110
|
},
|
|
95
|
-
query_name="
|
|
111
|
+
query_name="SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG",
|
|
96
112
|
gsm_client=gsm_client,
|
|
97
113
|
)
|
|
98
114
|
|
|
@@ -209,6 +225,44 @@ def query_sync_results_for_version(
|
|
|
209
225
|
)
|
|
210
226
|
|
|
211
227
|
|
|
228
|
+
def query_failed_sync_attempts_for_version(
|
|
229
|
+
connector_version_id: str,
|
|
230
|
+
days: int = 7,
|
|
231
|
+
limit: int = 100,
|
|
232
|
+
*,
|
|
233
|
+
gsm_client: secretmanager.SecretManagerServiceClient | None = None,
|
|
234
|
+
) -> list[dict[str, Any]]:
|
|
235
|
+
"""Query failed sync job results with attempt details for actors pinned to a version.
|
|
236
|
+
|
|
237
|
+
This query joins to the attempts table to include failure_summary and other
|
|
238
|
+
attempt-level details useful for debugging. Date filters are applied to both
|
|
239
|
+
jobs and attempts tables to optimize join performance.
|
|
240
|
+
|
|
241
|
+
Note: This may return multiple rows per job (one per attempt). Results are
|
|
242
|
+
ordered by job_updated_at DESC, then attempt_number DESC.
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
connector_version_id: Connector version UUID to filter by
|
|
246
|
+
days: Number of days to look back (default: 7)
|
|
247
|
+
limit: Maximum number of results (default: 100)
|
|
248
|
+
gsm_client: GCP Secret Manager client. If None, a new client will be instantiated.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
List of failed sync job results with attempt details including failure_summary
|
|
252
|
+
"""
|
|
253
|
+
cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
|
|
254
|
+
return _run_sql_query(
|
|
255
|
+
SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION,
|
|
256
|
+
parameters={
|
|
257
|
+
"actor_definition_version_id": connector_version_id,
|
|
258
|
+
"cutoff_date": cutoff_date,
|
|
259
|
+
"limit": limit,
|
|
260
|
+
},
|
|
261
|
+
query_name="SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION",
|
|
262
|
+
gsm_client=gsm_client,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
212
266
|
def query_dataplanes_list(
|
|
213
267
|
*,
|
|
214
268
|
gsm_client: secretmanager.SecretManagerServiceClient | None = None,
|
|
@@ -74,7 +74,9 @@ import sqlalchemy
|
|
|
74
74
|
# Connection Queries
|
|
75
75
|
# =============================================================================
|
|
76
76
|
|
|
77
|
-
# Query connections by connector type
|
|
77
|
+
# Query connections by connector type (no organization filter)
|
|
78
|
+
# Note: pg8000 cannot determine the type of NULL parameters in patterns like
|
|
79
|
+
# "(:param IS NULL OR column = :param)", so we use separate queries instead
|
|
78
80
|
SELECT_CONNECTIONS_BY_CONNECTOR = sqlalchemy.text(
|
|
79
81
|
"""
|
|
80
82
|
SELECT
|
|
@@ -97,7 +99,34 @@ SELECT_CONNECTIONS_BY_CONNECTOR = sqlalchemy.text(
|
|
|
97
99
|
ON workspace.dataplane_group_id = dataplane_group.id
|
|
98
100
|
WHERE
|
|
99
101
|
source_actor.actor_definition_id = :connector_definition_id
|
|
100
|
-
|
|
102
|
+
LIMIT :limit
|
|
103
|
+
"""
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# Query connections by connector type, filtered by organization
|
|
107
|
+
SELECT_CONNECTIONS_BY_CONNECTOR_AND_ORG = sqlalchemy.text(
|
|
108
|
+
"""
|
|
109
|
+
SELECT
|
|
110
|
+
connection.id AS connection_id,
|
|
111
|
+
connection.name AS connection_name,
|
|
112
|
+
connection.source_id,
|
|
113
|
+
workspace.id AS workspace_id,
|
|
114
|
+
workspace.name AS workspace_name,
|
|
115
|
+
workspace.organization_id,
|
|
116
|
+
workspace.dataplane_group_id,
|
|
117
|
+
dataplane_group.name AS dataplane_name,
|
|
118
|
+
source_actor.actor_definition_id AS source_definition_id,
|
|
119
|
+
source_actor.name AS source_name
|
|
120
|
+
FROM connection
|
|
121
|
+
JOIN actor AS source_actor
|
|
122
|
+
ON connection.source_id = source_actor.id
|
|
123
|
+
JOIN workspace
|
|
124
|
+
ON source_actor.workspace_id = workspace.id
|
|
125
|
+
LEFT JOIN dataplane_group
|
|
126
|
+
ON workspace.dataplane_group_id = dataplane_group.id
|
|
127
|
+
WHERE
|
|
128
|
+
source_actor.actor_definition_id = :connector_definition_id
|
|
129
|
+
AND workspace.organization_id = :organization_id
|
|
101
130
|
LIMIT :limit
|
|
102
131
|
"""
|
|
103
132
|
)
|
|
@@ -276,6 +305,66 @@ SELECT_SUCCESSFUL_SYNCS_FOR_VERSION = sqlalchemy.text(
|
|
|
276
305
|
"""
|
|
277
306
|
)
|
|
278
307
|
|
|
308
|
+
# Get failed attempt results for actors pinned to a specific connector definition VERSION ID
|
|
309
|
+
# Includes attempt details (failure_summary, etc.) for research/debugging
|
|
310
|
+
# Filters on attempts.status = 'failed' to capture all failed attempts, including those
|
|
311
|
+
# from jobs that eventually succeeded via retry. The latest_job_attempt_status field
|
|
312
|
+
# indicates whether the job eventually succeeded or remained failed.
|
|
313
|
+
# Query starts from attempts table to leverage indexed columns (ended_at, status)
|
|
314
|
+
# Note: attempts.ended_at and attempts.status are indexed (btree)
|
|
315
|
+
SELECT_FAILED_SYNC_ATTEMPTS_FOR_VERSION = sqlalchemy.text(
|
|
316
|
+
"""
|
|
317
|
+
SELECT
|
|
318
|
+
jobs.id AS job_id,
|
|
319
|
+
jobs.scope AS connection_id,
|
|
320
|
+
jobs.status AS latest_job_attempt_status,
|
|
321
|
+
jobs.started_at AS job_started_at,
|
|
322
|
+
jobs.updated_at AS job_updated_at,
|
|
323
|
+
connection.name AS connection_name,
|
|
324
|
+
actor.id AS actor_id,
|
|
325
|
+
actor.name AS actor_name,
|
|
326
|
+
actor.actor_definition_id,
|
|
327
|
+
scoped_configuration.origin_type AS pin_origin_type,
|
|
328
|
+
scoped_configuration.origin AS pin_origin,
|
|
329
|
+
workspace.id AS workspace_id,
|
|
330
|
+
workspace.name AS workspace_name,
|
|
331
|
+
workspace.organization_id,
|
|
332
|
+
workspace.dataplane_group_id,
|
|
333
|
+
dataplane_group.name AS dataplane_name,
|
|
334
|
+
attempts.id AS failed_attempt_id,
|
|
335
|
+
attempts.attempt_number AS failed_attempt_number,
|
|
336
|
+
attempts.status AS failed_attempt_status,
|
|
337
|
+
attempts.created_at AS failed_attempt_created_at,
|
|
338
|
+
attempts.ended_at AS failed_attempt_ended_at,
|
|
339
|
+
attempts.failure_summary,
|
|
340
|
+
attempts.processing_task_queue
|
|
341
|
+
FROM attempts
|
|
342
|
+
JOIN jobs
|
|
343
|
+
ON jobs.id = attempts.job_id
|
|
344
|
+
AND jobs.config_type = 'sync'
|
|
345
|
+
AND jobs.updated_at >= :cutoff_date
|
|
346
|
+
JOIN connection
|
|
347
|
+
ON jobs.scope = connection.id::text
|
|
348
|
+
JOIN actor
|
|
349
|
+
ON connection.source_id = actor.id
|
|
350
|
+
JOIN scoped_configuration
|
|
351
|
+
ON scoped_configuration.scope_id = actor.id
|
|
352
|
+
AND scoped_configuration.key = 'connector_version'
|
|
353
|
+
AND scoped_configuration.scope_type = 'actor'
|
|
354
|
+
AND scoped_configuration.value = :actor_definition_version_id
|
|
355
|
+
JOIN workspace
|
|
356
|
+
ON actor.workspace_id = workspace.id
|
|
357
|
+
LEFT JOIN dataplane_group
|
|
358
|
+
ON workspace.dataplane_group_id = dataplane_group.id
|
|
359
|
+
WHERE
|
|
360
|
+
attempts.ended_at >= :cutoff_date
|
|
361
|
+
AND attempts.status = 'failed'
|
|
362
|
+
ORDER BY
|
|
363
|
+
attempts.ended_at DESC
|
|
364
|
+
LIMIT :limit
|
|
365
|
+
"""
|
|
366
|
+
)
|
|
367
|
+
|
|
279
368
|
# =============================================================================
|
|
280
369
|
# Dataplane and Workspace Queries
|
|
281
370
|
# =============================================================================
|
|
File without changes
|
{airbyte_internal_ops-0.1.8.dist-info → airbyte_internal_ops-0.1.10.dist-info}/entry_points.txt
RENAMED
|
File without changes
|