airbyte-internal-ops 0.1.2.post2.dev20080805740__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_internal_ops-0.1.2.post2.dev20080805740.dist-info → airbyte_internal_ops-0.1.4.dist-info}/METADATA +8 -5
- {airbyte_internal_ops-0.1.2.post2.dev20080805740.dist-info → airbyte_internal_ops-0.1.4.dist-info}/RECORD +31 -11
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/common.py +1 -1
- airbyte_ops_mcp/cli/cloud.py +309 -38
- airbyte_ops_mcp/cloud_admin/connection_config.py +131 -0
- airbyte_ops_mcp/live_tests/__init__.py +16 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/__init__.py +35 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/audit_logging.py +88 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/consts.py +33 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/db_access.py +82 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/retrieval.py +391 -0
- airbyte_ops_mcp/live_tests/_connection_retriever/secrets_resolution.py +130 -0
- airbyte_ops_mcp/live_tests/config.py +190 -0
- airbyte_ops_mcp/live_tests/connection_fetcher.py +159 -2
- airbyte_ops_mcp/live_tests/connection_secret_retriever.py +173 -0
- airbyte_ops_mcp/live_tests/evaluation_modes.py +45 -0
- airbyte_ops_mcp/live_tests/http_metrics.py +81 -0
- airbyte_ops_mcp/live_tests/message_cache/__init__.py +15 -0
- airbyte_ops_mcp/live_tests/message_cache/duckdb_cache.py +415 -0
- airbyte_ops_mcp/live_tests/obfuscation.py +126 -0
- airbyte_ops_mcp/live_tests/regression/__init__.py +29 -0
- airbyte_ops_mcp/live_tests/regression/comparators.py +466 -0
- airbyte_ops_mcp/live_tests/schema_generation.py +154 -0
- airbyte_ops_mcp/live_tests/validation/__init__.py +43 -0
- airbyte_ops_mcp/live_tests/validation/catalog_validators.py +389 -0
- airbyte_ops_mcp/live_tests/validation/record_validators.py +227 -0
- airbyte_ops_mcp/mcp/_mcp_utils.py +3 -0
- airbyte_ops_mcp/mcp/live_tests.py +500 -0
- airbyte_ops_mcp/mcp/server.py +3 -0
- {airbyte_internal_ops-0.1.2.post2.dev20080805740.dist-info → airbyte_internal_ops-0.1.4.dist-info}/WHEEL +0 -0
- {airbyte_internal_ops-0.1.2.post2.dev20080805740.dist-info → airbyte_internal_ops-0.1.4.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: airbyte-internal-ops
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: MCP and API interfaces that let the agents do the admin work
|
|
5
5
|
Author-email: Aaron Steers <aj@airbyte.io>
|
|
6
6
|
Keywords: admin,airbyte,api,mcp
|
|
@@ -10,18 +10,20 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
-
|
|
14
|
-
Requires-
|
|
15
|
-
Requires-Dist: airbyte-cdk<7.0,>=6.0.0
|
|
13
|
+
Requires-Python: <3.13,>=3.11
|
|
14
|
+
Requires-Dist: airbyte-cdk<8.0,>=7.3.9
|
|
16
15
|
Requires-Dist: airbyte-protocol-models-pdv2>=0.13.0
|
|
17
|
-
Requires-Dist: airbyte>=0.
|
|
16
|
+
Requires-Dist: airbyte>=0.34.0
|
|
18
17
|
Requires-Dist: asyncer<1.0,>=0.0.4
|
|
19
18
|
Requires-Dist: click<9.0,>=8.1.3
|
|
19
|
+
Requires-Dist: cloud-sql-python-connector[pg8000]<2.0,>=1.7.0
|
|
20
20
|
Requires-Dist: cyclopts<5.0,>=4.0.0
|
|
21
21
|
Requires-Dist: docker<7.0,>=6.0
|
|
22
22
|
Requires-Dist: dpath<3.0,>=2.1.5
|
|
23
23
|
Requires-Dist: fastmcp<3.0,>=2.12.1
|
|
24
24
|
Requires-Dist: gitpython<4.0,>=3.1.29
|
|
25
|
+
Requires-Dist: google-cloud-logging<4.0,>=3.9.0
|
|
26
|
+
Requires-Dist: google-cloud-secret-manager<3.0,>=2.18.0
|
|
25
27
|
Requires-Dist: jinja2<4.0,>=3.1.2
|
|
26
28
|
Requires-Dist: pydantic>=2.0.0
|
|
27
29
|
Requires-Dist: python-dotenv<2.0,>=1.0.0
|
|
@@ -30,6 +32,7 @@ Requires-Dist: requests<3.0,>=2.31.0
|
|
|
30
32
|
Requires-Dist: rich<14.0,>=13.0.0
|
|
31
33
|
Requires-Dist: semver<4.0,>=3.0.1
|
|
32
34
|
Requires-Dist: slack-sdk<4.0,>=3.36.0
|
|
35
|
+
Requires-Dist: sqlalchemy<3.0,>=2.0.0
|
|
33
36
|
Requires-Dist: toml<1.0,>=0.10.2
|
|
34
37
|
Description-Content-Type: text/markdown
|
|
35
38
|
|
|
@@ -138,7 +138,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/tes
|
|
|
138
138
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/context.py,sha256=ounJj-ZjLBTtZx088t6u_UKy_aHJZmMmfXskSTbWjFM,9789
|
|
139
139
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/pipeline.py,sha256=QaxeepQxl5PX5xitTt7pbt2eIkQJTgAqmmxL2nFqvvI,3020
|
|
140
140
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
141
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/common.py,sha256=
|
|
141
|
+
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/common.py,sha256=Gv9kFNgteTqGV92TgQxzlUefEi6NL0VNZy31V1l7DwM,42948
|
|
142
142
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/java_connectors.py,sha256=UOwoLiSpVdDSGFhrV__XTbJgF4eQj-MyS3tezej09qA,5774
|
|
143
143
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py,sha256=aVD-FYW_5yDryZUXIjgm-H_jJetiAfmxRwiPQV4Fnlc,9529
|
|
144
144
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/python_connectors.py,sha256=BJKZLovnfA1_vYNcEC2ko8xeD0FhSLorhccVCx7q7RY,15755
|
|
@@ -358,36 +358,56 @@ airbyte_ops_mcp/cli/__init__.py,sha256=XpL7FyVfgabfBF2JR7u7NwJ2krlYqjd_OwLcWf-Xc
|
|
|
358
358
|
airbyte_ops_mcp/cli/_base.py,sha256=I8tWnyQf0ks4r3J8N8h-5GZxyn37T-55KsbuHnxYlcg,415
|
|
359
359
|
airbyte_ops_mcp/cli/_shared.py,sha256=jg-xMyGzTCGPqKd8VTfE_3kGPIyO_3Kx5sQbG4rPc0Y,1311
|
|
360
360
|
airbyte_ops_mcp/cli/app.py,sha256=SEdBpqFUG2O8zGV5ifwptxrLGFph_dLr66-MX9d69gQ,789
|
|
361
|
-
airbyte_ops_mcp/cli/cloud.py,sha256=
|
|
361
|
+
airbyte_ops_mcp/cli/cloud.py,sha256=iEkwromM8VByNa9V-2zBnAJZfr6CzCTiYSajqu4hSok,31996
|
|
362
362
|
airbyte_ops_mcp/cli/gh.py,sha256=91b1AxFXvHQCFyXhrrym-756ZjnMCqvxFdmwCtma1zI,2046
|
|
363
363
|
airbyte_ops_mcp/cli/registry.py,sha256=wHyfiysASuy-HGvLJIiU8TRguaiuqRaXQP9QJ-LC7bk,2940
|
|
364
364
|
airbyte_ops_mcp/cli/repo.py,sha256=2LurE6AvRpdgYpiRZuvA33bI1-e7_e0QsFbEO7Bba_g,15780
|
|
365
365
|
airbyte_ops_mcp/cloud_admin/__init__.py,sha256=cqE96Q10Kp6elhH9DAi6TVsIwSUy3sooDLLrxTaktGk,816
|
|
366
366
|
airbyte_ops_mcp/cloud_admin/api_client.py,sha256=4vZv1J4S2Q8ETl6gIB20X1X6KHTVV-bx__b2Ax8oqyc,17358
|
|
367
367
|
airbyte_ops_mcp/cloud_admin/auth.py,sha256=j45pRR8fg6CLwVdn7Uu5KW_kTz_CjRP6ZJGUzqHj_Dk,2558
|
|
368
|
+
airbyte_ops_mcp/cloud_admin/connection_config.py,sha256=UtbIwuB7CA3WJr9oYRwlKDsjciqd_9ewWdml2f8DuXw,4887
|
|
368
369
|
airbyte_ops_mcp/cloud_admin/models.py,sha256=YZ3FbEW-tZa50khKTTl4Bzvy_LsGyyQd6qcpXo62jls,2670
|
|
369
|
-
airbyte_ops_mcp/live_tests/__init__.py,sha256=
|
|
370
|
+
airbyte_ops_mcp/live_tests/__init__.py,sha256=qJac67dt6DQCqif39HqeiG3Tr9xrxfP-ala8HsLZKis,1020
|
|
370
371
|
airbyte_ops_mcp/live_tests/ci_output.py,sha256=NQATOGid0OCbLEl2NOtGK4cHLL5OxXhjmtanBjIlCyE,11369
|
|
371
|
-
airbyte_ops_mcp/live_tests/
|
|
372
|
+
airbyte_ops_mcp/live_tests/config.py,sha256=dwWeY0tatdbwl9BqbhZ7EljoZDCtKmGO5fvOAIxeXmA,5873
|
|
373
|
+
airbyte_ops_mcp/live_tests/connection_fetcher.py,sha256=5wIiA0VvCFNEc-fr6Po18gZMX3E5fyPOGf2SuVOqv5U,12799
|
|
374
|
+
airbyte_ops_mcp/live_tests/connection_secret_retriever.py,sha256=XBuPfmA9GnNCHlJdDocwiB_UIx8F0D6OHBNNIDPrfk0,5869
|
|
372
375
|
airbyte_ops_mcp/live_tests/connector_runner.py,sha256=fGE_TCii9zhC3pbyBupJ3JVkuxOWB59Q1DgigcF3q04,9707
|
|
373
|
-
airbyte_ops_mcp/live_tests/
|
|
376
|
+
airbyte_ops_mcp/live_tests/evaluation_modes.py,sha256=lAL6pEDmy_XCC7_m4_NXjt_f6Z8CXeAhMkc0FU8bm_M,1364
|
|
377
|
+
airbyte_ops_mcp/live_tests/http_metrics.py,sha256=oTD7f2MnQOvx4plOxHop2bInQ0-whvuToSsrC7TIM-M,12469
|
|
374
378
|
airbyte_ops_mcp/live_tests/models.py,sha256=brtAT9oO1TwjFcP91dFcu0XcUNqQb-jf7di1zkoVEuo,8782
|
|
379
|
+
airbyte_ops_mcp/live_tests/obfuscation.py,sha256=JanpCLj6M9-_Zto6PABzNaY3OA93Frq3YpJ1411QNtQ,4395
|
|
380
|
+
airbyte_ops_mcp/live_tests/schema_generation.py,sha256=VQfn2WbsMptfjO_ub709FYSwwwvjOuWy2Jut7o5ThIs,5308
|
|
381
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/__init__.py,sha256=3Mm9Lauqf4TcdeXhujbXXitl_A7_3iUbc0iQqd6gfog,1125
|
|
382
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/audit_logging.py,sha256=DbziKNbZppb9YBly9x3WGWtPhSTkg9MIreIr7mUBkl8,2823
|
|
383
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/consts.py,sha256=k1FJk9Gyc9fWwuzh9TFUU-XpbohhbsrGC5EEJaIPy2c,1021
|
|
384
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/db_access.py,sha256=riTQ7OjwJhZqPQqjwT3WLxbx3TenI2PfgpUpuwoVsd0,2686
|
|
385
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/retrieval.py,sha256=wehmHbEoai_ab9lyobzq6ifMzacJEZv5mwJh53tOi60,12443
|
|
386
|
+
airbyte_ops_mcp/live_tests/_connection_retriever/secrets_resolution.py,sha256=TnqhzkpIaM3aHV34MMw65i1-jVdtRAk9Qqefz7DZJP8,4384
|
|
375
387
|
airbyte_ops_mcp/live_tests/commons/__init__.py,sha256=lNew_sAL4c8dPy3gMFbGC5_FuUX1P6QzGULbqS2H4M0,104
|
|
388
|
+
airbyte_ops_mcp/live_tests/message_cache/__init__.py,sha256=h6G_c73k2_OR8otg1YYV42rfKgKG7eiMIRiNxfo5fzU,545
|
|
389
|
+
airbyte_ops_mcp/live_tests/message_cache/duckdb_cache.py,sha256=LVj8sCA1pgJimT8NFQJzbsW8dEzIX5hB280sFtkP3xA,15355
|
|
390
|
+
airbyte_ops_mcp/live_tests/regression/__init__.py,sha256=SsgBKULvowwYeeS5GmrtyyVzghS9-cfZwdxskWCcRBg,851
|
|
391
|
+
airbyte_ops_mcp/live_tests/regression/comparators.py,sha256=MJkLZEKHivgrG0-3Y8BzLtnXlOb16t9z-reC9lhyObQ,15967
|
|
392
|
+
airbyte_ops_mcp/live_tests/validation/__init__.py,sha256=To16AfC5k4xwtKI4Ep8FZrdl1DexesVKe002reWwmGA,1520
|
|
393
|
+
airbyte_ops_mcp/live_tests/validation/catalog_validators.py,sha256=jqqVAMOk0mtdPgwu4d0hA0ZEjtsNh5gapvGydRv3_qk,12553
|
|
394
|
+
airbyte_ops_mcp/live_tests/validation/record_validators.py,sha256=-7Ir2LWGCrtadK2JLuBgppSyk0RFJX6Nsy0lrabtwrs,7411
|
|
376
395
|
airbyte_ops_mcp/mcp/__init__.py,sha256=QqkNkxzdXlg-W03urBAQ3zmtOKFPf35rXgO9ceUjpng,334
|
|
377
396
|
airbyte_ops_mcp/mcp/_guidance.py,sha256=48tQSnDnxqXtyGJxxgjz0ZiI814o_7Fj7f6R8jpQ7so,2375
|
|
378
|
-
airbyte_ops_mcp/mcp/_mcp_utils.py,sha256=
|
|
397
|
+
airbyte_ops_mcp/mcp/_mcp_utils.py,sha256=Im3J3yCtXPbW_Kp-oh5b11Z2187V3-Zsos5LBgdvQrc,9111
|
|
379
398
|
airbyte_ops_mcp/mcp/cloud_connector_versions.py,sha256=G8mVzhvSCmrTEqDseV57-wwL0s3oWdydgyvgMr027tU,10443
|
|
380
399
|
airbyte_ops_mcp/mcp/connector_analysis.py,sha256=OC4KrOSkMkKPkOisWnSv96BDDE5TQYHq-Jxa2vtjJpo,298
|
|
381
400
|
airbyte_ops_mcp/mcp/connector_qa.py,sha256=aImpqdnqBPDrz10BS0owsV4kuIU2XdalzgbaGZsbOL0,258
|
|
382
401
|
airbyte_ops_mcp/mcp/github.py,sha256=5ZPsSTy4-gummS96xGoG-n2RwCgyg3-UWAvmEmxd5x4,7686
|
|
383
402
|
airbyte_ops_mcp/mcp/github_repo_ops.py,sha256=D7yDtqMISFqaUzqnyA0dLE_6j6G3wHrNz8Byo8ajR8E,4929
|
|
403
|
+
airbyte_ops_mcp/mcp/live_tests.py,sha256=4KTGCfct69WTCCrxZ23l-Nrz3iKZA5tuieNM_Aa4aTs,17408
|
|
384
404
|
airbyte_ops_mcp/mcp/metadata.py,sha256=fwGW97WknR5lfKcQnFtK6dU87aA6TmLj1NkKyqDAV9g,270
|
|
385
405
|
airbyte_ops_mcp/mcp/prerelease.py,sha256=2Mr0LdCLhEc9Q7CEtmganJXHGHCLCXODKlkSapLsSsY,9484
|
|
386
406
|
airbyte_ops_mcp/mcp/prompts.py,sha256=6opN4ZweQxfSdtoK0gL6wTrlxkRvxTQvH1VTmAuhoBE,1645
|
|
387
407
|
airbyte_ops_mcp/mcp/registry.py,sha256=PW-VYUj42qx2pQ_apUkVaoUFq7VgB9zEU7-aGrkSCCw,290
|
|
388
|
-
airbyte_ops_mcp/mcp/server.py,sha256=
|
|
408
|
+
airbyte_ops_mcp/mcp/server.py,sha256=oJOrwZP7hAwx2pjChZUZYv-aA4NITWiCRiXFiKVOf8E,2843
|
|
389
409
|
airbyte_ops_mcp/mcp/server_info.py,sha256=VcNLn6CSy2UcgM2V1ep9-JGwV63p3cK1G14BFHXEn0U,2402
|
|
390
|
-
airbyte_internal_ops-0.1.
|
|
391
|
-
airbyte_internal_ops-0.1.
|
|
392
|
-
airbyte_internal_ops-0.1.
|
|
393
|
-
airbyte_internal_ops-0.1.
|
|
410
|
+
airbyte_internal_ops-0.1.4.dist-info/METADATA,sha256=5vi9tCrTgfHK0NnN7hG2jkdRsuBalql3pCVO8nVh2RE,2866
|
|
411
|
+
airbyte_internal_ops-0.1.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
412
|
+
airbyte_internal_ops-0.1.4.dist-info/entry_points.txt,sha256=eUgJ9xIy9PlR-CgRbqRMsh1NVp6jz08v9bul9vCYlU4,111
|
|
413
|
+
airbyte_internal_ops-0.1.4.dist-info/RECORD,,
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/test/steps/common.py
CHANGED
|
@@ -976,7 +976,7 @@ class LiveTests(Step):
|
|
|
976
976
|
# Add GCP credentials from the environment and point google to their location (also required for connection-retriever)
|
|
977
977
|
.with_new_file(
|
|
978
978
|
"/tmp/credentials.json",
|
|
979
|
-
contents=os.getenv("
|
|
979
|
+
contents=os.getenv("GCP_PROD_DB_ACCESS_CREDENTIALS", ""),
|
|
980
980
|
)
|
|
981
981
|
.with_env_variable(
|
|
982
982
|
"GOOGLE_APPLICATION_CREDENTIALS", "/tmp/credentials.json"
|
airbyte_ops_mcp/cli/cloud.py
CHANGED
|
@@ -7,6 +7,7 @@ Commands:
|
|
|
7
7
|
airbyte-ops cloud connector clear-version-override - Clear connector version override
|
|
8
8
|
airbyte-ops cloud connector live-test - Run live validation tests on a connector
|
|
9
9
|
airbyte-ops cloud connector regression-test - Run regression tests comparing connector versions
|
|
10
|
+
airbyte-ops cloud connector fetch-connection-config - Fetch connection config to local file
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
13
|
from __future__ import annotations
|
|
@@ -15,11 +16,15 @@ import json
|
|
|
15
16
|
from pathlib import Path
|
|
16
17
|
from typing import Annotated, Literal
|
|
17
18
|
|
|
19
|
+
from airbyte_cdk.models.connector_metadata import MetadataFile
|
|
20
|
+
from airbyte_cdk.utils.connector_paths import find_connector_root_from_name
|
|
21
|
+
from airbyte_cdk.utils.docker import build_connector_image, verify_docker_installation
|
|
18
22
|
from airbyte_protocol.models import ConfiguredAirbyteCatalog
|
|
19
23
|
from cyclopts import App, Parameter
|
|
20
24
|
|
|
21
25
|
from airbyte_ops_mcp.cli._base import app
|
|
22
26
|
from airbyte_ops_mcp.cli._shared import print_error, print_json, print_success
|
|
27
|
+
from airbyte_ops_mcp.cloud_admin.connection_config import fetch_connection_config
|
|
23
28
|
from airbyte_ops_mcp.live_tests.ci_output import (
|
|
24
29
|
generate_regression_report,
|
|
25
30
|
get_report_summary,
|
|
@@ -52,6 +57,9 @@ from airbyte_ops_mcp.mcp.cloud_connector_versions import (
|
|
|
52
57
|
set_cloud_connector_version_override,
|
|
53
58
|
)
|
|
54
59
|
|
|
60
|
+
# Path to connectors directory within the airbyte repo
|
|
61
|
+
CONNECTORS_SUBDIR = Path("airbyte-integrations") / "connectors"
|
|
62
|
+
|
|
55
63
|
# Create the cloud sub-app
|
|
56
64
|
cloud_app = App(name="cloud", help="Airbyte Cloud operations.")
|
|
57
65
|
app.command(cloud_app)
|
|
@@ -250,14 +258,83 @@ def _run_connector_command(
|
|
|
250
258
|
}
|
|
251
259
|
|
|
252
260
|
|
|
261
|
+
def _build_connector_image_from_source(
|
|
262
|
+
connector_name: str,
|
|
263
|
+
repo_root: Path | None = None,
|
|
264
|
+
tag: str = "dev",
|
|
265
|
+
) -> str | None:
|
|
266
|
+
"""Build a connector image from source code.
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
connector_name: Name of the connector (e.g., 'source-pokeapi').
|
|
270
|
+
repo_root: Optional path to the airbyte repo root. If not provided,
|
|
271
|
+
will attempt to auto-detect from current directory.
|
|
272
|
+
tag: Tag to apply to the built image (default: 'dev').
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
The full image name with tag if successful, None if build fails.
|
|
276
|
+
"""
|
|
277
|
+
if not verify_docker_installation():
|
|
278
|
+
print_error("Docker is not installed or not running")
|
|
279
|
+
return None
|
|
280
|
+
|
|
281
|
+
try:
|
|
282
|
+
connector_directory = find_connector_root_from_name(connector_name)
|
|
283
|
+
except FileNotFoundError:
|
|
284
|
+
if repo_root:
|
|
285
|
+
connector_directory = repo_root / CONNECTORS_SUBDIR / connector_name
|
|
286
|
+
if not connector_directory.exists():
|
|
287
|
+
print_error(f"Connector directory not found: {connector_directory}")
|
|
288
|
+
return None
|
|
289
|
+
else:
|
|
290
|
+
print_error(
|
|
291
|
+
f"Could not find connector '{connector_name}'. "
|
|
292
|
+
"Try providing --repo-root to specify the airbyte repo location."
|
|
293
|
+
)
|
|
294
|
+
return None
|
|
295
|
+
|
|
296
|
+
metadata_file_path = connector_directory / "metadata.yaml"
|
|
297
|
+
if not metadata_file_path.exists():
|
|
298
|
+
print_error(f"metadata.yaml not found at {metadata_file_path}")
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
metadata = MetadataFile.from_file(metadata_file_path)
|
|
302
|
+
print_success(f"Building image for connector: {connector_name}")
|
|
303
|
+
|
|
304
|
+
built_image = build_connector_image(
|
|
305
|
+
connector_name=connector_name,
|
|
306
|
+
connector_directory=connector_directory,
|
|
307
|
+
metadata=metadata,
|
|
308
|
+
tag=tag,
|
|
309
|
+
no_verify=False,
|
|
310
|
+
)
|
|
311
|
+
print_success(f"Successfully built image: {built_image}")
|
|
312
|
+
return built_image
|
|
313
|
+
|
|
314
|
+
|
|
253
315
|
@connector_app.command(name="live-test")
|
|
254
316
|
def live_test(
|
|
255
317
|
connector_image: Annotated[
|
|
256
|
-
str,
|
|
318
|
+
str | None,
|
|
257
319
|
Parameter(
|
|
258
|
-
help="Full connector image name with tag (e.g., airbyte/source-github:1.0.0)."
|
|
320
|
+
help="Full connector image name with tag (e.g., airbyte/source-github:1.0.0). "
|
|
321
|
+
"Optional if connector_name or connection_id is provided."
|
|
259
322
|
),
|
|
260
|
-
],
|
|
323
|
+
] = None,
|
|
324
|
+
connector_name: Annotated[
|
|
325
|
+
str | None,
|
|
326
|
+
Parameter(
|
|
327
|
+
help="Connector name to build from source (e.g., 'source-pokeapi'). "
|
|
328
|
+
"If provided, builds the image locally with tag 'dev'."
|
|
329
|
+
),
|
|
330
|
+
] = None,
|
|
331
|
+
repo_root: Annotated[
|
|
332
|
+
str | None,
|
|
333
|
+
Parameter(
|
|
334
|
+
help="Path to the airbyte repo root. Required if connector_name is provided "
|
|
335
|
+
"and the repo cannot be auto-detected."
|
|
336
|
+
),
|
|
337
|
+
] = None,
|
|
261
338
|
command: Annotated[
|
|
262
339
|
Literal["spec", "check", "discover", "read"],
|
|
263
340
|
Parameter(help="The Airbyte command to run."),
|
|
@@ -266,7 +343,8 @@ def live_test(
|
|
|
266
343
|
str | None,
|
|
267
344
|
Parameter(
|
|
268
345
|
help="Airbyte Cloud connection ID to fetch config/catalog from. "
|
|
269
|
-
"Mutually exclusive with config-path/catalog-path."
|
|
346
|
+
"Mutually exclusive with config-path/catalog-path. "
|
|
347
|
+
"If provided, connector_image can be auto-detected."
|
|
270
348
|
),
|
|
271
349
|
] = None,
|
|
272
350
|
config_path: Annotated[
|
|
@@ -292,6 +370,11 @@ def live_test(
|
|
|
292
370
|
and validates the output. Results are written to the output directory and
|
|
293
371
|
to GitHub Actions outputs if running in CI.
|
|
294
372
|
|
|
373
|
+
You can provide the connector image in three ways:
|
|
374
|
+
1. --connector-image: Use a pre-built image from Docker registry
|
|
375
|
+
2. --connector-name: Build the image locally from source code
|
|
376
|
+
3. --connection-id: Auto-detect from an Airbyte Cloud connection
|
|
377
|
+
|
|
295
378
|
You can provide config/catalog either via file paths OR via a connection_id
|
|
296
379
|
that fetches them from Airbyte Cloud.
|
|
297
380
|
"""
|
|
@@ -300,15 +383,32 @@ def live_test(
|
|
|
300
383
|
|
|
301
384
|
cmd = Command(command)
|
|
302
385
|
|
|
303
|
-
if not ensure_image_available(connector_image):
|
|
304
|
-
print_error(f"Failed to pull connector image: {connector_image}")
|
|
305
|
-
write_github_output("success", False)
|
|
306
|
-
write_github_output("error", f"Failed to pull image: {connector_image}")
|
|
307
|
-
return
|
|
308
|
-
|
|
309
386
|
config_file: Path | None = None
|
|
310
387
|
catalog_file: Path | None = None
|
|
311
388
|
state_file = Path(state_path) if state_path else None
|
|
389
|
+
resolved_connector_image: str | None = connector_image
|
|
390
|
+
|
|
391
|
+
# If connector_name is provided, build the image from source
|
|
392
|
+
if connector_name:
|
|
393
|
+
if connector_image:
|
|
394
|
+
print_error("Cannot specify both connector_image and connector_name")
|
|
395
|
+
write_github_output("success", False)
|
|
396
|
+
write_github_output(
|
|
397
|
+
"error", "Cannot specify both connector_image and connector_name"
|
|
398
|
+
)
|
|
399
|
+
return
|
|
400
|
+
|
|
401
|
+
repo_root_path = Path(repo_root) if repo_root else None
|
|
402
|
+
built_image = _build_connector_image_from_source(
|
|
403
|
+
connector_name=connector_name,
|
|
404
|
+
repo_root=repo_root_path,
|
|
405
|
+
tag="dev",
|
|
406
|
+
)
|
|
407
|
+
if not built_image:
|
|
408
|
+
write_github_output("success", False)
|
|
409
|
+
write_github_output("error", f"Failed to build image for {connector_name}")
|
|
410
|
+
return
|
|
411
|
+
resolved_connector_image = built_image
|
|
312
412
|
|
|
313
413
|
if connection_id:
|
|
314
414
|
if config_path or catalog_path:
|
|
@@ -330,12 +430,37 @@ def live_test(
|
|
|
330
430
|
f"Fetched config for source: {connection_data.source_name} "
|
|
331
431
|
f"with {len(connection_data.stream_names)} streams"
|
|
332
432
|
)
|
|
433
|
+
|
|
434
|
+
if not resolved_connector_image and connection_data.connector_image:
|
|
435
|
+
resolved_connector_image = connection_data.connector_image
|
|
436
|
+
print_success(f"Auto-detected connector image: {resolved_connector_image}")
|
|
333
437
|
else:
|
|
334
438
|
config_file = Path(config_path) if config_path else None
|
|
335
439
|
catalog_file = Path(catalog_path) if catalog_path else None
|
|
336
440
|
|
|
441
|
+
if not resolved_connector_image:
|
|
442
|
+
print_error(
|
|
443
|
+
"You must provide one of the following: a connector_image, a connector_name, "
|
|
444
|
+
"or a connection_id for a connection that has an associated connector image. "
|
|
445
|
+
"If using connection_id, ensure the connection has a connector image configured."
|
|
446
|
+
)
|
|
447
|
+
write_github_output("success", False)
|
|
448
|
+
write_github_output("error", "Missing connector image")
|
|
449
|
+
return
|
|
450
|
+
|
|
451
|
+
# If connector_name was provided, we just built the image locally and it is already
|
|
452
|
+
# available in Docker, so we skip the image availability check/pull. Only try to pull
|
|
453
|
+
# if we didn't just build it (i.e., using a pre-built image from registry).
|
|
454
|
+
if not connector_name and not ensure_image_available(resolved_connector_image):
|
|
455
|
+
print_error(f"Failed to pull connector image: {resolved_connector_image}")
|
|
456
|
+
write_github_output("success", False)
|
|
457
|
+
write_github_output(
|
|
458
|
+
"error", f"Failed to pull image: {resolved_connector_image}"
|
|
459
|
+
)
|
|
460
|
+
return
|
|
461
|
+
|
|
337
462
|
result = _run_connector_command(
|
|
338
|
-
connector_image=
|
|
463
|
+
connector_image=resolved_connector_image,
|
|
339
464
|
command=cmd,
|
|
340
465
|
output_dir=output_path,
|
|
341
466
|
target_or_control=TargetOrControl.TARGET,
|
|
@@ -349,14 +474,14 @@ def live_test(
|
|
|
349
474
|
write_github_outputs(
|
|
350
475
|
{
|
|
351
476
|
"success": result["success"],
|
|
352
|
-
"connector":
|
|
477
|
+
"connector": resolved_connector_image,
|
|
353
478
|
"command": command,
|
|
354
479
|
"exit_code": result["exit_code"],
|
|
355
480
|
}
|
|
356
481
|
)
|
|
357
482
|
|
|
358
483
|
write_test_summary(
|
|
359
|
-
connector_image=
|
|
484
|
+
connector_image=resolved_connector_image,
|
|
360
485
|
test_type="live-test",
|
|
361
486
|
success=result["success"],
|
|
362
487
|
results={
|
|
@@ -367,9 +492,9 @@ def live_test(
|
|
|
367
492
|
)
|
|
368
493
|
|
|
369
494
|
if result["success"]:
|
|
370
|
-
print_success(f"Live test passed for {
|
|
495
|
+
print_success(f"Live test passed for {resolved_connector_image}")
|
|
371
496
|
else:
|
|
372
|
-
print_error(f"Live test failed for {
|
|
497
|
+
print_error(f"Live test failed for {resolved_connector_image}")
|
|
373
498
|
|
|
374
499
|
|
|
375
500
|
def _run_with_optional_http_metrics(
|
|
@@ -451,17 +576,33 @@ def _run_with_optional_http_metrics(
|
|
|
451
576
|
@connector_app.command(name="regression-test")
|
|
452
577
|
def regression_test(
|
|
453
578
|
target_image: Annotated[
|
|
454
|
-
str,
|
|
579
|
+
str | None,
|
|
455
580
|
Parameter(
|
|
456
|
-
help="Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0)."
|
|
581
|
+
help="Target connector image (new version) with tag (e.g., airbyte/source-github:2.0.0). "
|
|
582
|
+
"Optional if connector_name is provided."
|
|
457
583
|
),
|
|
458
|
-
],
|
|
584
|
+
] = None,
|
|
459
585
|
control_image: Annotated[
|
|
460
|
-
str,
|
|
586
|
+
str | None,
|
|
461
587
|
Parameter(
|
|
462
|
-
help="Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0)."
|
|
588
|
+
help="Control connector image (baseline version) with tag (e.g., airbyte/source-github:1.0.0). "
|
|
589
|
+
"Optional if connection_id is provided (auto-detected from connection)."
|
|
463
590
|
),
|
|
464
|
-
],
|
|
591
|
+
] = None,
|
|
592
|
+
connector_name: Annotated[
|
|
593
|
+
str | None,
|
|
594
|
+
Parameter(
|
|
595
|
+
help="Connector name to build target image from source (e.g., 'source-pokeapi'). "
|
|
596
|
+
"If provided, builds the target image locally with tag 'dev'."
|
|
597
|
+
),
|
|
598
|
+
] = None,
|
|
599
|
+
repo_root: Annotated[
|
|
600
|
+
str | None,
|
|
601
|
+
Parameter(
|
|
602
|
+
help="Path to the airbyte repo root. Required if connector_name is provided "
|
|
603
|
+
"and the repo cannot be auto-detected."
|
|
604
|
+
),
|
|
605
|
+
] = None,
|
|
465
606
|
command: Annotated[
|
|
466
607
|
Literal["spec", "check", "discover", "read"],
|
|
467
608
|
Parameter(help="The Airbyte command to run."),
|
|
@@ -470,7 +611,8 @@ def regression_test(
|
|
|
470
611
|
str | None,
|
|
471
612
|
Parameter(
|
|
472
613
|
help="Airbyte Cloud connection ID to fetch config/catalog from. "
|
|
473
|
-
"Mutually exclusive with config-path/catalog-path."
|
|
614
|
+
"Mutually exclusive with config-path/catalog-path. "
|
|
615
|
+
"If provided, control_image can be auto-detected."
|
|
474
616
|
),
|
|
475
617
|
] = None,
|
|
476
618
|
config_path: Annotated[
|
|
@@ -506,6 +648,14 @@ def regression_test(
|
|
|
506
648
|
Results are written to the output directory and to GitHub Actions outputs
|
|
507
649
|
if running in CI.
|
|
508
650
|
|
|
651
|
+
You can provide the target image in two ways:
|
|
652
|
+
1. --target-image: Use a pre-built image from Docker registry
|
|
653
|
+
2. --connector-name: Build the target image locally from source code
|
|
654
|
+
|
|
655
|
+
You can provide the control image in two ways:
|
|
656
|
+
1. --control-image: Use a pre-built image from Docker registry
|
|
657
|
+
2. --connection-id: Auto-detect from an Airbyte Cloud connection
|
|
658
|
+
|
|
509
659
|
You can provide config/catalog either via file paths OR via a connection_id
|
|
510
660
|
that fetches them from Airbyte Cloud.
|
|
511
661
|
"""
|
|
@@ -514,16 +664,33 @@ def regression_test(
|
|
|
514
664
|
|
|
515
665
|
cmd = Command(command)
|
|
516
666
|
|
|
517
|
-
for image in [target_image, control_image]:
|
|
518
|
-
if not ensure_image_available(image):
|
|
519
|
-
print_error(f"Failed to pull connector image: {image}")
|
|
520
|
-
write_github_output("success", False)
|
|
521
|
-
write_github_output("error", f"Failed to pull image: {image}")
|
|
522
|
-
return
|
|
523
|
-
|
|
524
667
|
config_file: Path | None = None
|
|
525
668
|
catalog_file: Path | None = None
|
|
526
669
|
state_file = Path(state_path) if state_path else None
|
|
670
|
+
resolved_target_image: str | None = target_image
|
|
671
|
+
resolved_control_image: str | None = control_image
|
|
672
|
+
|
|
673
|
+
# If connector_name is provided, build the target image from source
|
|
674
|
+
if connector_name:
|
|
675
|
+
if target_image:
|
|
676
|
+
print_error("Cannot specify both target_image and connector_name")
|
|
677
|
+
write_github_output("success", False)
|
|
678
|
+
write_github_output(
|
|
679
|
+
"error", "Cannot specify both target_image and connector_name"
|
|
680
|
+
)
|
|
681
|
+
return
|
|
682
|
+
|
|
683
|
+
repo_root_path = Path(repo_root) if repo_root else None
|
|
684
|
+
built_image = _build_connector_image_from_source(
|
|
685
|
+
connector_name=connector_name,
|
|
686
|
+
repo_root=repo_root_path,
|
|
687
|
+
tag="dev",
|
|
688
|
+
)
|
|
689
|
+
if not built_image:
|
|
690
|
+
write_github_output("success", False)
|
|
691
|
+
write_github_output("error", f"Failed to build image for {connector_name}")
|
|
692
|
+
return
|
|
693
|
+
resolved_target_image = built_image
|
|
527
694
|
|
|
528
695
|
if connection_id:
|
|
529
696
|
if config_path or catalog_path:
|
|
@@ -545,15 +712,53 @@ def regression_test(
|
|
|
545
712
|
f"Fetched config for source: {connection_data.source_name} "
|
|
546
713
|
f"with {len(connection_data.stream_names)} streams"
|
|
547
714
|
)
|
|
715
|
+
|
|
716
|
+
# Auto-detect control_image from connection if not provided
|
|
717
|
+
if not resolved_control_image and connection_data.connector_image:
|
|
718
|
+
resolved_control_image = connection_data.connector_image
|
|
719
|
+
print_success(f"Auto-detected control image: {resolved_control_image}")
|
|
548
720
|
else:
|
|
549
721
|
config_file = Path(config_path) if config_path else None
|
|
550
722
|
catalog_file = Path(catalog_path) if catalog_path else None
|
|
551
723
|
|
|
724
|
+
# Validate that we have both images
|
|
725
|
+
if not resolved_target_image:
|
|
726
|
+
print_error(
|
|
727
|
+
"You must provide one of the following: a target_image or a connector_name "
|
|
728
|
+
"to build the target image from source."
|
|
729
|
+
)
|
|
730
|
+
write_github_output("success", False)
|
|
731
|
+
write_github_output("error", "No target image specified")
|
|
732
|
+
return
|
|
733
|
+
|
|
734
|
+
if not resolved_control_image:
|
|
735
|
+
print_error(
|
|
736
|
+
"You must provide one of the following: a control_image or a connection_id "
|
|
737
|
+
"for a connection that has an associated connector image."
|
|
738
|
+
)
|
|
739
|
+
write_github_output("success", False)
|
|
740
|
+
write_github_output("error", "No control image specified")
|
|
741
|
+
return
|
|
742
|
+
|
|
743
|
+
# Pull images if they weren't just built locally
|
|
744
|
+
# If connector_name was provided, we just built the target image locally
|
|
745
|
+
if not connector_name and not ensure_image_available(resolved_target_image):
|
|
746
|
+
print_error(f"Failed to pull target connector image: {resolved_target_image}")
|
|
747
|
+
write_github_output("success", False)
|
|
748
|
+
write_github_output("error", f"Failed to pull image: {resolved_target_image}")
|
|
749
|
+
return
|
|
750
|
+
|
|
751
|
+
if not ensure_image_available(resolved_control_image):
|
|
752
|
+
print_error(f"Failed to pull control connector image: {resolved_control_image}")
|
|
753
|
+
write_github_output("success", False)
|
|
754
|
+
write_github_output("error", f"Failed to pull image: {resolved_control_image}")
|
|
755
|
+
return
|
|
756
|
+
|
|
552
757
|
target_output = output_path / "target"
|
|
553
758
|
control_output = output_path / "control"
|
|
554
759
|
|
|
555
760
|
target_result = _run_with_optional_http_metrics(
|
|
556
|
-
connector_image=
|
|
761
|
+
connector_image=resolved_target_image,
|
|
557
762
|
command=cmd,
|
|
558
763
|
output_dir=target_output,
|
|
559
764
|
target_or_control=TargetOrControl.TARGET,
|
|
@@ -564,7 +769,7 @@ def regression_test(
|
|
|
564
769
|
)
|
|
565
770
|
|
|
566
771
|
control_result = _run_with_optional_http_metrics(
|
|
567
|
-
connector_image=
|
|
772
|
+
connector_image=resolved_control_image,
|
|
568
773
|
command=cmd,
|
|
569
774
|
output_dir=control_output,
|
|
570
775
|
target_or_control=TargetOrControl.CONTROL,
|
|
@@ -589,8 +794,8 @@ def regression_test(
|
|
|
589
794
|
write_github_outputs(
|
|
590
795
|
{
|
|
591
796
|
"success": both_succeeded and not regression_detected,
|
|
592
|
-
"target_image":
|
|
593
|
-
"control_image":
|
|
797
|
+
"target_image": resolved_target_image,
|
|
798
|
+
"control_image": resolved_control_image,
|
|
594
799
|
"command": command,
|
|
595
800
|
"target_exit_code": target_result["exit_code"],
|
|
596
801
|
"control_exit_code": control_result["exit_code"],
|
|
@@ -601,8 +806,8 @@ def regression_test(
|
|
|
601
806
|
write_json_output("regression_report", combined_result)
|
|
602
807
|
|
|
603
808
|
report_path = generate_regression_report(
|
|
604
|
-
target_image=
|
|
605
|
-
control_image=
|
|
809
|
+
target_image=resolved_target_image,
|
|
810
|
+
control_image=resolved_control_image,
|
|
606
811
|
command=command,
|
|
607
812
|
target_result=target_result,
|
|
608
813
|
control_result=control_result,
|
|
@@ -614,8 +819,74 @@ def regression_test(
|
|
|
614
819
|
write_github_summary(summary)
|
|
615
820
|
|
|
616
821
|
if regression_detected:
|
|
617
|
-
print_error(
|
|
822
|
+
print_error(
|
|
823
|
+
f"Regression detected between {resolved_target_image} and {resolved_control_image}"
|
|
824
|
+
)
|
|
618
825
|
elif both_succeeded:
|
|
619
|
-
print_success(
|
|
826
|
+
print_success(
|
|
827
|
+
f"Regression test passed for {resolved_target_image} vs {resolved_control_image}"
|
|
828
|
+
)
|
|
829
|
+
else:
|
|
830
|
+
print_error(
|
|
831
|
+
f"Both versions failed for {resolved_target_image} vs {resolved_control_image}"
|
|
832
|
+
)
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
@connector_app.command(name="fetch-connection-config")
|
|
836
|
+
def fetch_connection_config_cmd(
|
|
837
|
+
connection_id: Annotated[
|
|
838
|
+
str,
|
|
839
|
+
Parameter(help="The UUID of the Airbyte Cloud connection."),
|
|
840
|
+
],
|
|
841
|
+
output_path: Annotated[
|
|
842
|
+
str | None,
|
|
843
|
+
Parameter(
|
|
844
|
+
help="Path to output file or directory. "
|
|
845
|
+
"If directory, writes connection-<id>-config.json. "
|
|
846
|
+
"Default: ./connection-<id>-config.json"
|
|
847
|
+
),
|
|
848
|
+
] = None,
|
|
849
|
+
with_secrets: Annotated[
|
|
850
|
+
bool,
|
|
851
|
+
Parameter(
|
|
852
|
+
name="--with-secrets",
|
|
853
|
+
negative="--no-secrets",
|
|
854
|
+
help="If set, fetches unmasked secrets from the internal database. "
|
|
855
|
+
"Requires GCP_PROD_DB_ACCESS_CREDENTIALS env var or `gcloud auth application-default login`. "
|
|
856
|
+
"Must be used with --oc-issue-url.",
|
|
857
|
+
),
|
|
858
|
+
] = False,
|
|
859
|
+
oc_issue_url: Annotated[
|
|
860
|
+
str | None,
|
|
861
|
+
Parameter(
|
|
862
|
+
help="OC issue URL for audit logging. Required when using --with-secrets."
|
|
863
|
+
),
|
|
864
|
+
] = None,
|
|
865
|
+
) -> None:
|
|
866
|
+
"""Fetch connection configuration from Airbyte Cloud to a local file.
|
|
867
|
+
|
|
868
|
+
This command retrieves the source configuration for a given connection ID
|
|
869
|
+
and writes it to a local JSON file.
|
|
870
|
+
|
|
871
|
+
Requires authentication via AIRBYTE_CLOUD_CLIENT_ID and
|
|
872
|
+
AIRBYTE_CLOUD_CLIENT_SECRET environment variables.
|
|
873
|
+
|
|
874
|
+
When --with-secrets is specified, the command fetches unmasked secrets from
|
|
875
|
+
the internal database using the connection-retriever. This additionally requires:
|
|
876
|
+
- An OC issue URL for audit logging (--oc-issue-url)
|
|
877
|
+
- GCP credentials via `GCP_PROD_DB_ACCESS_CREDENTIALS` env var or `gcloud auth application-default login`
|
|
878
|
+
- If `CI=true`: expects `cloud-sql-proxy` running on localhost, or
|
|
879
|
+
direct network access to the Cloud SQL instance.
|
|
880
|
+
"""
|
|
881
|
+
path = Path(output_path) if output_path else None
|
|
882
|
+
result = fetch_connection_config(
|
|
883
|
+
connection_id=connection_id,
|
|
884
|
+
output_path=path,
|
|
885
|
+
with_secrets=with_secrets,
|
|
886
|
+
oc_issue_url=oc_issue_url,
|
|
887
|
+
)
|
|
888
|
+
if result.success:
|
|
889
|
+
print_success(result.message)
|
|
620
890
|
else:
|
|
621
|
-
print_error(
|
|
891
|
+
print_error(result.message)
|
|
892
|
+
print_json(result.model_dump())
|