airbyte-internal-ops 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_internal_ops-0.1.4.dist-info → airbyte_internal_ops-0.1.6.dist-info}/METADATA +70 -1
- {airbyte_internal_ops-0.1.4.dist-info → airbyte_internal_ops-0.1.6.dist-info}/RECORD +30 -31
- airbyte_ops_mcp/__init__.py +30 -2
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pipeline.py +2 -8
- airbyte_ops_mcp/airbyte_repo/list_connectors.py +176 -4
- airbyte_ops_mcp/airbyte_repo/utils.py +5 -3
- airbyte_ops_mcp/cli/cloud.py +35 -36
- airbyte_ops_mcp/cli/registry.py +90 -1
- airbyte_ops_mcp/cli/repo.py +15 -0
- airbyte_ops_mcp/connection_config_retriever/__init__.py +26 -0
- airbyte_ops_mcp/{live_tests/_connection_retriever → connection_config_retriever}/audit_logging.py +5 -6
- airbyte_ops_mcp/{live_tests/_connection_retriever → connection_config_retriever}/retrieval.py +8 -22
- airbyte_ops_mcp/{live_tests/_connection_retriever → connection_config_retriever}/secrets_resolution.py +8 -42
- airbyte_ops_mcp/constants.py +35 -0
- airbyte_ops_mcp/live_tests/connection_secret_retriever.py +1 -1
- airbyte_ops_mcp/mcp/github_repo_ops.py +10 -0
- airbyte_ops_mcp/mcp/live_tests.py +21 -6
- airbyte_ops_mcp/mcp/prod_db_queries.py +357 -0
- airbyte_ops_mcp/mcp/server.py +2 -0
- airbyte_ops_mcp/mcp/server_info.py +2 -2
- airbyte_ops_mcp/prod_db_access/__init__.py +34 -0
- airbyte_ops_mcp/prod_db_access/db_engine.py +127 -0
- airbyte_ops_mcp/prod_db_access/py.typed +0 -0
- airbyte_ops_mcp/prod_db_access/queries.py +272 -0
- airbyte_ops_mcp/prod_db_access/sql.py +353 -0
- airbyte_ops_mcp/registry/__init__.py +34 -0
- airbyte_ops_mcp/registry/models.py +63 -0
- airbyte_ops_mcp/registry/publish.py +368 -0
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/__init__.py +0 -3
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/commands.py +0 -242
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/context.py +0 -175
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/pipeline.py +0 -1056
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/__init__.py +0 -3
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/commands.py +0 -127
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/python_registry.py +0 -238
- airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/python_registry_publish.py +0 -119
- airbyte_ops_mcp/live_tests/_connection_retriever/__init__.py +0 -35
- airbyte_ops_mcp/live_tests/_connection_retriever/consts.py +0 -33
- airbyte_ops_mcp/live_tests/_connection_retriever/db_access.py +0 -82
- {airbyte_internal_ops-0.1.4.dist-info → airbyte_internal_ops-0.1.6.dist-info}/WHEEL +0 -0
- {airbyte_internal_ops-0.1.4.dist-info → airbyte_internal_ops-0.1.6.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: airbyte-internal-ops
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: MCP and API interfaces that let the agents do the admin work
|
|
5
5
|
Author-email: Aaron Steers <aj@airbyte.io>
|
|
6
6
|
Keywords: admin,airbyte,api,mcp
|
|
@@ -24,6 +24,7 @@ Requires-Dist: fastmcp<3.0,>=2.12.1
|
|
|
24
24
|
Requires-Dist: gitpython<4.0,>=3.1.29
|
|
25
25
|
Requires-Dist: google-cloud-logging<4.0,>=3.9.0
|
|
26
26
|
Requires-Dist: google-cloud-secret-manager<3.0,>=2.18.0
|
|
27
|
+
Requires-Dist: google-cloud-storage<3.0,>=2.8.0
|
|
27
28
|
Requires-Dist: jinja2<4.0,>=3.1.2
|
|
28
29
|
Requires-Dist: pydantic>=2.0.0
|
|
29
30
|
Requires-Dist: python-dotenv<2.0,>=1.0.0
|
|
@@ -95,3 +96,71 @@ AIRBYTE_CLOUD_TEST_WORKSPACE_ID="..."
|
|
|
95
96
|
Once configured, use the `test_my_tools` prompt by typing "/test" into your agent and selecting the auto-complete option for the `test_my_tools` prompt.
|
|
96
97
|
|
|
97
98
|
This prompt will step through all the tools, demoing their capabilities.
|
|
99
|
+
|
|
100
|
+
## Usage Examples
|
|
101
|
+
|
|
102
|
+
### Testing MCP Tools Locally
|
|
103
|
+
|
|
104
|
+
Use the `mcp-tool-test` poe task to test tools directly:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# List connectors in a repo
|
|
108
|
+
poe mcp-tool-test list_connectors_in_repo '{"repo_path": "/path/to/airbyte"}'
|
|
109
|
+
|
|
110
|
+
# Get cloud connector version
|
|
111
|
+
poe mcp-tool-test get_cloud_connector_version '{"workspace_id": "...", "actor_id": "..."}'
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Using Cloud SQL Proxy for Database Tools
|
|
115
|
+
|
|
116
|
+
Some tools (like `list_org_connections_by_source_type_db`) require access to the Airbyte Cloud Prod DB Replica. To test these locally:
|
|
117
|
+
|
|
118
|
+
1. Authenticate with GCP:
|
|
119
|
+
```bash
|
|
120
|
+
gcloud auth login
|
|
121
|
+
gcloud auth application-default login
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
2. Start Cloud SQL Proxy:
|
|
125
|
+
```bash
|
|
126
|
+
cloud-sql-proxy prod-ab-cloud-proj:us-west3:prod-pgsql-replica --port=15432
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
3. Run the tool with proxy environment variables:
|
|
130
|
+
```bash
|
|
131
|
+
USE_CLOUD_SQL_PROXY=1 DB_PORT=15432 poe mcp-tool-test list_org_connections_by_source_type_db \
|
|
132
|
+
'{"organization_id": "...", "connector_canonical_name": "source-youtube-analytics"}'
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## Instructions for Agents
|
|
136
|
+
|
|
137
|
+
When working with a user to set up and use this MCP server, agents should follow these steps for authentication:
|
|
138
|
+
|
|
139
|
+
### GCP Authentication Flow
|
|
140
|
+
|
|
141
|
+
For tools that require GCP access (database queries, secret manager, etc.), guide the user through authentication:
|
|
142
|
+
|
|
143
|
+
1. **Check if gcloud is installed.** If not, install it:
|
|
144
|
+
```bash
|
|
145
|
+
curl -sSL https://sdk.cloud.google.com | bash -s -- --disable-prompts
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
2. **Run gcloud auth login with `--no-launch-browser`** to get a verification URL:
|
|
149
|
+
```bash
|
|
150
|
+
gcloud auth login --no-launch-browser
|
|
151
|
+
```
|
|
152
|
+
Send the verification URL to the user and ask them to complete sign-in and provide the verification code.
|
|
153
|
+
|
|
154
|
+
3. **Set up Application Default Credentials (ADC)** for library access:
|
|
155
|
+
```bash
|
|
156
|
+
gcloud auth application-default login --no-launch-browser
|
|
157
|
+
```
|
|
158
|
+
Again, send the URL to the user and collect the verification code.
|
|
159
|
+
|
|
160
|
+
4. **For database tools**, start Cloud SQL Proxy on an available port and use the `USE_CLOUD_SQL_PROXY` and `DB_PORT` environment variables.
|
|
161
|
+
|
|
162
|
+
### Airbyte Cloud Authentication
|
|
163
|
+
|
|
164
|
+
For tools that interact with Airbyte Cloud API, ensure the user has configured:
|
|
165
|
+
- `AIRBYTE_CLOUD_CLIENT_ID` and `AIRBYTE_CLOUD_CLIENT_SECRET` in their `.env` file
|
|
166
|
+
- For admin operations: `AIRBYTE_INTERNAL_ADMIN_FLAG=airbyte.io` and `AIRBYTE_INTERNAL_ADMIN_USER`
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
airbyte_ops_mcp/__init__.py,sha256=
|
|
1
|
+
airbyte_ops_mcp/__init__.py,sha256=HhzURuYr29_UIdMrnWYaZB8ENr_kFkBdm4uqeiIW3Vw,760
|
|
2
2
|
airbyte_ops_mcp/_annotations.py,sha256=MO-SBDnbykxxHDESG7d8rviZZ4WlZgJKv0a8eBqcEzQ,1757
|
|
3
|
-
airbyte_ops_mcp/constants.py,sha256=
|
|
3
|
+
airbyte_ops_mcp/constants.py,sha256=eDO262DXtlPbWENwpk9GTllmkz9S_xT7G4qX2-BcfkE,1743
|
|
4
4
|
airbyte_ops_mcp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
airbyte_ops_mcp/_legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
airbyte_ops_mcp/_legacy/airbyte_ci/README.md,sha256=qEYx4geDR8AEDjrcA303h7Nol-CMDLojxUyiGzQprM8,236
|
|
@@ -106,7 +106,7 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/__i
|
|
|
106
106
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/commands.py,sha256=omSpuwd7t8O-BuJI0YRTncfP1_r1n8bzFsGY_1ok11M,12294
|
|
107
107
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/consts.py,sha256=DGw8amiCwBcnFEtv1GmeNM-SQx04QRuITiifulA5bsM,1677
|
|
108
108
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/context.py,sha256=F1qKYh38JoA1k_BwL_Mt1LL82ErzJdGfJsrFHuLwoD0,13217
|
|
109
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pipeline.py,sha256=
|
|
109
|
+
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pipeline.py,sha256=z5zef7RGdhFRDr5IdvxS57GSagwafv8hpW8U-Z-jaUQ,5068
|
|
110
110
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/reports.py,sha256=vl7AAX6ZxBk4KCEAJHzXqWntQ2k00abxGs5zDOWeRsU,9999
|
|
111
111
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/build_image/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
112
112
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/build_image/commands.py,sha256=W4goGOU8MqgLOdCneSGiO4uVQX4dCVEGU-UYyXekPl8,3518
|
|
@@ -126,10 +126,6 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/lis
|
|
|
126
126
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
127
127
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py,sha256=v-YHJDaR8MAT5VsguXxXbw2Gr5XHoiK9X1Sr75LkhMk,1001
|
|
128
128
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py,sha256=GAPtRw9F1R_NtN1zKfOTAYezfovvr_IfTBjFJkg0yV8,14783
|
|
129
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
130
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/commands.py,sha256=2T8xxO0z9PPjvEbyDb1maLmN5GNDtb7Wj_T_Je1aW2A,9203
|
|
131
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/context.py,sha256=95LXULM_1T-9ojfjSV8fzkJqpkmlLTWVmDlVnhZTXL8,6979
|
|
132
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/publish/pipeline.py,sha256=jVEw0HXYViHW7EVcQpOaUg64vwIFfs6Z1TFIsWtfIYY,43457
|
|
133
129
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pull_request/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
134
130
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pull_request/commands.py,sha256=ieLFxllbFHCgKwQnc196zodEN5Zo10SSH5zmEI7vZbE,2295
|
|
135
131
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/connectors/pull_request/pipeline.py,sha256=sJ6GLqJoYlEym5cshTxuqqvXD5_BFkREo-vdOJ3a8ks,3869
|
|
@@ -156,8 +152,6 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/metadata/comma
|
|
|
156
152
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/metadata/pipeline.py,sha256=J1Lk53KBshpoe11uI0VO3--xD59o5pozPx3vCKxvonI,10681
|
|
157
153
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
158
154
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/commands.py,sha256=ACHYsNDQ8kegCncZ-kb3jeN4T9if_vykwksiUDsFNpM,946
|
|
159
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
160
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/poetry/publish/commands.py,sha256=cvmaqdXKXNvqizAAdcBfm36RCfTRr3-Zdoe30BZWvJ4,4234
|
|
161
155
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
162
156
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/base_image.py,sha256=yUw6xifY9lrKj3Lv5OpJoR95haz-XPBavSBfJjeH6uU,7382
|
|
163
157
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/docker.py,sha256=inJ66IFQslQ6dSLdTOGgJow6IkjnZwzymgy36NLXduM,5235
|
|
@@ -165,7 +159,6 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/gradle.p
|
|
|
165
159
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/no_op.py,sha256=3NrvixYy092kKQ08aDFoEkc0ZGVdcdGU7qdIeJNV2dM,612
|
|
166
160
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/poetry.py,sha256=ww0M-qYzl6fBjU8a4HhZ_Lz7mEa0Ku3eBAx0eCJ6rjM,1603
|
|
167
161
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/pull_request.py,sha256=5PUJXmuZb6VgWcFJVlDZlQETeDwnwndSomo7ELg0TH8,2027
|
|
168
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/steps/python_registry.py,sha256=ozpe2t3l4zEA5oUQJB8z1-0ONCzeEY-8_KPSYVO087c,9560
|
|
169
162
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/test/__init__.py,sha256=AH4QS9r16-zwNGtTzbA9kbvZ2PaByYdw2SWYtWstoIM,708
|
|
170
163
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/test/commands.py,sha256=P5UlzG7hG4twSe7HKxWTMDb9qmh-7T4dsdTA7pyfYRk,5375
|
|
171
164
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/airbyte_ci/test/models.py,sha256=OXP-WtenvcEgU2ks5RhVxLkHbsFPmV_RwLqdiLtvA6s,1902
|
|
@@ -192,7 +185,6 @@ airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/steps.py,sha256=t7
|
|
|
192
185
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
|
193
186
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/click_pipeline_context.py,sha256=aPwt8XTzt2NOIU2ItAj0mLTZu4lmQwgwut7iOfM6lnI,5458
|
|
194
187
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/pipeline_context.py,sha256=68lfAR5NuLMqUgaHayJKkwXt-vqcucsKWq5L6PA4rqA,15640
|
|
195
|
-
airbyte_ops_mcp/_legacy/airbyte_ci/connector_pipelines/models/contexts/python_registry_publish.py,sha256=wblo574PR0ScmVQ1Loap-GLM-_jvwl7KK3j2Z6bFzro,5058
|
|
196
188
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/README.md,sha256=ajuFHew-gZPqGwoX3AAPbGvuysUyQJqTKmzvVEDJRWk,5146
|
|
197
189
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
198
190
|
airbyte_ops_mcp/_legacy/airbyte_ci/connector_qa/cli.py,sha256=-GpHJvlZDBU49XJmepBDqXlkRmf47K4GIpQYG0tVkKk,5152
|
|
@@ -352,38 +344,36 @@ airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/templates/render.py,sha256=g
|
|
|
352
344
|
airbyte_ops_mcp/_legacy/airbyte_ci/metadata_service/validators/metadata_validator.py,sha256=xh3Q7XgXcFaHS4Fa9A6anDecudaEXQX-SOetjgm5L4Q,14653
|
|
353
345
|
airbyte_ops_mcp/airbyte_repo/__init__.py,sha256=3yEpCOop1h33UuCtU7-1UB3kun2QsxtgvsfVEpuqN6s,1572
|
|
354
346
|
airbyte_ops_mcp/airbyte_repo/bump_version.py,sha256=rtXYv105BhElj7twQmy20VjNsj6vCb0Ulrzmc4Z-1cE,15384
|
|
355
|
-
airbyte_ops_mcp/airbyte_repo/list_connectors.py,sha256=
|
|
356
|
-
airbyte_ops_mcp/airbyte_repo/utils.py,sha256=
|
|
347
|
+
airbyte_ops_mcp/airbyte_repo/list_connectors.py,sha256=UI4RKMfFWOXHBtvh7TIw1v0BnV4rQQGuUgh8e9yvpos,19907
|
|
348
|
+
airbyte_ops_mcp/airbyte_repo/utils.py,sha256=TXlOAfhiu_hVRNjCxB4PRPVDhTWCU5lYmgqz4QG_-EA,3201
|
|
357
349
|
airbyte_ops_mcp/cli/__init__.py,sha256=XpL7FyVfgabfBF2JR7u7NwJ2krlYqjd_OwLcWf-Xc7s,114
|
|
358
350
|
airbyte_ops_mcp/cli/_base.py,sha256=I8tWnyQf0ks4r3J8N8h-5GZxyn37T-55KsbuHnxYlcg,415
|
|
359
351
|
airbyte_ops_mcp/cli/_shared.py,sha256=jg-xMyGzTCGPqKd8VTfE_3kGPIyO_3Kx5sQbG4rPc0Y,1311
|
|
360
352
|
airbyte_ops_mcp/cli/app.py,sha256=SEdBpqFUG2O8zGV5ifwptxrLGFph_dLr66-MX9d69gQ,789
|
|
361
|
-
airbyte_ops_mcp/cli/cloud.py,sha256=
|
|
353
|
+
airbyte_ops_mcp/cli/cloud.py,sha256=BMFYs5bTEgdOhxwzBrtSyYMKaHhXnMM_SGzK2hFDPBY,32076
|
|
362
354
|
airbyte_ops_mcp/cli/gh.py,sha256=91b1AxFXvHQCFyXhrrym-756ZjnMCqvxFdmwCtma1zI,2046
|
|
363
|
-
airbyte_ops_mcp/cli/registry.py,sha256=
|
|
364
|
-
airbyte_ops_mcp/cli/repo.py,sha256=
|
|
355
|
+
airbyte_ops_mcp/cli/registry.py,sha256=OpON1p4_A-G-FSfIpr6UlKYTjcj_zyiprKOu7qxwuhc,5787
|
|
356
|
+
airbyte_ops_mcp/cli/repo.py,sha256=G1hoQpH0XYhUH3FFOsia9xabGB0LP9o3XcwBuqvFVo0,16331
|
|
365
357
|
airbyte_ops_mcp/cloud_admin/__init__.py,sha256=cqE96Q10Kp6elhH9DAi6TVsIwSUy3sooDLLrxTaktGk,816
|
|
366
358
|
airbyte_ops_mcp/cloud_admin/api_client.py,sha256=4vZv1J4S2Q8ETl6gIB20X1X6KHTVV-bx__b2Ax8oqyc,17358
|
|
367
359
|
airbyte_ops_mcp/cloud_admin/auth.py,sha256=j45pRR8fg6CLwVdn7Uu5KW_kTz_CjRP6ZJGUzqHj_Dk,2558
|
|
368
360
|
airbyte_ops_mcp/cloud_admin/connection_config.py,sha256=UtbIwuB7CA3WJr9oYRwlKDsjciqd_9ewWdml2f8DuXw,4887
|
|
369
361
|
airbyte_ops_mcp/cloud_admin/models.py,sha256=YZ3FbEW-tZa50khKTTl4Bzvy_LsGyyQd6qcpXo62jls,2670
|
|
362
|
+
airbyte_ops_mcp/connection_config_retriever/__init__.py,sha256=Xoi-YvARrNPhECdpwEDDkdwEpnvj8zuUlwULpf4iRrU,800
|
|
363
|
+
airbyte_ops_mcp/connection_config_retriever/audit_logging.py,sha256=GjT4dVa0TtvGDmiBz9qwzcYCnSf9hTo7UM6l7ubUNE8,2846
|
|
364
|
+
airbyte_ops_mcp/connection_config_retriever/retrieval.py,sha256=Bpbl1qoR0z1F8dgQOj-dB95UxqzGTXbkW4nHMPBIO-g,12112
|
|
365
|
+
airbyte_ops_mcp/connection_config_retriever/secrets_resolution.py,sha256=12g0lZzhCzAPl4Iv4eMW6d76mvXjIBGspOnNhywzks4,3644
|
|
370
366
|
airbyte_ops_mcp/live_tests/__init__.py,sha256=qJac67dt6DQCqif39HqeiG3Tr9xrxfP-ala8HsLZKis,1020
|
|
371
367
|
airbyte_ops_mcp/live_tests/ci_output.py,sha256=NQATOGid0OCbLEl2NOtGK4cHLL5OxXhjmtanBjIlCyE,11369
|
|
372
368
|
airbyte_ops_mcp/live_tests/config.py,sha256=dwWeY0tatdbwl9BqbhZ7EljoZDCtKmGO5fvOAIxeXmA,5873
|
|
373
369
|
airbyte_ops_mcp/live_tests/connection_fetcher.py,sha256=5wIiA0VvCFNEc-fr6Po18gZMX3E5fyPOGf2SuVOqv5U,12799
|
|
374
|
-
airbyte_ops_mcp/live_tests/connection_secret_retriever.py,sha256=
|
|
370
|
+
airbyte_ops_mcp/live_tests/connection_secret_retriever.py,sha256=x8JNG2byxnS5xNLnFK10LzsqeY9FU8ZECyReh1WOf0s,5864
|
|
375
371
|
airbyte_ops_mcp/live_tests/connector_runner.py,sha256=fGE_TCii9zhC3pbyBupJ3JVkuxOWB59Q1DgigcF3q04,9707
|
|
376
372
|
airbyte_ops_mcp/live_tests/evaluation_modes.py,sha256=lAL6pEDmy_XCC7_m4_NXjt_f6Z8CXeAhMkc0FU8bm_M,1364
|
|
377
373
|
airbyte_ops_mcp/live_tests/http_metrics.py,sha256=oTD7f2MnQOvx4plOxHop2bInQ0-whvuToSsrC7TIM-M,12469
|
|
378
374
|
airbyte_ops_mcp/live_tests/models.py,sha256=brtAT9oO1TwjFcP91dFcu0XcUNqQb-jf7di1zkoVEuo,8782
|
|
379
375
|
airbyte_ops_mcp/live_tests/obfuscation.py,sha256=JanpCLj6M9-_Zto6PABzNaY3OA93Frq3YpJ1411QNtQ,4395
|
|
380
376
|
airbyte_ops_mcp/live_tests/schema_generation.py,sha256=VQfn2WbsMptfjO_ub709FYSwwwvjOuWy2Jut7o5ThIs,5308
|
|
381
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/__init__.py,sha256=3Mm9Lauqf4TcdeXhujbXXitl_A7_3iUbc0iQqd6gfog,1125
|
|
382
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/audit_logging.py,sha256=DbziKNbZppb9YBly9x3WGWtPhSTkg9MIreIr7mUBkl8,2823
|
|
383
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/consts.py,sha256=k1FJk9Gyc9fWwuzh9TFUU-XpbohhbsrGC5EEJaIPy2c,1021
|
|
384
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/db_access.py,sha256=riTQ7OjwJhZqPQqjwT3WLxbx3TenI2PfgpUpuwoVsd0,2686
|
|
385
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/retrieval.py,sha256=wehmHbEoai_ab9lyobzq6ifMzacJEZv5mwJh53tOi60,12443
|
|
386
|
-
airbyte_ops_mcp/live_tests/_connection_retriever/secrets_resolution.py,sha256=TnqhzkpIaM3aHV34MMw65i1-jVdtRAk9Qqefz7DZJP8,4384
|
|
387
377
|
airbyte_ops_mcp/live_tests/commons/__init__.py,sha256=lNew_sAL4c8dPy3gMFbGC5_FuUX1P6QzGULbqS2H4M0,104
|
|
388
378
|
airbyte_ops_mcp/live_tests/message_cache/__init__.py,sha256=h6G_c73k2_OR8otg1YYV42rfKgKG7eiMIRiNxfo5fzU,545
|
|
389
379
|
airbyte_ops_mcp/live_tests/message_cache/duckdb_cache.py,sha256=LVj8sCA1pgJimT8NFQJzbsW8dEzIX5hB280sFtkP3xA,15355
|
|
@@ -399,15 +389,24 @@ airbyte_ops_mcp/mcp/cloud_connector_versions.py,sha256=G8mVzhvSCmrTEqDseV57-wwL0
|
|
|
399
389
|
airbyte_ops_mcp/mcp/connector_analysis.py,sha256=OC4KrOSkMkKPkOisWnSv96BDDE5TQYHq-Jxa2vtjJpo,298
|
|
400
390
|
airbyte_ops_mcp/mcp/connector_qa.py,sha256=aImpqdnqBPDrz10BS0owsV4kuIU2XdalzgbaGZsbOL0,258
|
|
401
391
|
airbyte_ops_mcp/mcp/github.py,sha256=5ZPsSTy4-gummS96xGoG-n2RwCgyg3-UWAvmEmxd5x4,7686
|
|
402
|
-
airbyte_ops_mcp/mcp/github_repo_ops.py,sha256=
|
|
403
|
-
airbyte_ops_mcp/mcp/live_tests.py,sha256=
|
|
392
|
+
airbyte_ops_mcp/mcp/github_repo_ops.py,sha256=pE6RyATBfUKBOffQ6iyNJhHfCLjwq0pzVmmFYUg39UQ,5375
|
|
393
|
+
airbyte_ops_mcp/mcp/live_tests.py,sha256=KnxZLuUNmm_3Clt0DU8H9rJ01zOKefnL_wqdSCMDjkE,17992
|
|
404
394
|
airbyte_ops_mcp/mcp/metadata.py,sha256=fwGW97WknR5lfKcQnFtK6dU87aA6TmLj1NkKyqDAV9g,270
|
|
405
395
|
airbyte_ops_mcp/mcp/prerelease.py,sha256=2Mr0LdCLhEc9Q7CEtmganJXHGHCLCXODKlkSapLsSsY,9484
|
|
396
|
+
airbyte_ops_mcp/mcp/prod_db_queries.py,sha256=L0p7LlF7GV5mXoH_2GwgQp04sXL2QA5v-RiuLsTb-yA,12614
|
|
406
397
|
airbyte_ops_mcp/mcp/prompts.py,sha256=6opN4ZweQxfSdtoK0gL6wTrlxkRvxTQvH1VTmAuhoBE,1645
|
|
407
398
|
airbyte_ops_mcp/mcp/registry.py,sha256=PW-VYUj42qx2pQ_apUkVaoUFq7VgB9zEU7-aGrkSCCw,290
|
|
408
|
-
airbyte_ops_mcp/mcp/server.py,sha256=
|
|
409
|
-
airbyte_ops_mcp/mcp/server_info.py,sha256=
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
399
|
+
airbyte_ops_mcp/mcp/server.py,sha256=7zi91xioVTx1q-bEleekZH2c2JnbzDQt_6zxdEwnLbg,2958
|
|
400
|
+
airbyte_ops_mcp/mcp/server_info.py,sha256=4yNBA_N_vUyLwVJqp7abyFuzZkcnv6-ck_Beb2SXqTE,2426
|
|
401
|
+
airbyte_ops_mcp/prod_db_access/__init__.py,sha256=5pxouMPY1beyWlB0UwPnbaLTKTHqU6X82rbbgKY2vYU,1069
|
|
402
|
+
airbyte_ops_mcp/prod_db_access/db_engine.py,sha256=Pfc2wASTlinaRep9GwRpI9NInbFpIowJL3sLov3MyrU,4371
|
|
403
|
+
airbyte_ops_mcp/prod_db_access/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
404
|
+
airbyte_ops_mcp/prod_db_access/queries.py,sha256=bZZntlTXzvI2uSAFMkOfb_LcfCOlxyTuOddRVw4yU2M,8676
|
|
405
|
+
airbyte_ops_mcp/prod_db_access/sql.py,sha256=zHPucNuMlfxz3aU8vYo1ziiGk0lIncG9XmblEoRDd4c,12725
|
|
406
|
+
airbyte_ops_mcp/registry/__init__.py,sha256=iEaPlt9GrnlaLbc__98TguNeZG8wuQu7S-_2QkhHcbA,858
|
|
407
|
+
airbyte_ops_mcp/registry/models.py,sha256=B4L4TKr52wo0xs0CqvCBrpowqjShzVnZ5eTr2-EyhNs,2346
|
|
408
|
+
airbyte_ops_mcp/registry/publish.py,sha256=VoPxsM2_0zJ829orzCRN-kjgcJtuBNyXgW4I9J680ro,12717
|
|
409
|
+
airbyte_internal_ops-0.1.6.dist-info/METADATA,sha256=T4zoKmfLQHIBSZ0L-KIZql-OdczU1DJLB3rj3OlMJmQ,5282
|
|
410
|
+
airbyte_internal_ops-0.1.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
411
|
+
airbyte_internal_ops-0.1.6.dist-info/entry_points.txt,sha256=eUgJ9xIy9PlR-CgRbqRMsh1NVp6jz08v9bul9vCYlU4,111
|
|
412
|
+
airbyte_internal_ops-0.1.6.dist-info/RECORD,,
|
airbyte_ops_mcp/__init__.py
CHANGED
|
@@ -1,4 +1,19 @@
|
|
|
1
|
-
"""Airbyte Admin MCP - MCP and API interfaces that let the agents do the admin work.
|
|
1
|
+
"""Airbyte Admin MCP - MCP and API interfaces that let the agents do the admin work.
|
|
2
|
+
|
|
3
|
+
.. include:: ../../../README.md
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from airbyte_ops_mcp import (
|
|
7
|
+
airbyte_repo,
|
|
8
|
+
cli,
|
|
9
|
+
cloud_admin,
|
|
10
|
+
connection_config_retriever,
|
|
11
|
+
constants,
|
|
12
|
+
live_tests,
|
|
13
|
+
mcp,
|
|
14
|
+
prod_db_access,
|
|
15
|
+
registry,
|
|
16
|
+
)
|
|
2
17
|
|
|
3
18
|
__version__ = "0.1.0"
|
|
4
19
|
|
|
@@ -13,4 +28,17 @@ def get_version() -> str:
|
|
|
13
28
|
return __version__
|
|
14
29
|
|
|
15
30
|
|
|
16
|
-
__all__ = [
|
|
31
|
+
__all__ = [
|
|
32
|
+
"__version__",
|
|
33
|
+
"airbyte_repo",
|
|
34
|
+
"cli",
|
|
35
|
+
"cloud_admin",
|
|
36
|
+
"connection_config_retriever",
|
|
37
|
+
"constants",
|
|
38
|
+
"get_version",
|
|
39
|
+
"hello",
|
|
40
|
+
"live_tests",
|
|
41
|
+
"mcp",
|
|
42
|
+
"prod_db_access",
|
|
43
|
+
"registry",
|
|
44
|
+
]
|
|
@@ -16,7 +16,6 @@ from connector_ops.utils import ConnectorLanguage # type: ignore
|
|
|
16
16
|
from consts import ContextState
|
|
17
17
|
from dagger import Config
|
|
18
18
|
from pipelines.airbyte_ci.connectors.context import ConnectorContext
|
|
19
|
-
from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
|
|
20
19
|
from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
|
|
21
20
|
from pipelines.airbyte_ci.steps.no_op import NoOpStep
|
|
22
21
|
from pipelines.dagger.actions.system import docker
|
|
@@ -55,7 +54,6 @@ async def context_to_step_result(context: PipelineContext) -> StepResult:
|
|
|
55
54
|
async def run_report_complete_pipeline(
|
|
56
55
|
dagger_client: dagger.Client,
|
|
57
56
|
contexts: List[ConnectorContext]
|
|
58
|
-
| List[PublishConnectorContext]
|
|
59
57
|
| List[PipelineContext]
|
|
60
58
|
| List[ConnectorTestContext],
|
|
61
59
|
) -> None:
|
|
@@ -87,18 +85,14 @@ async def run_report_complete_pipeline(
|
|
|
87
85
|
|
|
88
86
|
|
|
89
87
|
async def run_connectors_pipelines(
|
|
90
|
-
contexts: List[ConnectorContext]
|
|
91
|
-
| List[PublishConnectorContext]
|
|
92
|
-
| List[ConnectorTestContext],
|
|
88
|
+
contexts: List[ConnectorContext] | List[ConnectorTestContext],
|
|
93
89
|
connector_pipeline: Callable,
|
|
94
90
|
pipeline_name: str,
|
|
95
91
|
concurrency: int,
|
|
96
92
|
dagger_logs_path: Optional[Path],
|
|
97
93
|
execute_timeout: Optional[int],
|
|
98
94
|
*args: Any,
|
|
99
|
-
) ->
|
|
100
|
-
List[ConnectorContext] | List[PublishConnectorContext] | List[ConnectorTestContext]
|
|
101
|
-
):
|
|
95
|
+
) -> List[ConnectorContext] | List[ConnectorTestContext]:
|
|
102
96
|
"""Run a connector pipeline for all the connector contexts."""
|
|
103
97
|
|
|
104
98
|
default_connectors_semaphore = anyio.Semaphore(concurrency)
|
|
@@ -7,15 +7,22 @@ by comparing git diffs between branches.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import logging
|
|
10
11
|
import re
|
|
11
12
|
import subprocess
|
|
12
13
|
from dataclasses import dataclass
|
|
13
14
|
from enum import StrEnum
|
|
14
15
|
from functools import lru_cache
|
|
15
16
|
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import yaml
|
|
16
20
|
|
|
17
21
|
CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
|
|
18
22
|
METADATA_FILE_NAME = "metadata.yaml"
|
|
23
|
+
GIT_DEFAULT_BRANCH = "origin/master"
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
19
26
|
|
|
20
27
|
|
|
21
28
|
class ConnectorLanguage(StrEnum):
|
|
@@ -27,9 +34,25 @@ class ConnectorLanguage(StrEnum):
|
|
|
27
34
|
MANIFEST_ONLY = "manifest-only"
|
|
28
35
|
|
|
29
36
|
|
|
37
|
+
class ConnectorType(StrEnum):
|
|
38
|
+
"""Connector types (source or destination)."""
|
|
39
|
+
|
|
40
|
+
SOURCE = "source"
|
|
41
|
+
DESTINATION = "destination"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ConnectorSubtype(StrEnum):
|
|
45
|
+
"""Connector subtypes based on data source category."""
|
|
46
|
+
|
|
47
|
+
API = "api"
|
|
48
|
+
DATABASE = "database"
|
|
49
|
+
FILE = "file"
|
|
50
|
+
CUSTOM = "custom"
|
|
51
|
+
|
|
52
|
+
|
|
30
53
|
def get_modified_connectors(
|
|
31
54
|
repo_path: str | Path,
|
|
32
|
-
base_ref: str =
|
|
55
|
+
base_ref: str = GIT_DEFAULT_BRANCH,
|
|
33
56
|
head_ref: str = "HEAD",
|
|
34
57
|
) -> list[str]:
|
|
35
58
|
"""Get list of connector IDs that have been modified.
|
|
@@ -40,14 +63,14 @@ def get_modified_connectors(
|
|
|
40
63
|
|
|
41
64
|
Args:
|
|
42
65
|
repo_path: Path to the Airbyte monorepo
|
|
43
|
-
base_ref: Base git reference to compare against (default: "origin/
|
|
66
|
+
base_ref: Base git reference to compare against (default: "origin/master")
|
|
44
67
|
head_ref: Head git reference to compare (default: "HEAD")
|
|
45
68
|
|
|
46
69
|
Returns:
|
|
47
70
|
List of connector technical names (e.g., ["source-faker", "destination-postgres"])
|
|
48
71
|
|
|
49
72
|
Example:
|
|
50
|
-
>>> connectors = get_changed_connectors("/path/to/airbyte", "origin/
|
|
73
|
+
>>> connectors = get_changed_connectors("/path/to/airbyte", "origin/master")
|
|
51
74
|
>>> print(connectors)
|
|
52
75
|
['source-faker', 'destination-postgres']
|
|
53
76
|
"""
|
|
@@ -158,6 +181,39 @@ def get_all_connectors(repo_path: str | Path) -> set[str]:
|
|
|
158
181
|
return {p.name for p in connectors_dir.iterdir() if p.is_dir()}
|
|
159
182
|
|
|
160
183
|
|
|
184
|
+
def get_connector_metadata(
|
|
185
|
+
repo_path: str | Path,
|
|
186
|
+
connector_name: str,
|
|
187
|
+
) -> dict[str, Any] | None:
|
|
188
|
+
"""Get metadata for a specific connector.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
repo_path: Path to the Airbyte monorepo
|
|
192
|
+
connector_name: Technical name of the connector (e.g., "source-faker")
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
The connector's metadata dict (the 'data' section), or None if not found
|
|
196
|
+
|
|
197
|
+
Example:
|
|
198
|
+
>>> metadata = get_connector_metadata("/path/to/airbyte", "source-faker")
|
|
199
|
+
>>> metadata.get("supportLevel")
|
|
200
|
+
'certified'
|
|
201
|
+
"""
|
|
202
|
+
repo_path = Path(repo_path)
|
|
203
|
+
connector_dir = repo_path / CONNECTOR_PATH_PREFIX / connector_name
|
|
204
|
+
metadata_file = connector_dir / METADATA_FILE_NAME
|
|
205
|
+
|
|
206
|
+
if not metadata_file.exists():
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
try:
|
|
210
|
+
with open(metadata_file) as f:
|
|
211
|
+
metadata = yaml.safe_load(f)
|
|
212
|
+
return metadata.get("data", {})
|
|
213
|
+
except Exception:
|
|
214
|
+
return None
|
|
215
|
+
|
|
216
|
+
|
|
161
217
|
def get_connectors_by_language(
|
|
162
218
|
repo_path: str | Path,
|
|
163
219
|
language: ConnectorLanguage,
|
|
@@ -205,6 +261,106 @@ def get_connectors_by_language(
|
|
|
205
261
|
return connectors_by_language
|
|
206
262
|
|
|
207
263
|
|
|
264
|
+
def get_connectors_by_type(
|
|
265
|
+
repo_path: str | Path,
|
|
266
|
+
connector_type: ConnectorType,
|
|
267
|
+
) -> set[str]:
|
|
268
|
+
"""Get set of all connector IDs for a specific type (source or destination).
|
|
269
|
+
|
|
270
|
+
This function reads connector directories to determine which connectors
|
|
271
|
+
match the specified type based on their metadata.yaml or name prefix.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
repo_path: Path to the Airbyte monorepo
|
|
275
|
+
connector_type: Type to filter by (source or destination)
|
|
276
|
+
|
|
277
|
+
Returns:
|
|
278
|
+
Set of connector technical names matching the specified type
|
|
279
|
+
|
|
280
|
+
Example:
|
|
281
|
+
>>> source_connectors = get_connectors_by_type(
|
|
282
|
+
... "/path/to/airbyte", ConnectorType.SOURCE
|
|
283
|
+
... )
|
|
284
|
+
>>> "source-postgres" in source_connectors
|
|
285
|
+
True
|
|
286
|
+
"""
|
|
287
|
+
repo_path = Path(repo_path)
|
|
288
|
+
connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
|
|
289
|
+
|
|
290
|
+
if not connectors_dir.exists():
|
|
291
|
+
raise ValueError(f"Connectors directory not found: {connectors_dir}")
|
|
292
|
+
|
|
293
|
+
type_value = connector_type.value
|
|
294
|
+
connectors_by_type = set()
|
|
295
|
+
|
|
296
|
+
for connector_dir in connectors_dir.iterdir():
|
|
297
|
+
if not connector_dir.is_dir():
|
|
298
|
+
continue
|
|
299
|
+
|
|
300
|
+
connector_name = connector_dir.name
|
|
301
|
+
|
|
302
|
+
# First try to get type from metadata.yaml
|
|
303
|
+
metadata = get_connector_metadata(repo_path, connector_name)
|
|
304
|
+
if metadata:
|
|
305
|
+
metadata_type = metadata.get("connectorType")
|
|
306
|
+
if metadata_type == type_value:
|
|
307
|
+
connectors_by_type.add(connector_name)
|
|
308
|
+
continue
|
|
309
|
+
|
|
310
|
+
# Fallback to name prefix detection
|
|
311
|
+
if connector_name.startswith(f"{type_value}-"):
|
|
312
|
+
connectors_by_type.add(connector_name)
|
|
313
|
+
|
|
314
|
+
return connectors_by_type
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def get_connectors_by_subtype(
|
|
318
|
+
repo_path: str | Path,
|
|
319
|
+
connector_subtype: ConnectorSubtype,
|
|
320
|
+
) -> set[str]:
|
|
321
|
+
"""Get set of all connector IDs for a specific subtype (api, database, file, etc.).
|
|
322
|
+
|
|
323
|
+
This function reads connector metadata.yaml files to determine which connectors
|
|
324
|
+
match the specified subtype.
|
|
325
|
+
|
|
326
|
+
Args:
|
|
327
|
+
repo_path: Path to the Airbyte monorepo
|
|
328
|
+
connector_subtype: Subtype to filter by (api, database, file, custom)
|
|
329
|
+
|
|
330
|
+
Returns:
|
|
331
|
+
Set of connector technical names matching the specified subtype
|
|
332
|
+
|
|
333
|
+
Example:
|
|
334
|
+
>>> database_connectors = get_connectors_by_subtype(
|
|
335
|
+
... "/path/to/airbyte", ConnectorSubtype.DATABASE
|
|
336
|
+
... )
|
|
337
|
+
>>> "source-postgres" in database_connectors
|
|
338
|
+
True
|
|
339
|
+
"""
|
|
340
|
+
repo_path = Path(repo_path)
|
|
341
|
+
connectors_dir = repo_path / CONNECTOR_PATH_PREFIX
|
|
342
|
+
|
|
343
|
+
if not connectors_dir.exists():
|
|
344
|
+
raise ValueError(f"Connectors directory not found: {connectors_dir}")
|
|
345
|
+
|
|
346
|
+
subtype_value = connector_subtype.value
|
|
347
|
+
connectors_by_subtype = set()
|
|
348
|
+
|
|
349
|
+
for connector_dir in connectors_dir.iterdir():
|
|
350
|
+
if not connector_dir.is_dir():
|
|
351
|
+
continue
|
|
352
|
+
|
|
353
|
+
connector_name = connector_dir.name
|
|
354
|
+
metadata = get_connector_metadata(repo_path, connector_name)
|
|
355
|
+
|
|
356
|
+
if metadata:
|
|
357
|
+
metadata_subtype = metadata.get("connectorSubtype")
|
|
358
|
+
if metadata_subtype == subtype_value:
|
|
359
|
+
connectors_by_subtype.add(connector_name)
|
|
360
|
+
|
|
361
|
+
return connectors_by_subtype
|
|
362
|
+
|
|
363
|
+
|
|
208
364
|
@lru_cache(maxsize=1024)
|
|
209
365
|
def _detect_connector_language(connector_dir: Path, connector_name: str) -> str | None:
|
|
210
366
|
"""Detect the language of a connector based on its file structure.
|
|
@@ -361,6 +517,8 @@ def list_connectors(
|
|
|
361
517
|
modified: bool | None = None,
|
|
362
518
|
language_filter: set[str] | None = None,
|
|
363
519
|
language_exclude: set[str] | None = None,
|
|
520
|
+
connector_type: str | None = None,
|
|
521
|
+
connector_subtype: str | None = None,
|
|
364
522
|
base_ref: str | None = None,
|
|
365
523
|
head_ref: str | None = None,
|
|
366
524
|
) -> ConnectorListResult:
|
|
@@ -377,6 +535,8 @@ def list_connectors(
|
|
|
377
535
|
False=not-modified only, None=all)
|
|
378
536
|
language_filter: Set of languages to include (python, java, low-code, manifest-only)
|
|
379
537
|
language_exclude: Set of languages to exclude (mutually exclusive with language_filter)
|
|
538
|
+
connector_type: Filter by connector type (source, destination, or None for all)
|
|
539
|
+
connector_subtype: Filter by connector subtype (api, database, file, custom, or None for all)
|
|
380
540
|
base_ref: Base git reference for modification detection (default: "origin/main")
|
|
381
541
|
head_ref: Head git reference for modification detection (default: "HEAD")
|
|
382
542
|
|
|
@@ -408,7 +568,7 @@ def list_connectors(
|
|
|
408
568
|
|
|
409
569
|
# Apply modified filter
|
|
410
570
|
if modified is not None:
|
|
411
|
-
base = base_ref if base_ref is not None else
|
|
571
|
+
base = base_ref if base_ref is not None else GIT_DEFAULT_BRANCH
|
|
412
572
|
head = head_ref if head_ref is not None else "HEAD"
|
|
413
573
|
changed_set = set(get_modified_connectors(repo_path, base, head))
|
|
414
574
|
if modified:
|
|
@@ -435,6 +595,18 @@ def list_connectors(
|
|
|
435
595
|
excluded = get_connectors_by_language(repo_path, ConnectorLanguage(lang))
|
|
436
596
|
result -= excluded
|
|
437
597
|
|
|
598
|
+
# Apply connector type filter
|
|
599
|
+
if connector_type is not None:
|
|
600
|
+
type_set = get_connectors_by_type(repo_path, ConnectorType(connector_type))
|
|
601
|
+
result &= type_set
|
|
602
|
+
|
|
603
|
+
# Apply connector subtype filter
|
|
604
|
+
if connector_subtype is not None:
|
|
605
|
+
subtype_set = get_connectors_by_subtype(
|
|
606
|
+
repo_path, ConnectorSubtype(connector_subtype)
|
|
607
|
+
)
|
|
608
|
+
result &= subtype_set
|
|
609
|
+
|
|
438
610
|
return ConnectorListResult(
|
|
439
611
|
connectors=sorted(result),
|
|
440
612
|
count=len(result),
|
|
@@ -81,9 +81,11 @@ def resolve_diff_range(
|
|
|
81
81
|
|
|
82
82
|
# Determine base_ref and head_ref based on PR detection
|
|
83
83
|
if pr_number is not None:
|
|
84
|
-
# PR detected - use origin/
|
|
85
|
-
#
|
|
86
|
-
|
|
84
|
+
# PR detected - use origin/{base_branch} vs HEAD (assumes CI checked out the PR)
|
|
85
|
+
# Use GITHUB_BASE_REF if available (set by GitHub Actions for PRs)
|
|
86
|
+
# This handles repos with different default branches (main, master, etc.)
|
|
87
|
+
base_branch = os.getenv("GITHUB_BASE_REF", "master")
|
|
88
|
+
base_ref = f"origin/{base_branch}"
|
|
87
89
|
head_ref = "HEAD"
|
|
88
90
|
else:
|
|
89
91
|
# No PR detected - fallback to HEAD~1 vs HEAD (post-merge use case)
|