airbyte-internal-ops 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_internal_ops-0.6.1.dist-info → airbyte_internal_ops-0.7.0.dist-info}/METADATA +4 -1
- {airbyte_internal_ops-0.6.1.dist-info → airbyte_internal_ops-0.7.0.dist-info}/RECORD +33 -30
- airbyte_ops_mcp/_sentry.py +101 -0
- airbyte_ops_mcp/cli/app.py +1 -1
- airbyte_ops_mcp/cli/{repo.py → local.py} +131 -8
- airbyte_ops_mcp/connector_ops/__init__.py +17 -0
- airbyte_ops_mcp/connector_ops/utils.py +859 -0
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/assets.py +4 -5
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/__init__.py +1 -1
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/documentation.py +23 -22
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/models.py +7 -7
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/metadata.py +15 -15
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/packaging.py +11 -9
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/security.py +16 -20
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/version.py +94 -18
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/cli.py +6 -8
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/models.py +7 -8
- airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/utils.py +2 -2
- airbyte_ops_mcp/mcp/_guidance.py +37 -0
- airbyte_ops_mcp/mcp/cloud_connector_versions.py +46 -9
- airbyte_ops_mcp/mcp/server.py +5 -0
- {airbyte_internal_ops-0.6.1.dist-info → airbyte_internal_ops-0.7.0.dist-info}/WHEEL +0 -0
- {airbyte_internal_ops-0.6.1.dist-info → airbyte_internal_ops-0.7.0.dist-info}/entry_points.txt +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/README.md +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/__init__.py +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/__init__.py +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/helpers.py +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/templates/documentation_headers_check_description.md.j2 +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/templates/section_content_description.md.j2 +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/templates/template.md.j2 +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/consts.py +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/templates/__init__.py +0 -0
- /airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/templates/qa_checks.md.j2 +0 -0
|
@@ -5,10 +5,10 @@ from __future__ import annotations
|
|
|
5
5
|
import xml.etree.ElementTree as ET
|
|
6
6
|
from typing import TYPE_CHECKING
|
|
7
7
|
|
|
8
|
-
from airbyte_ops_mcp.
|
|
8
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
9
9
|
Connector, # type: ignore
|
|
10
10
|
)
|
|
11
|
-
from airbyte_ops_mcp.
|
|
11
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
12
12
|
Check,
|
|
13
13
|
CheckCategory,
|
|
14
14
|
CheckResult,
|
|
@@ -16,7 +16,6 @@ from airbyte_ops_mcp._legacy.airbyte_ci.connector_qa.models import (
|
|
|
16
16
|
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
18
|
from pathlib import Path
|
|
19
|
-
from typing import Tuple
|
|
20
19
|
|
|
21
20
|
DEFAULT_AIRBYTE_ICON = """<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none">
|
|
22
21
|
<path fill="#e8e8ed" fill-rule="evenodd" d="M95.775 53.416c20.364-22.88 54.087-29.592 81.811-16.385 36.836 17.549 50.274 62.252 30.219 96.734l-45.115 77.487a18.994 18.994 0 0 1-11.536 8.784 19.12 19.12 0 0 1-14.412-1.878l54.62-93.829c14.55-25.027 4.818-57.467-21.888-70.24-20.038-9.583-44.533-4.795-59.336 11.685a50.008 50.008 0 0 0-12.902 32.877 49.989 49.989 0 0 0 16.664 37.87l-31.887 54.875a18.917 18.917 0 0 1-4.885 5.534 19.041 19.041 0 0 1-6.647 3.255 19.13 19.13 0 0 1-7.395.482 19.087 19.087 0 0 1-7.018-2.365l34.617-59.575a68.424 68.424 0 0 1-10.524-23.544l-21.213 36.579a18.994 18.994 0 0 1-11.535 8.784A19.123 19.123 0 0 1 33 158.668l54.856-94.356a70.296 70.296 0 0 1 7.919-10.896Zm63.314 30.034c13.211 7.577 17.774 24.427 10.13 37.54l-52.603 90.251a18.997 18.997 0 0 1-11.536 8.784 19.122 19.122 0 0 1-14.412-1.878l48.843-84.024a27.778 27.778 0 0 1-10.825-4.847 27.545 27.545 0 0 1-7.783-8.907 27.344 27.344 0 0 1-3.307-11.326 27.293 27.293 0 0 1 1.776-11.66 27.454 27.454 0 0 1 6.533-9.846 27.703 27.703 0 0 1 10.087-6.222 27.858 27.858 0 0 1 23.097 2.135Zm-19.134 16.961a8.645 8.645 0 0 0-2.232 2.529h-.003a8.565 8.565 0 0 0 .632 9.556 8.68 8.68 0 0 0 4.097 2.915 8.738 8.738 0 0 0 5.036.163 8.692 8.692 0 0 0 4.279-2.642 8.59 8.59 0 0 0 2.079-4.558 8.563 8.563 0 0 0-.821-4.938 8.645 8.645 0 0 0-3.444-3.652 8.72 8.72 0 0 0-6.586-.86 8.7 8.7 0 0 0-3.037 1.487Z" clip-rule="evenodd"/>
|
|
@@ -32,10 +31,10 @@ class CheckConnectorIconIsAvailable(AssetsCheck):
|
|
|
32
31
|
description = "Each connector must have an icon available in at the root of the connector code directory. It must be an SVG file named `icon.svg` and must be a square."
|
|
33
32
|
requires_metadata = False
|
|
34
33
|
|
|
35
|
-
def _check_is_valid_svg(self, icon_path: Path) ->
|
|
34
|
+
def _check_is_valid_svg(self, icon_path: Path) -> tuple[bool, str | None]:
|
|
36
35
|
try:
|
|
37
36
|
# Ensure the file has an .svg extension
|
|
38
|
-
if
|
|
37
|
+
if icon_path.suffix.lower() != ".svg":
|
|
39
38
|
return False, "Icon file is not a SVG file"
|
|
40
39
|
|
|
41
40
|
# Parse the file as XML
|
airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/__init__.py
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
-
from airbyte_ops_mcp.
|
|
2
|
+
from airbyte_ops_mcp.connector_qa.checks.documentation.documentation import (
|
|
3
3
|
CheckChangelogEntry,
|
|
4
4
|
CheckChangelogSectionContent,
|
|
5
5
|
CheckDocumentationExists,
|
|
@@ -3,16 +3,16 @@ import abc
|
|
|
3
3
|
import textwrap
|
|
4
4
|
from difflib import get_close_matches, ndiff
|
|
5
5
|
from threading import Thread
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import ClassVar
|
|
7
7
|
|
|
8
8
|
import requests # type: ignore
|
|
9
9
|
from pydash.objects import get # type: ignore
|
|
10
10
|
|
|
11
|
-
from airbyte_ops_mcp.
|
|
11
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
12
12
|
Connector,
|
|
13
13
|
ConnectorLanguage,
|
|
14
14
|
) # type: ignore
|
|
15
|
-
from airbyte_ops_mcp.
|
|
15
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
16
16
|
Check,
|
|
17
17
|
CheckCategory,
|
|
18
18
|
CheckResult,
|
|
@@ -37,7 +37,7 @@ class DocumentationCheck(Check):
|
|
|
37
37
|
|
|
38
38
|
class CheckMigrationGuide(DocumentationCheck):
|
|
39
39
|
name = "Breaking changes must be accompanied by a migration guide"
|
|
40
|
-
description = "When a breaking change is introduced, we check that a migration guide is available. It should be stored under `./docs/integrations/<connector-type>s/<connector-name>-migrations.md`.\nThis document should contain a section for each breaking change, in order of the version descending. It must explain users which action to take to migrate to the new version."
|
|
40
|
+
description = "When a breaking change is introduced, we check that a migration guide is available. It should be stored under `./docs/integrations/<connector-type>s/<connector-name>-migrations.md`.\nThis document should contain a section for each breaking change, in order of the version descending. It must explain users which action to take to migrate to the new version.\nSee the Breaking Changes Policy for full requirements: https://docs.airbyte.com/platform/connector-development/connector-breaking-changes"
|
|
41
41
|
|
|
42
42
|
def _run(self, connector: Connector) -> CheckResult:
|
|
43
43
|
breaking_changes = get(connector.metadata, "releases.breakingChanges")
|
|
@@ -65,7 +65,7 @@ class CheckMigrationGuide(DocumentationCheck):
|
|
|
65
65
|
first_line = migration_guide_content.splitlines()[0]
|
|
66
66
|
except IndexError:
|
|
67
67
|
first_line = migration_guide_content
|
|
68
|
-
if
|
|
68
|
+
if first_line != expected_title:
|
|
69
69
|
return self.create_check_result(
|
|
70
70
|
connector=connector,
|
|
71
71
|
passed=False,
|
|
@@ -134,19 +134,19 @@ class CheckDocumentationContent(DocumentationCheck):
|
|
|
134
134
|
For now, we check documentation structure for sources with sl >= 300.
|
|
135
135
|
"""
|
|
136
136
|
|
|
137
|
-
applies_to_connector_languages = [
|
|
137
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
138
138
|
ConnectorLanguage.PYTHON,
|
|
139
139
|
ConnectorLanguage.LOW_CODE,
|
|
140
140
|
]
|
|
141
141
|
applies_to_connector_ab_internal_sl = 300
|
|
142
|
-
applies_to_connector_types = ["source"]
|
|
142
|
+
applies_to_connector_types: ClassVar[list[str]] = ["source"]
|
|
143
143
|
|
|
144
144
|
|
|
145
145
|
class CheckDocumentationLinks(CheckDocumentationContent):
|
|
146
146
|
name = "Links used in connector documentation are valid"
|
|
147
147
|
description = "The user facing connector documentation should update invalid links in connector documentation. For links that are used as example and return 404 status code, use `example: ` before link to skip it."
|
|
148
148
|
|
|
149
|
-
def validate_links(self, connector: Connector) ->
|
|
149
|
+
def validate_links(self, connector: Connector) -> list[str]:
|
|
150
150
|
errors = []
|
|
151
151
|
threads = []
|
|
152
152
|
|
|
@@ -246,7 +246,7 @@ class CheckDocumentationHeadersOrder(CheckDocumentationContent):
|
|
|
246
246
|
]
|
|
247
247
|
return not_required
|
|
248
248
|
|
|
249
|
-
def check_headers(self, connector: Connector) ->
|
|
249
|
+
def check_headers(self, connector: Connector) -> list[str]:
|
|
250
250
|
"""
|
|
251
251
|
test_docs_structure gets all top-level headers from source documentation file and check that the order is correct.
|
|
252
252
|
The order of the headers should follow our standard template connectors_qa/checks/documentation/templates/template.md.j2,
|
|
@@ -356,9 +356,15 @@ class CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec(
|
|
|
356
356
|
)
|
|
357
357
|
|
|
358
358
|
PREREQUISITES = "Prerequisites"
|
|
359
|
-
CREDENTIALS_KEYWORDS = [
|
|
359
|
+
CREDENTIALS_KEYWORDS: ClassVar[list[str]] = [
|
|
360
|
+
"account",
|
|
361
|
+
"auth",
|
|
362
|
+
"credentials",
|
|
363
|
+
"access",
|
|
364
|
+
"client",
|
|
365
|
+
]
|
|
360
366
|
|
|
361
|
-
def check_prerequisites(self, connector: Connector) ->
|
|
367
|
+
def check_prerequisites(self, connector: Connector) -> list[str]:
|
|
362
368
|
actual_connector_spec = connector.connector_spec_file_content
|
|
363
369
|
if not actual_connector_spec:
|
|
364
370
|
return []
|
|
@@ -383,7 +389,7 @@ class CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec(
|
|
|
383
389
|
) or actual_connector_spec.get("connection_specification")
|
|
384
390
|
required_titles, has_credentials = required_titles_from_spec(spec) # type: ignore
|
|
385
391
|
|
|
386
|
-
missing_fields:
|
|
392
|
+
missing_fields: list[str] = []
|
|
387
393
|
for title in required_titles:
|
|
388
394
|
if title.lower() not in section_text:
|
|
389
395
|
missing_fields.append(title)
|
|
@@ -458,7 +464,7 @@ class CheckSection(CheckDocumentationContent):
|
|
|
458
464
|
def header(self) -> str:
|
|
459
465
|
"""The name of header for validating content"""
|
|
460
466
|
|
|
461
|
-
def check_section(self, connector: Connector) ->
|
|
467
|
+
def check_section(self, connector: Connector) -> list[str]:
|
|
462
468
|
documentation = DocumentationContent(connector=connector)
|
|
463
469
|
|
|
464
470
|
if self.header not in documentation.headers:
|
|
@@ -466,7 +472,7 @@ class CheckSection(CheckDocumentationContent):
|
|
|
466
472
|
return [f"Documentation does not have {self.header} section."]
|
|
467
473
|
return []
|
|
468
474
|
|
|
469
|
-
errors:
|
|
475
|
+
errors: list[str] = []
|
|
470
476
|
|
|
471
477
|
expected = TemplateContent(connector.name_from_metadata).section(self.header)[
|
|
472
478
|
self.expected_section_index
|
|
@@ -532,19 +538,14 @@ class CheckSourceSectionContent(CheckDocumentationContent):
|
|
|
532
538
|
template = TemplateContent("CONNECTOR_NAME_FROM_METADATA").section(
|
|
533
539
|
"CONNECTOR_NAME_FROM_METADATA"
|
|
534
540
|
)
|
|
535
|
-
if template is None
|
|
536
|
-
template_content = (
|
|
537
|
-
"" # Provide default empty template if section is missing
|
|
538
|
-
)
|
|
539
|
-
else:
|
|
540
|
-
template_content = template[0] # type: ignore
|
|
541
|
+
template_content = "" if template is None else template[0]
|
|
541
542
|
|
|
542
543
|
return generate_description(
|
|
543
544
|
"section_content_description.md.j2",
|
|
544
545
|
{"header": "CONNECTOR_NAME_FROM_METADATA", "template": template_content},
|
|
545
546
|
)
|
|
546
547
|
|
|
547
|
-
def check_source_follows_template(self, connector: Connector) ->
|
|
548
|
+
def check_source_follows_template(self, connector: Connector) -> list[str]:
|
|
548
549
|
documentation = DocumentationContent(connector=connector)
|
|
549
550
|
|
|
550
551
|
if connector.name_from_metadata not in documentation.headers:
|
|
@@ -635,7 +636,7 @@ class CheckTutorialsSectionContent(CheckSection):
|
|
|
635
636
|
class CheckChangelogSectionContent(CheckSection):
|
|
636
637
|
header = "Changelog"
|
|
637
638
|
|
|
638
|
-
def check_section(self, connector: Connector) ->
|
|
639
|
+
def check_section(self, connector: Connector) -> list[str]:
|
|
639
640
|
documentation = DocumentationContent(connector=connector)
|
|
640
641
|
|
|
641
642
|
if self.header not in documentation.headers:
|
airbyte_ops_mcp/{_legacy/airbyte_ci/connector_qa → connector_qa}/checks/documentation/models.py
RENAMED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import ClassVar
|
|
6
6
|
|
|
7
7
|
from jinja2 import Environment, FileSystemLoader
|
|
8
8
|
from markdown_it.tree import SyntaxTreeNode
|
|
9
9
|
|
|
10
|
-
from airbyte_ops_mcp.
|
|
10
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
11
11
|
Connector, # type: ignore
|
|
12
12
|
)
|
|
13
13
|
|
|
@@ -26,10 +26,10 @@ class SectionLines:
|
|
|
26
26
|
class SectionContent:
|
|
27
27
|
def __init__(self, header: str):
|
|
28
28
|
self.header = header
|
|
29
|
-
self._content:
|
|
29
|
+
self._content: list[str] = []
|
|
30
30
|
|
|
31
31
|
@property
|
|
32
|
-
def content(self) ->
|
|
32
|
+
def content(self) -> list[str]:
|
|
33
33
|
return self._content
|
|
34
34
|
|
|
35
35
|
@content.setter
|
|
@@ -42,7 +42,7 @@ class SectionContent:
|
|
|
42
42
|
|
|
43
43
|
class Content:
|
|
44
44
|
HEADING = "heading"
|
|
45
|
-
supported_header_levels = ["h1", "h2", "h3", "h4"]
|
|
45
|
+
supported_header_levels: ClassVar[list[str]] = ["h1", "h2", "h3", "h4"]
|
|
46
46
|
|
|
47
47
|
def __init__(self) -> None:
|
|
48
48
|
self.content = self._content()
|
|
@@ -69,10 +69,10 @@ class Content:
|
|
|
69
69
|
|
|
70
70
|
return headers
|
|
71
71
|
|
|
72
|
-
def _header_line_map(self) ->
|
|
72
|
+
def _header_line_map(self) -> dict[str, list[SectionLines]]:
|
|
73
73
|
headers = []
|
|
74
74
|
starts = []
|
|
75
|
-
header_line_map:
|
|
75
|
+
header_line_map: dict[str, list[SectionLines]] = {}
|
|
76
76
|
|
|
77
77
|
for n in self.node: # type: ignore
|
|
78
78
|
if n.type == self.HEADING:
|
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
2
|
|
|
3
3
|
from datetime import datetime, timedelta
|
|
4
|
+
from typing import ClassVar
|
|
4
5
|
|
|
5
6
|
import toml
|
|
6
7
|
|
|
7
|
-
from airbyte_ops_mcp.
|
|
8
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
8
9
|
Connector,
|
|
9
10
|
ConnectorLanguage,
|
|
10
11
|
) # type: ignore
|
|
11
|
-
from airbyte_ops_mcp.
|
|
12
|
-
from airbyte_ops_mcp.
|
|
12
|
+
from airbyte_ops_mcp.connector_qa import consts
|
|
13
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
13
14
|
Check,
|
|
14
15
|
CheckCategory,
|
|
15
16
|
CheckResult,
|
|
16
17
|
)
|
|
17
|
-
from airbyte_ops_mcp.
|
|
18
|
+
from airbyte_ops_mcp.metadata_validator import (
|
|
18
19
|
PRE_UPLOAD_VALIDATORS,
|
|
19
20
|
ValidatorOptions,
|
|
20
21
|
validate_and_load,
|
|
21
|
-
)
|
|
22
|
+
)
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
class MetadataCheck(Check):
|
|
@@ -107,7 +108,7 @@ class CheckConnectorLanguageTag(MetadataCheck):
|
|
|
107
108
|
class CheckConnectorCDKTag(MetadataCheck):
|
|
108
109
|
name = "Python connectors must have a CDK tag in metadata"
|
|
109
110
|
description = f"Python connectors must have a CDK tag in their metadata. It must be set in the `tags` field in {consts.METADATA_FILE_NAME}. The values can be `cdk:low-code`, `cdk:python`, or `cdk:file`."
|
|
110
|
-
applies_to_connector_languages = [
|
|
111
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
111
112
|
ConnectorLanguage.PYTHON,
|
|
112
113
|
ConnectorLanguage.LOW_CODE,
|
|
113
114
|
]
|
|
@@ -138,12 +139,11 @@ class CheckConnectorCDKTag(MetadataCheck):
|
|
|
138
139
|
and "file-based" in cdk_deps.get("extras", [])
|
|
139
140
|
):
|
|
140
141
|
return self.CDKTag.FILE
|
|
141
|
-
if setup_py_file.exists()
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
return self.CDKTag.FILE
|
|
142
|
+
if setup_py_file.exists() and (
|
|
143
|
+
"airbyte-cdk[file-based]"
|
|
144
|
+
in (connector.code_directory / consts.SETUP_PY_FILE_NAME).read_text()
|
|
145
|
+
):
|
|
146
|
+
return self.CDKTag.FILE
|
|
147
147
|
return self.CDKTag.PYTHON
|
|
148
148
|
|
|
149
149
|
def _run(self, connector: Connector) -> CheckResult:
|
|
@@ -179,7 +179,7 @@ class ValidateBreakingChangesDeadlines(MetadataCheck):
|
|
|
179
179
|
"""
|
|
180
180
|
|
|
181
181
|
name = "Breaking change deadline should be a week in the future"
|
|
182
|
-
description = "If the connector version has a breaking change, the deadline field must be set to at least a week in the future."
|
|
182
|
+
description = "If the connector version has a breaking change, the deadline field must be set to at least a week in the future. See the Breaking Changes Policy for full requirements: https://docs.airbyte.com/platform/connector-development/connector-breaking-changes"
|
|
183
183
|
runs_on_released_connectors = False
|
|
184
184
|
minimum_days_until_deadline = 7
|
|
185
185
|
|
|
@@ -238,8 +238,8 @@ class ValidateBreakingChangesDeadlines(MetadataCheck):
|
|
|
238
238
|
class CheckConnectorMaxSecondsBetweenMessagesValue(MetadataCheck):
|
|
239
239
|
name = "Certified source connector must have a value filled out for maxSecondsBetweenMessages in metadata"
|
|
240
240
|
description = "Certified source connectors must have a value filled out for `maxSecondsBetweenMessages` in metadata. This value represents the maximum number of seconds we could expect between messages for API connectors. And it's used by platform to tune connectors heartbeat timeout. The value must be set in the 'data' field in connector's `metadata.yaml` file."
|
|
241
|
-
applies_to_connector_types = ["source"]
|
|
242
|
-
applies_to_connector_support_levels = ["certified"]
|
|
241
|
+
applies_to_connector_types: ClassVar[list[str]] = ["source"]
|
|
242
|
+
applies_to_connector_support_levels: ClassVar[list[str]] = ["certified"]
|
|
243
243
|
|
|
244
244
|
def _run(self, connector: Connector) -> CheckResult:
|
|
245
245
|
max_seconds_between_messages = connector.metadata.get(
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
2
|
|
|
3
|
+
from typing import ClassVar
|
|
4
|
+
|
|
3
5
|
import semver
|
|
4
6
|
import toml
|
|
5
7
|
from pydash.objects import get # type: ignore
|
|
6
8
|
|
|
7
|
-
from airbyte_ops_mcp.
|
|
9
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
8
10
|
Connector,
|
|
9
11
|
ConnectorLanguage,
|
|
10
12
|
) # type: ignore
|
|
11
|
-
from airbyte_ops_mcp.
|
|
12
|
-
from airbyte_ops_mcp.
|
|
13
|
+
from airbyte_ops_mcp.connector_qa import consts
|
|
14
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
13
15
|
Check,
|
|
14
16
|
CheckCategory,
|
|
15
17
|
CheckResult,
|
|
@@ -25,7 +27,7 @@ class CheckConnectorUsesPoetry(PackagingCheck):
|
|
|
25
27
|
description = "Connectors must use [Poetry](https://python-poetry.org/) for dependency management. This is to ensure that all connectors use a dependency management tool which locks dependencies and ensures reproducible installs."
|
|
26
28
|
requires_metadata = False
|
|
27
29
|
runs_on_released_connectors = False
|
|
28
|
-
applies_to_connector_languages = [
|
|
30
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
29
31
|
ConnectorLanguage.PYTHON,
|
|
30
32
|
ConnectorLanguage.LOW_CODE,
|
|
31
33
|
]
|
|
@@ -56,11 +58,11 @@ class CheckConnectorUsesPoetry(PackagingCheck):
|
|
|
56
58
|
class CheckPublishToPyPiIsDeclared(PackagingCheck):
|
|
57
59
|
name = "Python connectors must have PyPi publishing declared."
|
|
58
60
|
description = f"Python connectors must have [PyPi](https://pypi.org/) publishing enabled in their `{consts.METADATA_FILE_NAME}` file. This is declared by setting `remoteRegistries.pypi.enabled` to `true` in {consts.METADATA_FILE_NAME}. This is to ensure that all connectors can be published to PyPi and can be used in `PyAirbyte`."
|
|
59
|
-
applies_to_connector_languages = [
|
|
61
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
60
62
|
ConnectorLanguage.PYTHON,
|
|
61
63
|
ConnectorLanguage.LOW_CODE,
|
|
62
64
|
]
|
|
63
|
-
applies_to_connector_types = ["source"]
|
|
65
|
+
applies_to_connector_types: ClassVar[list[str]] = ["source"]
|
|
64
66
|
|
|
65
67
|
def _run(self, connector: Connector) -> CheckResult:
|
|
66
68
|
publish_to_pypi_is_enabled = get(
|
|
@@ -80,7 +82,7 @@ class CheckPublishToPyPiIsDeclared(PackagingCheck):
|
|
|
80
82
|
class CheckManifestOnlyConnectorBaseImage(PackagingCheck):
|
|
81
83
|
name = "Manifest-only connectors must use `source-declarative-manifest` as their base image"
|
|
82
84
|
description = "Manifest-only connectors must use `airbyte/source-declarative-manifest` as their base image."
|
|
83
|
-
applies_to_connector_languages = [ConnectorLanguage.MANIFEST_ONLY]
|
|
85
|
+
applies_to_connector_languages: ClassVar[list] = [ConnectorLanguage.MANIFEST_ONLY]
|
|
84
86
|
|
|
85
87
|
def _run(self, connector: Connector) -> CheckResult:
|
|
86
88
|
base_image = get(connector.metadata, "connectorBuildOptions.baseImage")
|
|
@@ -125,7 +127,7 @@ class CheckConnectorLicense(PackagingCheck):
|
|
|
125
127
|
class CheckConnectorLicenseMatchInPyproject(PackagingCheck):
|
|
126
128
|
name = f"Connector license in {consts.METADATA_FILE_NAME} and {consts.PYPROJECT_FILE_NAME} file must match"
|
|
127
129
|
description = f"Connectors license in {consts.METADATA_FILE_NAME} and {consts.PYPROJECT_FILE_NAME} file must match. This is to ensure that all connectors are consistently licensed."
|
|
128
|
-
applies_to_connector_languages = [
|
|
130
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
129
131
|
ConnectorLanguage.PYTHON,
|
|
130
132
|
ConnectorLanguage.LOW_CODE,
|
|
131
133
|
]
|
|
@@ -174,7 +176,7 @@ class CheckConnectorLicenseMatchInPyproject(PackagingCheck):
|
|
|
174
176
|
class CheckConnectorVersionMatchInPyproject(PackagingCheck):
|
|
175
177
|
name = f"Connector version in {consts.METADATA_FILE_NAME} and {consts.PYPROJECT_FILE_NAME} file must match"
|
|
176
178
|
description = f"Connector version in {consts.METADATA_FILE_NAME} and {consts.PYPROJECT_FILE_NAME} file must match. This is to ensure that connector release is consistent."
|
|
177
|
-
applies_to_connector_languages = [
|
|
179
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
178
180
|
ConnectorLanguage.PYTHON,
|
|
179
181
|
ConnectorLanguage.LOW_CODE,
|
|
180
182
|
]
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import
|
|
4
|
+
from typing import ClassVar, Iterable
|
|
5
5
|
|
|
6
6
|
from pydash.objects import get # type: ignore
|
|
7
7
|
|
|
8
|
-
from airbyte_ops_mcp.
|
|
8
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
9
9
|
Connector,
|
|
10
10
|
ConnectorLanguage,
|
|
11
11
|
) # type: ignore
|
|
12
|
-
from airbyte_ops_mcp.
|
|
13
|
-
from airbyte_ops_mcp.
|
|
12
|
+
from airbyte_ops_mcp.connector_qa import consts
|
|
13
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
14
14
|
Check,
|
|
15
15
|
CheckCategory,
|
|
16
16
|
CheckResult,
|
|
@@ -31,7 +31,7 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
31
31
|
|
|
32
32
|
ignore_comment = "# ignore-https-check" # Define the ignore comment pattern
|
|
33
33
|
|
|
34
|
-
ignored_directories_for_https_checks = {
|
|
34
|
+
ignored_directories_for_https_checks: ClassVar[set[str]] = {
|
|
35
35
|
".venv",
|
|
36
36
|
"tests",
|
|
37
37
|
"unit_tests",
|
|
@@ -46,7 +46,7 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
46
46
|
"htmlcov",
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
ignored_file_name_pattern_for_https_checks = {
|
|
49
|
+
ignored_file_name_pattern_for_https_checks: ClassVar[set[str]] = {
|
|
50
50
|
"*Test.java",
|
|
51
51
|
"*.jar",
|
|
52
52
|
"*.pyc",
|
|
@@ -56,7 +56,7 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
56
56
|
"expected_records.json",
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
ignored_url_prefixes = {
|
|
59
|
+
ignored_url_prefixes: ClassVar[set[str]] = {
|
|
60
60
|
"http://json-schema.org",
|
|
61
61
|
"http://localhost",
|
|
62
62
|
}
|
|
@@ -64,9 +64,9 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
64
64
|
@staticmethod
|
|
65
65
|
def _read_all_files_in_directory(
|
|
66
66
|
directory: Path,
|
|
67
|
-
ignored_directories:
|
|
68
|
-
ignored_filename_patterns:
|
|
69
|
-
) -> Iterable[
|
|
67
|
+
ignored_directories: set[str] | None = None,
|
|
68
|
+
ignored_filename_patterns: set[str] | None = None,
|
|
69
|
+
) -> Iterable[tuple[Path, str]]:
|
|
70
70
|
ignored_directories = (
|
|
71
71
|
ignored_directories if ignored_directories is not None else set()
|
|
72
72
|
)
|
|
@@ -78,21 +78,17 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
78
78
|
|
|
79
79
|
for path in directory.rglob("*"):
|
|
80
80
|
ignore_directory = any(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
for ignored_directory in ignored_directories
|
|
84
|
-
]
|
|
81
|
+
ignored_directory in path.parts
|
|
82
|
+
for ignored_directory in ignored_directories
|
|
85
83
|
)
|
|
86
84
|
ignore_filename = any(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
for ignored_filename_pattern in ignored_filename_patterns
|
|
90
|
-
]
|
|
85
|
+
path.match(ignored_filename_pattern)
|
|
86
|
+
for ignored_filename_pattern in ignored_filename_patterns
|
|
91
87
|
)
|
|
92
88
|
ignore = ignore_directory or ignore_filename
|
|
93
89
|
if path.is_file() and not ignore:
|
|
94
90
|
try:
|
|
95
|
-
for line in
|
|
91
|
+
for line in path.read_text().splitlines():
|
|
96
92
|
yield path, line
|
|
97
93
|
except UnicodeDecodeError:
|
|
98
94
|
continue
|
|
@@ -146,7 +142,7 @@ class CheckConnectorUsesHTTPSOnly(SecurityCheck):
|
|
|
146
142
|
class CheckConnectorUsesPythonBaseImage(SecurityCheck):
|
|
147
143
|
name = f"Python connectors must not use a {consts.DOCKERFILE_NAME} and must declare their base image in {consts.METADATA_FILE_NAME} file"
|
|
148
144
|
description = f"Connectors must use our Python connector base image (`{consts.AIRBYTE_PYTHON_CONNECTOR_BASE_IMAGE_NAME}`), declared through the `connectorBuildOptions.baseImage` in their `{consts.METADATA_FILE_NAME}`.\nThis is to ensure that all connectors use a base image which is maintained and has security updates."
|
|
149
|
-
applies_to_connector_languages = [
|
|
145
|
+
applies_to_connector_languages: ClassVar[list] = [
|
|
150
146
|
ConnectorLanguage.PYTHON,
|
|
151
147
|
ConnectorLanguage.LOW_CODE,
|
|
152
148
|
ConnectorLanguage.MANIFEST_ONLY,
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
|
2
2
|
|
|
3
|
-
from typing import Any
|
|
3
|
+
from typing import Any
|
|
4
4
|
|
|
5
5
|
import requests # type: ignore
|
|
6
6
|
import semver
|
|
7
7
|
import yaml # type: ignore
|
|
8
8
|
|
|
9
|
-
from airbyte_ops_mcp.
|
|
9
|
+
from airbyte_ops_mcp.connector_ops.utils import (
|
|
10
10
|
Connector, # type: ignore
|
|
11
11
|
)
|
|
12
|
-
from airbyte_ops_mcp.
|
|
13
|
-
from airbyte_ops_mcp.
|
|
12
|
+
from airbyte_ops_mcp.connector_qa import consts
|
|
13
|
+
from airbyte_ops_mcp.connector_qa.models import (
|
|
14
14
|
Check,
|
|
15
15
|
CheckCategory,
|
|
16
16
|
CheckResult,
|
|
@@ -47,7 +47,7 @@ class CheckVersionIncrement(Check):
|
|
|
47
47
|
# TODO: don't run if only files changed are in the bypass list or running in the context of the master branch
|
|
48
48
|
return True
|
|
49
49
|
|
|
50
|
-
def _get_master_metadata(self, connector: Connector) ->
|
|
50
|
+
def _get_master_metadata(self, connector: Connector) -> dict[str, Any] | None:
|
|
51
51
|
"""Get the metadata from the master branch or None if unable to retrieve."""
|
|
52
52
|
# TODO: test out if this works on the private airbyte-enterprise repo - consider using git-based approach
|
|
53
53
|
github_url_prefix = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors"
|
|
@@ -59,7 +59,7 @@ class CheckVersionIncrement(Check):
|
|
|
59
59
|
return None
|
|
60
60
|
return yaml.safe_load(response.text)["data"]
|
|
61
61
|
|
|
62
|
-
def _parse_version_from_metadata(self, metadata:
|
|
62
|
+
def _parse_version_from_metadata(self, metadata: dict[str, Any]) -> semver.Version:
|
|
63
63
|
return semver.Version.parse(str(metadata["dockerImageTag"]))
|
|
64
64
|
|
|
65
65
|
def _get_master_connector_version(self, connector: Connector) -> semver.Version:
|
|
@@ -95,6 +95,52 @@ class CheckVersionIncrement(Check):
|
|
|
95
95
|
and master_version.patch == current_version.patch
|
|
96
96
|
)
|
|
97
97
|
|
|
98
|
+
@staticmethod
|
|
99
|
+
def _get_registry_override_tag(
|
|
100
|
+
metadata: dict[str, Any] | None, channel: str
|
|
101
|
+
) -> str | None:
|
|
102
|
+
"""Extract the dockerImageTag from registryOverrides for a given channel.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
metadata: The metadata dictionary (master or current).
|
|
106
|
+
channel: The channel to extract from ("cloud" or "oss").
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
The dockerImageTag value if present, None otherwise.
|
|
110
|
+
"""
|
|
111
|
+
if metadata is None:
|
|
112
|
+
return None
|
|
113
|
+
try:
|
|
114
|
+
return (
|
|
115
|
+
metadata.get("registryOverrides", {})
|
|
116
|
+
.get(channel, {})
|
|
117
|
+
.get("dockerImageTag")
|
|
118
|
+
)
|
|
119
|
+
except (TypeError, AttributeError):
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
def _has_registry_override_docker_tag_change(
|
|
123
|
+
self,
|
|
124
|
+
master_metadata: dict[str, Any] | None,
|
|
125
|
+
current_metadata: dict[str, Any] | None,
|
|
126
|
+
) -> bool:
|
|
127
|
+
"""Check if registryOverrides.cloud.dockerImageTag or registryOverrides.oss.dockerImageTag has changed.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
master_metadata: The metadata from the master branch.
|
|
131
|
+
current_metadata: The current metadata.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
bool: True if either cloud or oss dockerImageTag has been added, removed, or changed.
|
|
135
|
+
"""
|
|
136
|
+
master_cloud = self._get_registry_override_tag(master_metadata, "cloud")
|
|
137
|
+
current_cloud = self._get_registry_override_tag(current_metadata, "cloud")
|
|
138
|
+
|
|
139
|
+
master_oss = self._get_registry_override_tag(master_metadata, "oss")
|
|
140
|
+
current_oss = self._get_registry_override_tag(current_metadata, "oss")
|
|
141
|
+
|
|
142
|
+
return (master_cloud != current_cloud) or (master_oss != current_oss)
|
|
143
|
+
|
|
98
144
|
def _run(self, connector: Connector) -> CheckResult:
|
|
99
145
|
"""Run the version increment check."""
|
|
100
146
|
if (
|
|
@@ -109,29 +155,59 @@ class CheckVersionIncrement(Check):
|
|
|
109
155
|
)
|
|
110
156
|
|
|
111
157
|
try:
|
|
112
|
-
|
|
158
|
+
master_metadata = self._get_master_metadata(connector)
|
|
159
|
+
# Note: master_metadata being None is expected for new connectors that don't exist
|
|
160
|
+
# in master yet. In this case, we use 0.0.0 as the baseline, allowing any version.
|
|
161
|
+
master_version = (
|
|
162
|
+
self._parse_version_from_metadata(master_metadata)
|
|
163
|
+
if master_metadata
|
|
164
|
+
else semver.Version.parse("0.0.0")
|
|
165
|
+
)
|
|
113
166
|
current_version = self._get_current_connector_version(connector)
|
|
114
167
|
|
|
115
168
|
# Require a version increment
|
|
116
|
-
if current_version
|
|
169
|
+
if current_version < master_version:
|
|
170
|
+
return self.fail(
|
|
171
|
+
connector,
|
|
172
|
+
f"The dockerImageTag in {consts.METADATA_FILE_NAME} appears to be lower than the "
|
|
173
|
+
f"version on the default branch. Master version is {master_version}, current "
|
|
174
|
+
f"version is {current_version}. "
|
|
175
|
+
f"Update your PR branch from the default branch to get the latest connector version. "
|
|
176
|
+
f"Maintainers can use the `/bump-version` PR slash command. "
|
|
177
|
+
f"AI agents can use the `bump_version_in_repo` tool from the `airbyte-ops-mcp` MCP server.",
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
if current_version == master_version:
|
|
181
|
+
# Allow version to stay the same if registryOverrides.cloud.dockerImageTag or
|
|
182
|
+
# registryOverrides.oss.dockerImageTag has been added, removed, or changed
|
|
183
|
+
if self._has_registry_override_docker_tag_change(
|
|
184
|
+
master_metadata, connector.metadata
|
|
185
|
+
):
|
|
186
|
+
return self.skip(
|
|
187
|
+
connector,
|
|
188
|
+
f"The current change is modifying the registryOverrides pinned version on Cloud or OSS. "
|
|
189
|
+
f"Skipping this check because the defined version {current_version} is allowed to be unchanged.",
|
|
190
|
+
)
|
|
117
191
|
return self.fail(
|
|
118
192
|
connector,
|
|
119
193
|
f"The dockerImageTag in {consts.METADATA_FILE_NAME} was not incremented. "
|
|
120
|
-
f"Master version is {master_version}, current version is {current_version}"
|
|
194
|
+
f"Master version is {master_version}, current version is {current_version}. "
|
|
195
|
+
f"Ignore this message if you do not intend to re-release the connector. "
|
|
196
|
+
f"Maintainers can use the `/bump-version` PR slash command. "
|
|
197
|
+
f"AI agents can use the `bump_version_in_repo` tool from the `airbyte-ops-mcp` MCP server.",
|
|
121
198
|
)
|
|
122
199
|
|
|
123
200
|
if self._are_both_versions_release_candidates(
|
|
124
201
|
master_version, current_version
|
|
202
|
+
) and not self._have_same_major_minor_patch(
|
|
203
|
+
master_version, current_version
|
|
125
204
|
):
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
f"Release candidates should only differ in the prerelease part. Master version is {master_version}, "
|
|
133
|
-
f"current version is {current_version}",
|
|
134
|
-
)
|
|
205
|
+
return self.fail(
|
|
206
|
+
connector,
|
|
207
|
+
f"Master and current version are release candidates but they have different major, minor or patch versions. "
|
|
208
|
+
f"Release candidates should only differ in the prerelease part. Master version is {master_version}, "
|
|
209
|
+
f"current version is {current_version}",
|
|
210
|
+
)
|
|
135
211
|
|
|
136
212
|
return self.pass_(
|
|
137
213
|
connector,
|