airbyte-cdk 6.7.0rc2__tar.gz → 6.7.0rc3__tar.gz

{airbyte_cdk-6.7.0rc2 → airbyte_cdk-6.7.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.7.0rc2
+Version: 6.7.0rc3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -37,7 +37,7 @@ Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
-Requires-Dist: jsonschema (>=3.2.0,<3.3.0)
+Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
 Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
 Requires-Dist: langchain_core (==0.1.42)
 Requires-Dist: markdown ; extra == "file-based"
@@ -60,6 +60,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
 Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
 Requires-Dist: pytz (==2024.1)
+Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
 Requires-Dist: requests
 Requires-Dist: requests_cache
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)

airbyte_cdk-6.7.0rc3/airbyte_cdk/cli/source_declarative_manifest/__init__.py

@@ -0,0 +1,6 @@
+from airbyte_cdk.cli.source_declarative_manifest._run import run
+
+
+__all__ = [
+    "run",
+]

airbyte_cdk-6.7.0rc3/airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Defines the `source-declarative-manifest` connector, which installs alongside CDK.
+
+This file was originally imported from the dedicated connector directory, under the
+`airbyte` monorepo.
+
+Usage:
+
+```
+pipx install airbyte-cdk
+source-declarative-manifest --help
+source-declarative-manifest spec
+...
+```
+"""
+
+from __future__ import annotations
+
+import json
+import pkgutil
+import sys
+import traceback
+from collections.abc import Mapping
+from datetime import datetime
+from pathlib import Path
+from typing import Any, cast
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.models import (
+    AirbyteErrorTraceMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    AirbyteStateMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecificationSerializer,
+    TraceType,
+    Type,
+)
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
+from orjson import orjson
+
+
+class SourceLocalYaml(YamlDeclarativeSource):
+    """
+    Declarative source defined by a yaml file in the local filesystem
+    """
+
+    def __init__(
+        self,
+        catalog: ConfiguredAirbyteCatalog | None,
+        config: Mapping[str, Any] | None,
+        state: TState,
+        **kwargs: Any,
+    ) -> None:
+        """
+        HACK!
+            Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
+            Implication: If you call YamlDeclarativeSource directly it will look for the yaml file in the wrong place. (e.g. the airbyte-cdk package)
+            Solution: Subclass YamlDeclarativeSource from the same location as the manifest to load.
+
+            When can we remove this?
+                When the airbyte-cdk is updated to not rely on the calling module name/path to find the yaml file.
+                When all manifest connectors are updated to use the new airbyte-cdk.
+                When all manifest connectors are updated to use the source-declarative-manifest as the base image.
+        """
+        super().__init__(
+            catalog=catalog,
+            config=config,
+            state=state,
+            path_to_yaml="manifest.yaml",
+        )
+
+
+def _is_local_manifest_command(args: list[str]) -> bool:
+    # Check for a local manifest.yaml file
+    return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
+
+
+def handle_command(args: list[str]) -> None:
+    if _is_local_manifest_command(args):
+        handle_local_manifest_command(args)
+    else:
+        handle_remote_manifest_command(args)
+
+
+def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
+    try:
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        return SourceLocalYaml(config=config, catalog=catalog, state=state)
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def handle_local_manifest_command(args: list[str]) -> None:
+    source = _get_local_yaml_source(args)
+    launch(
+        source=source,
+        args=args,
+    )
+
+
+def handle_remote_manifest_command(args: list[str]) -> None:
+    """Overrides the spec command to return the generalized spec for the declarative manifest source.
+
+    This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
+    because that will have a spec method that returns the spec for that specific source. Other than spec,
+    the generalized connector behaves the same as any other, since the manifest is provided in the config.
+    """
+    if args[0] == "spec":
+        json_spec = pkgutil.get_data(
+            "airbyte_cdk.cli.source_declarative_manifest",
+            "spec.json",
+        )
+        if json_spec is None:
+            raise FileNotFoundError(
+                "Could not find `spec.json` file for source-declarative-manifest"
+            )
+
+        spec_obj = json.loads(json_spec)
+        spec = ConnectorSpecificationSerializer.load(spec_obj)
+
+        message = AirbyteMessage(type=Type.SPEC, spec=spec)
+        print(AirbyteEntrypoint.airbyte_message_to_string(message))
+    else:
+        source = create_declarative_source(args)
+        launch(
+            source=source,
+            args=args,
+        )
+
+
+def create_declarative_source(args: list[str]) -> ConcurrentDeclarativeSource:
+    """Creates the source with the injected config.
+
+    This essentially does what other low-code sources do at build time, but at runtime,
+    with a user-provided manifest in the config. This better reflects what happens in the
+    connector builder.
+    """
+    try:
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        if "__injected_declarative_manifest" not in config:
+            raise ValueError(
+                f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
+            )
+        return ConcurrentDeclarativeSource(
+            config=config,
+            catalog=catalog,
+            state=state,
+            source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
+        )
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def _parse_inputs_into_config_catalog_state(
+    args: list[str],
+) -> tuple[
+    Mapping[str, Any] | None,
+    ConfiguredAirbyteCatalog | None,
+    list[AirbyteStateMessage],
+]:
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    config = (
+        ConcurrentDeclarativeSource.read_config(parsed_args.config)
+        if hasattr(parsed_args, "config")
+        else None
+    )
+    catalog = (
+        ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)
+        if hasattr(parsed_args, "catalog")
+        else None
+    )
+    state = (
+        ConcurrentDeclarativeSource.read_state(parsed_args.state)
+        if hasattr(parsed_args, "state")
+        else []
+    )
+
+    return config, catalog, state
+
+
+def run() -> None:
+    args: list[str] = sys.argv[1:]
+    handle_command(args)

airbyte_cdk-6.7.0rc3/airbyte_cdk/cli/source_declarative_manifest/spec.json

@@ -0,0 +1,17 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/sources/low-code",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Low-code source spec",
+    "type": "object",
+    "required": ["__injected_declarative_manifest"],
+    "additionalProperties": true,
+    "properties": {
+      "__injected_declarative_manifest": {
+        "title": "Low-code manifest",
+        "type": "object",
+        "description": "The low-code manifest that defines the components of the source."
+      }
+    }
+  }
+}

{airbyte_cdk-6.7.0rc2 → airbyte_cdk-6.7.0rc3}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2790,21 +2790,21 @@ interpolation:
         - created_at: "2020-01-01 00:00:00.000+00:00"
         - updated_at: "2020-01-02 00:00:00.000+00:00"
   macros:
-    - title: Now (UTC)
+    - title: now_utc
       description: Returns the current date and time in the UTC timezone.
       arguments: {}
       return_type: Datetime
       examples:
         - "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'"
         - "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'"
-    - title: Today (UTC)
+    - title: today_utc
       description: Returns the current date in UTC timezone. The output is a date object.
       arguments: {}
       return_type: Date
       examples:
         - "'{{ today_utc() }}' -> '2021-09-01'"
         - "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'"
-    - title: Timestamp
+    - title: timestamp
       description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
       arguments:
         datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
@@ -2815,7 +2815,7 @@ interpolation:
         - "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200"
-    - title: Max
+    - title: max
       description: Returns the largest object of a iterable, or or two or more arguments.
       arguments:
         args: iterable or a sequence of two or more arguments
@@ -2823,7 +2823,7 @@ interpolation:
       examples:
         - "'{{ max(2, 3) }}' -> 3"
         - "'{{ max([2, 3]) }}' -> 3"
-    - title: Day Delta
+    - title: day_delta
       description: Returns the datetime of now() + num_days.
       arguments:
         num_days: The number of days to add to now
@@ -2833,8 +2833,8 @@ interpolation:
         - "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'"
         - "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'"
         - "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'"
-    - title: Duration
-      description: Converts an ISO8601 duratioin to datetime.timedelta.
+    - title: duration
+      description: Converts an ISO8601 duration to datetime timedelta.
       arguments:
         duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
       return_type: datetime.timedelta
@@ -2842,7 +2842,7 @@ interpolation:
         - "'{{ duration('P1D') }}' -> '1 day, 0:00:00'"
         - "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'"
         - "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'"
-    - title: Format Datetime
+    - title: format_datetime
       description: Converts a datetime or a datetime-string to the specified format.
       arguments:
         datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
@@ -2854,7 +2854,7 @@ interpolation:
         - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
         - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
   filters:
-    - title: Hash
+    - title: hash
       description: Convert the specified value to a hashed string.
       arguments:
         hash_type: Valid hash type for converts ('md5' as default value).
@@ -2864,26 +2864,26 @@ interpolation:
         - "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'"
-    - title: Base64 encoder
+    - title: base64encode
       description: Convert the specified value to a string in the base64 format.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'"
-    - title: Base64 decoder
+    - title: base64decode
       description: Decodes the specified base64 format value into a common string.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'"
-    - title: String
+    - title: string
       description: Converts the specified value to a string.
       arguments: {}
       return_type: str
       examples:
         - '{{ 1 | string }} -> "1"'
         - '{{ ["hello", "world" | string }} -> "["hello", "world"]"'
-    - title: Regex Search
+    - title: regex_search
       description: Match the input string against a regular expression and return the first match.
       arguments:
         regex: The regular expression to search for. It must include a capture group.

airbyte_cdk-6.7.0rc3/airbyte_cdk/test/utils/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.

{airbyte_cdk-6.7.0rc2 → airbyte_cdk-6.7.0rc3}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.7.0rc2"  # Version will be calculated dynamically.
+version = "6.7.0rc3"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -40,7 +40,7 @@ genson = "1.3.0"
 isodate = "~0.6.1"
 Jinja2 = "~3.1.2"
 jsonref = "~0.2"
-jsonschema = "~3.2.0"
+jsonschema = "~4.17.3"  # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
 pandas = "2.2.2"
 pendulum = "<3.0.0"
 psutil = "6.1.0"
@@ -49,6 +49,7 @@ pyrate-limiter = "~3.1.0"
 python-dateutil = "*"
 python-ulid = "^3.0.0"
 PyYAML = "^6.0.1"
+rapidfuzz = "^3.10.1"
 requests = "*"
 requests_cache = "*"
 wcmatch = "10.0"
@@ -104,6 +105,10 @@ sphinx-docs = ["Sphinx", "sphinx-rtd-theme"]
 vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 
+[tool.poetry.scripts]
+
+source-declarative-manifest = "airbyte_cdk.cli.source_declarative_manifest:run"
+
 [tool.isort]
 skip = ["__init__.py"]  # TODO: Remove after this is fixed: https://github.com/airbytehq/airbyte-python-cdk/issues/12