airbyte-cdk 6.61.5__tar.gz → 6.61.6__tar.gz

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.5
+Version: 6.61.6
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -44,7 +44,7 @@ Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
-Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: jsonschema (>=4.17.3,<5.0)
 Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
 Requires-Dist: langchain_core (==0.1.42)
 Requires-Dist: markdown ; extra == "file-based"
@@ -69,6 +69,7 @@ Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
 Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
 Requires-Dist: pytz (==2024.2)
 Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: referencing (>=0.36.2)
 Requires-Dist: requests
 Requires-Dist: requests_cache
 Requires-Dist: rich

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -62,10 +62,10 @@ def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
 
 def create_source(
     config: Mapping[str, Any],
-    limits: TestLimits,
-    catalog: Optional[ConfiguredAirbyteCatalog],
-    state: Optional[List[AirbyteStateMessage]],
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+    limits: TestLimits | None = None,
+    catalog: ConfiguredAirbyteCatalog | None = None,
+    state: List[AirbyteStateMessage] | None = None,
+) -> ConcurrentDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
 
     # We enforce a concurrency level of 1 so that the stream is processed on a single thread
@@ -88,7 +88,7 @@ def create_source(
 
 
 def read_stream(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
@@ -127,7 +127,7 @@ def read_stream(
 
 
 def resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
 ) -> AirbyteMessage:
     try:
         return AirbyteMessage(
@@ -146,7 +146,7 @@ def resolve_manifest(
 
 
 def full_resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]], limits: TestLimits
+    source: ConcurrentDeclarativeSource, limits: TestLimits
 ) -> AirbyteMessage:
     try:
         manifest = {**source.resolved_manifest}

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/main.py

@@ -34,7 +34,7 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 def get_config_and_catalog_from_args(
     args: List[str],
-) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]:
+) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]]:
     # TODO: Add functionality for the `debug` logger.
     #  Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`.
     parsed_args = AirbyteEntrypoint.parse_args(args)
@@ -70,7 +70,7 @@ def get_config_and_catalog_from_args(
 
 
 def handle_connector_builder_request(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     command: str,
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/test_reader/reader.py

@@ -85,7 +85,7 @@ class TestReader:
 
     def run_test_read(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         stream_name: str,
@@ -383,7 +383,7 @@ class TestReader:
 
     def _read_stream(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         state: List[AirbyteStateMessage],

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/Dockerfile

@@ -11,7 +11,7 @@ FROM python:3.12-slim-bookworm
 RUN apt-get update && \
     apt-get install -y git && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install poetry==1.8.3
+    pip install poetry==2.0.1
 
 # Configure poetry to not create virtual environments and disable interactive mode
 ENV POETRY_NO_INTERACTION=1 \
@@ -42,4 +42,4 @@ USER airbyte:airbyte
 
 EXPOSE 8080
 
-CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/command_processor/processor.py

@@ -21,12 +21,10 @@ from airbyte_cdk.test.entrypoint_wrapper import AirbyteEntrypointException, Entr
 
 
 class ManifestCommandProcessor:
-    _source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
+    _source: ConcurrentDeclarativeSource
     _logger = logging.getLogger("airbyte.manifest-server")
 
-    def __init__(
-        self, source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
-    ) -> None:
+    def __init__(self, source: ConcurrentDeclarativeSource) -> None:
         self._source = source
 
     def test_read(

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/command_processor/utils.py

@@ -63,7 +63,7 @@ def build_source(
     record_limit: Optional[int] = None,
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+) -> ConcurrentDeclarativeSource:
     # We enforce a concurrency level of 1 so that the stream is processed on a single thread
     # to retain ordering for the grouping of the builder message responses.
     definition = copy.deepcopy(manifest)

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/routers/manifest.py

@@ -40,7 +40,7 @@ def safe_build_source(
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
     record_limit: Optional[int] = None,
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+) -> ConcurrentDeclarativeSource:
     """Wrapper around build_source that converts ValidationError to HTTPException."""
     try:
         return build_source(

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -162,16 +162,17 @@ def _get_declarative_component_schema() -> Dict[str, Any]:
 #  is no longer inherited from since the only external dependency is from that class.
 #
 # todo: It is worth investigating removal of the Generic[TState] since it will always be Optional[List[AirbyteStateMessage]]
-class ConcurrentDeclarativeSource(AbstractSource, Generic[TState]):
+class ConcurrentDeclarativeSource(AbstractSource):
     # By default, we defer to a value of 2. A value lower than could cause a PartitionEnqueuer to be stuck in a state of deadlock
     # because it has hit the limit of futures but not partition reader is consuming them.
     _LOWEST_SAFE_CONCURRENCY_LEVEL = 2
 
     def __init__(
         self,
-        catalog: Optional[ConfiguredAirbyteCatalog],
-        config: Optional[Mapping[str, Any]],
-        state: TState,
+        catalog: Optional[ConfiguredAirbyteCatalog] = None,
+        config: Optional[Mapping[str, Any]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
+        *,
         source_config: ConnectionDefinition,
         debug: bool = False,
         emit_connector_builder_messages: bool = False,

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -14,7 +14,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
 from airbyte_cdk.sources.types import ConnectionDefinition
 
 
-class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage]]):
+class YamlDeclarativeSource(ConcurrentDeclarativeSource):
     """Declarative source defined by a yaml file"""
 
     def __init__(

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/utils/schema_helpers.py

@@ -7,12 +7,16 @@ import importlib
 import json
 import os
 import pkgutil
-from typing import Any, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Tuple
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, MutableMapping, Tuple, cast
 
 import jsonref
-from jsonschema import RefResolver, validate
+from jsonschema import validate
 from jsonschema.exceptions import ValidationError
 from pydantic.v1 import BaseModel, Field
+from referencing import Registry, Resource
+from referencing._core import Resolver  # used for type hints
+from referencing.jsonschema import DRAFT7
 
 from airbyte_cdk.models import ConnectorSpecification, FailureType
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -63,18 +67,30 @@ def resolve_ref_links(obj: Any) -> Any:
         return obj
 
 
-def _expand_refs(schema: Any, ref_resolver: Optional[RefResolver] = None) -> None:
+def get_ref_resolver_registry(schema: dict[str, Any]) -> Registry:
+    """Get a reference resolver registry for the given schema."""
+    resource: Resource = Resource.from_contents(
+        contents=schema,
+        default_specification=DRAFT7,
+    )
+    return cast(  # Mypy has a hard time detecting this return type.
+        "Registry",
+        Registry().with_resource(
+            uri="",
+            resource=resource,
+        ),
+    )
+
+
+def _expand_refs(schema: Any, ref_resolver: Resolver) -> None:
     """Internal function to iterate over schema and replace all occurrences of $ref with their definitions. Recursive.
 
     :param schema: schema that will be patched
-    :param ref_resolver: resolver to get definition from $ref, if None pass it will be instantiated
     """
-    ref_resolver = ref_resolver or RefResolver.from_schema(schema)
-
     if isinstance(schema, MutableMapping):
         if "$ref" in schema:
             ref_url = schema.pop("$ref")
-            _, definition = ref_resolver.resolve(ref_url)
+            definition = ref_resolver.lookup(ref_url).contents
             _expand_refs(
                 definition, ref_resolver=ref_resolver
             )  # expand refs in definitions as well
@@ -90,10 +106,14 @@ def _expand_refs(schema: Any, ref_resolver: Optional[RefResolver] = None) -> Non
 def expand_refs(schema: Any) -> None:
     """Iterate over schema and replace all occurrences of $ref with their definitions.
 
+    If a "definitions" section is present at the root of the schema, it will be removed
+    after $ref resolution is complete.
+
     :param schema: schema that will be patched
     """
-    _expand_refs(schema)
-    schema.pop("definitions", None)  # remove definitions created by $ref
+    ref_resolver = get_ref_resolver_registry(schema).resolver()
+    _expand_refs(schema, ref_resolver)
+    schema.pop("definitions", None)
 
 
 def rename_key(schema: Any, old_key: str, new_key: str) -> None:

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/utils/transform.py

@@ -3,10 +3,25 @@
 #
 
 import logging
+from copy import deepcopy
 from enum import Flag, auto
-from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
+
+from jsonschema import Draft7Validator, ValidationError, validators
+from referencing import Registry, Resource
+from referencing._core import Resolver
+from referencing.exceptions import Unresolvable
+from referencing.jsonschema import DRAFT7
+
+from airbyte_cdk.sources.utils.schema_helpers import expand_refs
+
+from .schema_helpers import get_ref_resolver_registry
+
+try:
+    from jsonschema.validators import Validator
+except:
+    from jsonschema import Validator
 
-from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
 
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
@@ -191,15 +206,14 @@ class TypeTransformer:
             validators parameter for detailed description.
             :
             """
+            # Very first step is to expand $refs in the schema itself:
+            expand_refs(schema)
+
+            # Now we can expand $refs in the property value:
+            if isinstance(property_value, dict):
+                expand_refs(property_value)
 
-            def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
-                if "$ref" in subschema:
-                    _, resolved = cast(
-                        RefResolver,
-                        validator_instance.resolver,
-                    ).resolve(subschema["$ref"])
-                    return cast(dict[str, Any], resolved)
-                return subschema
+            # Now we can validate and normalize the values:
 
             # Transform object and array values before running json schema type checking for each element.
             # Recursively normalize every value of the "instance" sub-object,
@@ -207,14 +221,12 @@ class TypeTransformer:
             if schema_key == "properties" and isinstance(instance, dict):
                 for k, subschema in property_value.items():
                     if k in instance:
-                        subschema = resolve(subschema)
                         instance[k] = self.__normalize(instance[k], subschema)
             # Recursively normalize every item of the "instance" sub-array,
             # if "instance" is an incorrect type - skip recursive normalization of "instance"
             elif schema_key == "items" and isinstance(instance, list):
-                subschema = resolve(property_value)
                 for index, item in enumerate(instance):
-                    instance[index] = self.__normalize(item, subschema)
+                    instance[index] = self.__normalize(item, property_value)
 
             # Running native jsonschema traverse algorithm after field normalization is done.
             yield from original_validator(

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/airbyte_cdk/utils/spec_schema_transformations.py

@@ -6,7 +6,8 @@ import json
 import re
 from typing import Any
 
-from jsonschema import RefResolver
+from referencing import Registry, Resource
+from referencing.jsonschema import DRAFT7
 
 
 def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
@@ -14,13 +15,14 @@ def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
     For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object
     relationships.
     """
-    json_schema_ref_resolver = RefResolver.from_schema(schema)
+    resource = Resource.from_contents(schema, default_specification=DRAFT7)
+    registry = Registry().with_resource("", resource)
+    resolver = registry.resolver()
     str_schema = json.dumps(schema)
     for ref_block in re.findall(r'{"\$ref": "#\/definitions\/.+?(?="})"}', str_schema):
         ref = json.loads(ref_block)["$ref"]
-        str_schema = str_schema.replace(
-            ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])
-        )
+        resolved = resolver.lookup(ref).contents
+        str_schema = str_schema.replace(ref_block, json.dumps(resolved))
     pyschema: dict[str, Any] = json.loads(str_schema)
     del pyschema["definitions"]
     return pyschema

{airbyte_cdk-6.61.5 → airbyte_cdk-6.61.6}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.61.5"  # Version will be calculated dynamically.
+version = "6.61.6"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -43,8 +43,9 @@ google-cloud-secret-manager = "^2.17.0"
 isodate = "~0.6.1"
 Jinja2 = "~3.1.2"
 jsonref = "~0.2"
-jsonschema = "~4.17.3"  # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
+jsonschema = ">=4.17.3,<5.0"
 packaging = "*"  # Transitive dependency used directly in code
+referencing = ">=0.36.2"
 pandas = "2.2.3"
 psutil = "6.1.0" # TODO: Remove if unused
 pydantic = "^2.7"