airbyte-cdk 6.61.4__tar.gz → 6.61.6__tar.gz

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.61.4
+Version: 6.61.6
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<3.13
+Requires-Python: >=3.10,<3.14
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: dev
@@ -31,18 +32,19 @@ Requires-Dist: backoff
 Requires-Dist: boltons (>=25.0.0,<26.0.0)
 Requires-Dist: cachetools
 Requires-Dist: click (>=8.1.8,<9.0.0)
-Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
+Requires-Dist: cohere (>=4.21,<6.0.0) ; extra == "vector-db-based"
 Requires-Dist: cryptography (>=44.0.0,<45.0.0)
 Requires-Dist: dateparser (>=1.2.2,<2.0.0)
+Requires-Dist: ddtrace (>=3.12.3,<4.0.0) ; extra == "manifest-server"
 Requires-Dist: dpath (>=2.1.6,<3.0.0)
 Requires-Dist: dunamai (>=1.22.0,<2.0.0)
 Requires-Dist: fastapi (>=0.116.1) ; extra == "manifest-server"
-Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
+Requires-Dist: fastavro (>=1.11.0,<2.0.0) ; extra == "file-based"
 Requires-Dist: genson (==1.3.0)
 Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=0.2,<0.3)
-Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
+Requires-Dist: jsonschema (>=4.17.3,<5.0)
 Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
 Requires-Dist: langchain_core (==0.1.42)
 Requires-Dist: markdown ; extra == "file-based"
@@ -51,7 +53,7 @@ Requires-Dist: numpy (<2)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
 Requires-Dist: orjson (>=3.10.7,<4.0.0)
 Requires-Dist: packaging
-Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pandas (==2.2.3)
 Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
 Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
 Requires-Dist: psutil (==6.1.0)
@@ -67,6 +69,7 @@ Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
 Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
 Requires-Dist: pytz (==2024.2)
 Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
+Requires-Dist: referencing (>=0.36.2)
 Requires-Dist: requests
 Requires-Dist: requests_cache
 Requires-Dist: rich

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -62,10 +62,10 @@ def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
 
 def create_source(
     config: Mapping[str, Any],
-    limits: TestLimits,
-    catalog: Optional[ConfiguredAirbyteCatalog],
-    state: Optional[List[AirbyteStateMessage]],
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+    limits: TestLimits | None = None,
+    catalog: ConfiguredAirbyteCatalog | None = None,
+    state: List[AirbyteStateMessage] | None = None,
+) -> ConcurrentDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
 
     # We enforce a concurrency level of 1 so that the stream is processed on a single thread
@@ -88,7 +88,7 @@ def create_source(
 
 
 def read_stream(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
@@ -127,7 +127,7 @@ def read_stream(
 
 
 def resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
 ) -> AirbyteMessage:
     try:
         return AirbyteMessage(
@@ -146,7 +146,7 @@ def resolve_manifest(
 
 
 def full_resolve_manifest(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]], limits: TestLimits
+    source: ConcurrentDeclarativeSource, limits: TestLimits
 ) -> AirbyteMessage:
     try:
         manifest = {**source.resolved_manifest}

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/main.py

@@ -34,7 +34,7 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 def get_config_and_catalog_from_args(
     args: List[str],
-) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]:
+) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]]:
     # TODO: Add functionality for the `debug` logger.
     #  Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`.
     parsed_args = AirbyteEntrypoint.parse_args(args)
@@ -70,7 +70,7 @@ def get_config_and_catalog_from_args(
 
 
 def handle_connector_builder_request(
-    source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+    source: ConcurrentDeclarativeSource,
     command: str,
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/connector_builder/test_reader/reader.py

@@ -85,7 +85,7 @@ class TestReader:
 
     def run_test_read(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         stream_name: str,
@@ -383,7 +383,7 @@ class TestReader:
 
     def _read_stream(
         self,
-        source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]],
+        source: ConcurrentDeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
         state: List[AirbyteStateMessage],

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/Dockerfile

@@ -11,7 +11,7 @@ FROM python:3.12-slim-bookworm
 RUN apt-get update && \
     apt-get install -y git && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install poetry==1.8.3
+    pip install poetry==2.0.1
 
 # Configure poetry to not create virtual environments and disable interactive mode
 ENV POETRY_NO_INTERACTION=1 \
@@ -42,4 +42,4 @@ USER airbyte:airbyte
 
 EXPOSE 8080
 
-CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/README.md

@@ -37,26 +37,31 @@ The server will start on `http://localhost:8000` by default.
 ## API Endpoints
 
 ### `/v1/manifest/test_read`
+
 Test reading from a specific stream in the manifest.
 
 **POST** - Test stream reading with configurable limits for records, pages, and slices.
 
 ### `/v1/manifest/check`
+
 Check configuration against a manifest.
 
 **POST** - Validates connector configuration and returns success/failure status with message.
 
 ### `/v1/manifest/discover`
+
 Discover streams from a manifest.
 
 **POST** - Returns the catalog of available streams from the manifest.
 
-### `/v1/manifest/resolve` 
+### `/v1/manifest/resolve`
+
 Resolve a manifest to its final configuration.
 
 **POST** - Returns the resolved manifest without dynamic stream generation.
 
 ### `/v1/manifest/full_resolve`
+
 Fully resolve a manifest including dynamic streams.
 
 **POST** - Generates dynamic streams up to specified limits and includes them in the resolved manifest.
@@ -68,6 +73,7 @@ The manifest server supports custom Python components, but this feature is **dis
 ### Enabling Custom Components
 
 To allow custom Python components in your manifest files, set the environment variable:
+
 ```bash
 export AIRBYTE_ENABLE_UNSAFE_CODE=true
 ```
@@ -77,20 +83,25 @@ export AIRBYTE_ENABLE_UNSAFE_CODE=true
 The manifest server supports optional JWT bearer token authentication:
 
 ### Configuration
+
 Set the environment variable to enable authentication:
+
 ```bash
 export AB_JWT_SIGNATURE_SECRET="your-jwt-secret-key"
 ```
 
 ### Usage
+
 When authentication is enabled, include a valid JWT token in the Authorization header:
+
 ```bash
 curl -H "Authorization: Bearer <your-jwt-token>" \
   http://localhost:8000/v1/manifest/test_read
 ```
 
 ### Behavior
-- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through 
+
+- **Without `AB_JWT_SIGNATURE_SECRET`**: All requests pass through
 - **With `AB_JWT_SIGNATURE_SECRET`**: Requires valid JWT bearer token using HS256 algorithm
 
 ## OpenAPI Specification
@@ -98,6 +109,7 @@ curl -H "Authorization: Bearer <your-jwt-token>" \
 The manifest server provides an OpenAPI specification for API client generation:
 
 ### Generating the OpenAPI Spec
+
 ```bash
 # Generate OpenAPI YAML (default location)
 manifest-server generate-openapi
@@ -107,6 +119,7 @@ manifest-server generate-openapi --output /path/to/openapi.yaml
 ```
 
 The generated OpenAPI specification is consumed by other applications and tools to:
+
 - Generate API clients in various programming languages
 - Create SDK bindings for the manifest server
 - Provide API documentation and validation
@@ -115,6 +128,7 @@ The generated OpenAPI specification is consumed by other applications and tools
 ### Interactive API Documentation
 
 When running, interactive API documentation is available at:
+
 - Swagger UI: `http://localhost:8000/docs`
 - ReDoc: `http://localhost:8000/redoc`
 
@@ -139,4 +153,26 @@ docker build -f airbyte_cdk/manifest_server/Dockerfile -t manifest-server .
 docker run -p 8080:8080 manifest-server
 ```
 
-Note: The container runs on port 8080 by default.
+Note: The container runs on port 8080 by default.
+
+## Datadog APM
+
+The manifest server supports Datadog APM tracing for monitoring and observability:
+
+### Configuration
+
+To enable Datadog tracing, set the environment variable:
+
+```bash
+export DD_ENABLED=true
+```
+
+This requires the `ddtrace` dependency, which is included in the `manifest-server` extra. For additional configuration options via environment variables, see [ddtrace configuration](https://ddtrace.readthedocs.io/en/stable/configuration.html).
+
+### Usage
+
+```bash
+# Run with Datadog tracing enabled
+DD_ENABLED=true manifest-server start
+```
+

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/app.py

@@ -1,3 +1,9 @@
+import os
+
+if os.getenv("DD_ENABLED", "false").lower() == "true":
+    # Auto-instrumentation should be imported as early as possible.
+    import ddtrace.auto  # noqa: F401
+
 from fastapi import FastAPI
 
 from .routers import capabilities, health, manifest

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/cli/_common.py

@@ -7,6 +7,7 @@ import rich_click as click
 
 # Import server dependencies with graceful fallback
 try:
+    import ddtrace  # noqa: F401
     import fastapi  # noqa: F401
     import uvicorn  # noqa: F401
 

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/command_processor/processor.py

@@ -21,12 +21,10 @@ from airbyte_cdk.test.entrypoint_wrapper import AirbyteEntrypointException, Entr
 
 
 class ManifestCommandProcessor:
-    _source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
+    _source: ConcurrentDeclarativeSource
     _logger = logging.getLogger("airbyte.manifest-server")
 
-    def __init__(
-        self, source: ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]
-    ) -> None:
+    def __init__(self, source: ConcurrentDeclarativeSource) -> None:
         self._source = source
 
     def test_read(
@@ -41,7 +39,6 @@ class ManifestCommandProcessor:
         """
         Test the read method of the source.
         """
-
         test_read_handler = TestReader(
             max_pages_per_slice=page_limit,
             max_slices=slice_limit,

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/command_processor/utils.py

@@ -63,7 +63,7 @@ def build_source(
     record_limit: Optional[int] = None,
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+) -> ConcurrentDeclarativeSource:
     # We enforce a concurrency level of 1 so that the stream is processed on a single thread
     # to retain ordering for the grouping of the builder message responses.
     definition = copy.deepcopy(manifest)

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/manifest_server/routers/manifest.py

@@ -40,7 +40,7 @@ def safe_build_source(
     page_limit: Optional[int] = None,
     slice_limit: Optional[int] = None,
     record_limit: Optional[int] = None,
-) -> ConcurrentDeclarativeSource[Optional[List[AirbyteStateMessage]]]:
+) -> ConcurrentDeclarativeSource:
     """Wrapper around build_source that converts ValidationError to HTTPException."""
     try:
         return build_source(

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -162,16 +162,17 @@ def _get_declarative_component_schema() -> Dict[str, Any]:
 #  is no longer inherited from since the only external dependency is from that class.
 #
 # todo: It is worth investigating removal of the Generic[TState] since it will always be Optional[List[AirbyteStateMessage]]
-class ConcurrentDeclarativeSource(AbstractSource, Generic[TState]):
+class ConcurrentDeclarativeSource(AbstractSource):
     # By default, we defer to a value of 2. A value lower than could cause a PartitionEnqueuer to be stuck in a state of deadlock
     # because it has hit the limit of futures but not partition reader is consuming them.
     _LOWEST_SAFE_CONCURRENCY_LEVEL = 2
 
     def __init__(
         self,
-        catalog: Optional[ConfiguredAirbyteCatalog],
-        config: Optional[Mapping[str, Any]],
-        state: TState,
+        catalog: Optional[ConfiguredAirbyteCatalog] = None,
+        config: Optional[Mapping[str, Any]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
+        *,
         source_config: ConnectionDefinition,
         debug: bool = False,
         emit_connector_builder_messages: bool = False,

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py

@@ -149,6 +149,7 @@ class CartesianProductStreamSlicer(PartitionRouter):
         for stream_slice_tuple in product:
             partition = dict(ChainMap(*[s.partition for s in stream_slice_tuple]))  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
             cursor_slices = [s.cursor_slice for s in stream_slice_tuple if s.cursor_slice]
+            extra_fields = dict(ChainMap(*[s.extra_fields for s in stream_slice_tuple]))  # type: ignore # ChainMap expects a MutableMapping[Never, Never] for reasons
             if len(cursor_slices) > 1:
                 raise ValueError(
                     f"There should only be a single cursor slice. Found {cursor_slices}"
@@ -157,7 +158,9 @@ class CartesianProductStreamSlicer(PartitionRouter):
                 cursor_slice = cursor_slices[0]
             else:
                 cursor_slice = {}
-            yield StreamSlice(partition=partition, cursor_slice=cursor_slice)
+            yield StreamSlice(
+                partition=partition, cursor_slice=cursor_slice, extra_fields=extra_fields
+            )
 
     def set_initial_state(self, stream_state: StreamState) -> None:
         """

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -14,10 +14,21 @@ from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import Stre
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
+
 # For Connector Builder test read operations, we track the total number of records
-# read for the stream at the global level so that we can stop reading early if we
-# exceed the record limit
-total_record_counter = 0
+# read for the stream so that we can stop reading early if we exceed the record limit.
+class RecordCounter:
+    def __init__(self) -> None:
+        self.total_record_counter = 0
+
+    def increment(self) -> None:
+        self.total_record_counter += 1
+
+    def reset(self) -> None:
+        self.total_record_counter = 0
+
+    def get_total_records(self) -> int:
+        return self.total_record_counter
 
 
 class SchemaLoaderCachingDecorator(SchemaLoader):
@@ -51,6 +62,7 @@ class DeclarativePartitionFactory:
         self._retriever = retriever
         self._message_repository = message_repository
         self._max_records_limit = max_records_limit
+        self._record_counter = RecordCounter()
 
     def create(self, stream_slice: StreamSlice) -> Partition:
         return DeclarativePartition(
@@ -60,6 +72,7 @@ class DeclarativePartitionFactory:
             message_repository=self._message_repository,
             max_records_limit=self._max_records_limit,
             stream_slice=stream_slice,
+            record_counter=self._record_counter,
         )
 
 
@@ -72,6 +85,7 @@ class DeclarativePartition(Partition):
         message_repository: MessageRepository,
         max_records_limit: Optional[int],
         stream_slice: StreamSlice,
+        record_counter: RecordCounter,
     ):
         self._stream_name = stream_name
         self._schema_loader = schema_loader
@@ -80,17 +94,17 @@ class DeclarativePartition(Partition):
         self._max_records_limit = max_records_limit
         self._stream_slice = stream_slice
         self._hash = SliceHasher.hash(self._stream_name, self._stream_slice)
+        self._record_counter = record_counter
 
     def read(self) -> Iterable[Record]:
         if self._max_records_limit is not None:
-            global total_record_counter
-            if total_record_counter >= self._max_records_limit:
+            if self._record_counter.get_total_records() >= self._max_records_limit:
                 return
         for stream_data in self._retriever.read_records(
             self._schema_loader.get_json_schema(), self._stream_slice
         ):
             if self._max_records_limit is not None:
-                if total_record_counter >= self._max_records_limit:
+                if self._record_counter.get_total_records() >= self._max_records_limit:
                     break
 
             if isinstance(stream_data, Mapping):
@@ -108,7 +122,7 @@ class DeclarativePartition(Partition):
                 self._message_repository.emit_message(stream_data)
 
             if self._max_records_limit is not None:
-                total_record_counter += 1
+                self._record_counter.increment()
 
     def to_slice(self) -> Optional[Mapping[str, Any]]:
         return self._stream_slice

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -14,7 +14,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
 from airbyte_cdk.sources.types import ConnectionDefinition
 
 
-class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage]]):
+class YamlDeclarativeSource(ConcurrentDeclarativeSource):
     """Declarative source defined by a yaml file"""
 
     def __init__(

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/utils/schema_helpers.py

@@ -7,12 +7,16 @@ import importlib
 import json
 import os
 import pkgutil
-from typing import Any, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Tuple
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, MutableMapping, Tuple, cast
 
 import jsonref
-from jsonschema import RefResolver, validate
+from jsonschema import validate
 from jsonschema.exceptions import ValidationError
 from pydantic.v1 import BaseModel, Field
+from referencing import Registry, Resource
+from referencing._core import Resolver  # used for type hints
+from referencing.jsonschema import DRAFT7
 
 from airbyte_cdk.models import ConnectorSpecification, FailureType
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -63,18 +67,30 @@ def resolve_ref_links(obj: Any) -> Any:
         return obj
 
 
-def _expand_refs(schema: Any, ref_resolver: Optional[RefResolver] = None) -> None:
+def get_ref_resolver_registry(schema: dict[str, Any]) -> Registry:
+    """Get a reference resolver registry for the given schema."""
+    resource: Resource = Resource.from_contents(
+        contents=schema,
+        default_specification=DRAFT7,
+    )
+    return cast(  # Mypy has a hard time detecting this return type.
+        "Registry",
+        Registry().with_resource(
+            uri="",
+            resource=resource,
+        ),
+    )
+
+
+def _expand_refs(schema: Any, ref_resolver: Resolver) -> None:
     """Internal function to iterate over schema and replace all occurrences of $ref with their definitions. Recursive.
 
     :param schema: schema that will be patched
-    :param ref_resolver: resolver to get definition from $ref, if None pass it will be instantiated
     """
-    ref_resolver = ref_resolver or RefResolver.from_schema(schema)
-
     if isinstance(schema, MutableMapping):
         if "$ref" in schema:
             ref_url = schema.pop("$ref")
-            _, definition = ref_resolver.resolve(ref_url)
+            definition = ref_resolver.lookup(ref_url).contents
             _expand_refs(
                 definition, ref_resolver=ref_resolver
             )  # expand refs in definitions as well
@@ -90,10 +106,14 @@ def _expand_refs(schema: Any, ref_resolver: Optional[RefResolver] = None) -> Non
 def expand_refs(schema: Any) -> None:
     """Iterate over schema and replace all occurrences of $ref with their definitions.
 
+    If a "definitions" section is present at the root of the schema, it will be removed
+    after $ref resolution is complete.
+
     :param schema: schema that will be patched
     """
-    _expand_refs(schema)
-    schema.pop("definitions", None)  # remove definitions created by $ref
+    ref_resolver = get_ref_resolver_registry(schema).resolver()
+    _expand_refs(schema, ref_resolver)
+    schema.pop("definitions", None)
 
 
 def rename_key(schema: Any, old_key: str, new_key: str) -> None:

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/sources/utils/transform.py

@@ -3,10 +3,25 @@
 #
 
 import logging
+from copy import deepcopy
 from enum import Flag, auto
-from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
+
+from jsonschema import Draft7Validator, ValidationError, validators
+from referencing import Registry, Resource
+from referencing._core import Resolver
+from referencing.exceptions import Unresolvable
+from referencing.jsonschema import DRAFT7
+
+from airbyte_cdk.sources.utils.schema_helpers import expand_refs
+
+from .schema_helpers import get_ref_resolver_registry
+
+try:
+    from jsonschema.validators import Validator
+except:
+    from jsonschema import Validator
 
-from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
 
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
@@ -191,15 +206,14 @@ class TypeTransformer:
             validators parameter for detailed description.
             :
             """
+            # Very first step is to expand $refs in the schema itself:
+            expand_refs(schema)
+
+            # Now we can expand $refs in the property value:
+            if isinstance(property_value, dict):
+                expand_refs(property_value)
 
-            def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
-                if "$ref" in subschema:
-                    _, resolved = cast(
-                        RefResolver,
-                        validator_instance.resolver,
-                    ).resolve(subschema["$ref"])
-                    return cast(dict[str, Any], resolved)
-                return subschema
+            # Now we can validate and normalize the values:
 
             # Transform object and array values before running json schema type checking for each element.
             # Recursively normalize every value of the "instance" sub-object,
@@ -207,14 +221,12 @@ class TypeTransformer:
             if schema_key == "properties" and isinstance(instance, dict):
                 for k, subschema in property_value.items():
                     if k in instance:
-                        subschema = resolve(subschema)
                         instance[k] = self.__normalize(instance[k], subschema)
             # Recursively normalize every item of the "instance" sub-array,
             # if "instance" is an incorrect type - skip recursive normalization of "instance"
             elif schema_key == "items" and isinstance(instance, list):
-                subschema = resolve(property_value)
                 for index, item in enumerate(instance):
-                    instance[index] = self.__normalize(item, subschema)
+                    instance[index] = self.__normalize(item, property_value)
 
             # Running native jsonschema traverse algorithm after field normalization is done.
             yield from original_validator(

{airbyte_cdk-6.61.4 → airbyte_cdk-6.61.6}/airbyte_cdk/utils/spec_schema_transformations.py

@@ -6,7 +6,8 @@ import json
 import re
 from typing import Any
 
-from jsonschema import RefResolver
+from referencing import Registry, Resource
+from referencing.jsonschema import DRAFT7
 
 
 def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
@@ -14,13 +15,14 @@ def resolve_refs(schema: dict[str, Any]) -> dict[str, Any]:
     For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object
     relationships.
     """
-    json_schema_ref_resolver = RefResolver.from_schema(schema)
+    resource = Resource.from_contents(schema, default_specification=DRAFT7)
+    registry = Registry().with_resource("", resource)
+    resolver = registry.resolver()
     str_schema = json.dumps(schema)
     for ref_block in re.findall(r'{"\$ref": "#\/definitions\/.+?(?="})"}', str_schema):
         ref = json.loads(ref_block)["$ref"]
-        str_schema = str_schema.replace(
-            ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])
-        )
+        resolved = resolver.lookup(ref).contents
+        str_schema = str_schema.replace(ref_block, json.dumps(resolved))
     pyschema: dict[str, Any] = json.loads(str_schema)
     del pyschema["definitions"]
     return pyschema