airbyte-cdk 7.4.0.post2.dev18763111899__tar.gz → 7.4.1__tar.gz

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.4.0.post2.dev18763111899
+Version: 7.4.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -3353,7 +3353,7 @@ definitions:
       - ["code", "type"]
   PropertiesFromEndpoint:
     title: Properties from Endpoint
-    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records.
+    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records. Note that stream_slices can't be interpolated from this retriever.
     type: object
     required:
       - type

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -551,9 +551,13 @@ from airbyte_cdk.sources.declarative.schema import (
     DynamicSchemaLoader,
     InlineSchemaLoader,
     JsonFileSchemaLoader,
+    SchemaLoader,
     SchemaTypeIdentifier,
     TypesMap,
 )
+from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import (
+    CachingSchemaLoaderDecorator,
+)
 from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader
 from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec
 from airbyte_cdk.sources.declarative.stream_slicers import (
@@ -2095,13 +2099,7 @@ class ModelToComponentFactory:
         if isinstance(retriever, AsyncRetriever):
             stream_slicer = retriever.stream_slicer
 
-        schema_loader: Union[
-            CompositeSchemaLoader,
-            DefaultSchemaLoader,
-            DynamicSchemaLoader,
-            InlineSchemaLoader,
-            JsonFileSchemaLoader,
-        ]
+        schema_loader: SchemaLoader
         if model.schema_loader and isinstance(model.schema_loader, list):
             nested_schema_loaders = [
                 self._create_component_from_model(model=nested_schema_loader, config=config)
@@ -2120,6 +2118,7 @@ class ModelToComponentFactory:
             if "name" not in options:
                 options["name"] = model.name
             schema_loader = DefaultSchemaLoader(config=config, parameters=options)
+        schema_loader = CachingSchemaLoaderDecorator(schema_loader)
 
         stream_name = model.name or ""
         return DefaultStream(

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py

@@ -22,19 +22,27 @@ class PropertiesFromEndpoint:
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
+    _cached_properties: Optional[List[str]] = None
+
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._property_field_path = [
             InterpolatedString(string=property_field, parameters=parameters)
             for property_field in self.property_field_path
         ]
 
-    def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> Iterable[str]:
-        response_properties = self.retriever.read_records(
-            records_schema={}, stream_slice=stream_slice
-        )
-        for property_obj in response_properties:
-            path = [
-                node.eval(self.config) if not isinstance(node, str) else node
-                for node in self._property_field_path
-            ]
-            yield dpath.get(property_obj, path, default=[])  # type: ignore # extracted will be a MutableMapping, given input data structure
+    def get_properties_from_endpoint(self) -> List[str]:
+        if self._cached_properties is None:
+            self._cached_properties = list(
+                map(
+                    self._get_property,  # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records?
+                    self.retriever.read_records(records_schema={}, stream_slice=None),
+                )
+            )
+        return self._cached_properties
+
+    def _get_property(self, property_obj: Mapping[str, Any]) -> str:
+        path = [
+            node.eval(self.config) if not isinstance(node, str) else node
+            for node in self._property_field_path
+        ]
+        return str(dpath.get(property_obj, path, default=[]))  # type: ignore # extracted will be a MutableMapping, given input data structure

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py

@@ -41,7 +41,7 @@ class PropertyChunking:
 
     def get_request_property_chunks(
         self,
-        property_fields: Iterable[str],
+        property_fields: List[str],
         always_include_properties: Optional[List[str]],
         configured_properties: Optional[Set[str]],
     ) -> Iterable[List[str]]:

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py

@@ -30,23 +30,16 @@ class QueryProperties:
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
-    def get_request_property_chunks(
-        self,
-        stream_slice: Optional[StreamSlice] = None,
-    ) -> Iterable[List[str]]:
+    def get_request_property_chunks(self) -> Iterable[List[str]]:
         """
         Uses the defined property_list to fetch the total set of properties dynamically or from a static list
         and based on the resulting properties, performs property chunking if applicable.
-        :param stream_slice: The StreamSlice of the current partition being processed during the sync. This is included
-        because subcomponents of QueryProperties can make use of interpolation of the top-level StreamSlice object
-        :param configured_stream: The customer configured stream being synced which is needed to identify which
-        record fields to query for and emit.
         """
+        fields: List[str]
         configured_properties = self.property_selector.select() if self.property_selector else None
 
-        fields: Union[Iterable[str], List[str]]
         if isinstance(self.property_list, PropertiesFromEndpoint):
-            fields = self.property_list.get_properties_from_endpoint(stream_slice=stream_slice)
+            fields = self.property_list.get_properties_from_endpoint()
         else:
             fields = self.property_list if self.property_list else []
 

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -385,9 +385,9 @@ class SimpleRetriever(Retriever):
             response = None
             try:
                 if self.additional_query_properties:
-                    for properties in self.additional_query_properties.get_request_property_chunks(
-                        stream_slice=stream_slice,
-                    ):
+                    for (
+                        properties
+                    ) in self.additional_query_properties.get_request_property_chunks():
                         stream_slice = StreamSlice(
                             partition=stream_slice.partition or {},
                             cursor_slice=stream_slice.cursor_slice or {},
@@ -523,7 +523,6 @@ class SimpleRetriever(Retriever):
         """
         _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
 
-        most_recent_record_from_slice = None
         record_generator = partial(
             self._parse_records,
             stream_slice=stream_slice,

airbyte_cdk-7.4.1/airbyte_cdk/sources/declarative/schema/caching_schema_loader_decorator.py

@@ -0,0 +1,15 @@
+from typing import Any, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.schema import SchemaLoader
+
+
+class CachingSchemaLoaderDecorator(SchemaLoader):
+    def __init__(self, schema_loader: SchemaLoader):
+        self._decorated = schema_loader
+        self._loaded_schema: Optional[Mapping[str, Any]] = None
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        if self._loaded_schema is None:
+            self._loaded_schema = self._decorated.get_json_schema()
+
+        return self._loaded_schema  # type: ignore  # at that point, we assume the schema will be populated

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -31,18 +31,6 @@ class RecordCounter:
         return self.total_record_counter
 
 
-class SchemaLoaderCachingDecorator(SchemaLoader):
-    def __init__(self, schema_loader: SchemaLoader):
-        self._decorated = schema_loader
-        self._loaded_schema: Optional[Mapping[str, Any]] = None
-
-    def get_json_schema(self) -> Mapping[str, Any]:
-        if self._loaded_schema is None:
-            self._loaded_schema = self._decorated.get_json_schema()
-
-        return self._loaded_schema  # type: ignore  # at that point, we assume the schema will be populated
-
-
 class DeclarativePartitionFactory:
     def __init__(
         self,
@@ -58,7 +46,7 @@ class DeclarativePartitionFactory:
         In order to avoid these problems, we will create one retriever per thread which should make the processing thread-safe.
         """
         self._stream_name = stream_name
-        self._schema_loader = SchemaLoaderCachingDecorator(schema_loader)
+        self._schema_loader = schema_loader
         self._retriever = retriever
         self._message_repository = message_repository
         self._max_records_limit = max_records_limit

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/utils/transform.py

@@ -3,20 +3,11 @@
 #
 
 import logging
-from copy import deepcopy
 from enum import Flag, auto
-from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
+from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
 
 from jsonschema import Draft7Validator, ValidationError, validators
 from jsonschema.protocols import Validator
-from referencing import Registry, Resource
-from referencing._core import Resolver
-from referencing.exceptions import Unresolvable
-from referencing.jsonschema import DRAFT7
-
-from airbyte_cdk.sources.utils.schema_helpers import expand_refs
-
-from .schema_helpers import get_ref_resolver_registry
 
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
@@ -201,15 +192,6 @@ class TypeTransformer:
             validators parameter for detailed description.
             :
             """
-            # Very first step is to expand $refs in the schema itself:
-            expand_refs(schema)
-
-            # Now we can expand $refs in the property value:
-            if isinstance(property_value, dict):
-                expand_refs(property_value)
-
-            # Now we can validate and normalize the values:
-
             # Transform object and array values before running json schema type checking for each element.
             # Recursively normalize every value of the "instance" sub-object,
             # if "instance" is an incorrect type - skip recursive normalization of "instance"

{airbyte_cdk-7.4.0.post2.dev18763111899 → airbyte_cdk-7.4.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.4.0.post2.dev18763111899"  # Version will be calculated dynamically.
+version = "7.4.1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false