airbyte-cdk 7.4.0__tar.gz → 7.4.1__tar.gz

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.4.0
+Version: 7.4.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -45,8 +45,9 @@ Requires-Dist: google-cloud-secret-manager (>=2.17.0,<3.0.0)
 Requires-Dist: isodate (>=0.6.1,<0.7.0)
 Requires-Dist: jsonref (>=1,<2)
 Requires-Dist: jsonschema (>=4.17.3,<5.0)
-Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
-Requires-Dist: langchain_core (==0.1.42)
+Requires-Dist: langchain_community (>=0.4,<0.5) ; extra == "vector-db-based"
+Requires-Dist: langchain_core (>=1.0.0,<2.0.0) ; extra == "vector-db-based"
+Requires-Dist: langchain_text_splitters (>=1.0.0,<2.0.0) ; extra == "vector-db-based"
 Requires-Dist: markdown ; extra == "file-based"
 Requires-Dist: nltk (==3.9.1)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -8,9 +8,9 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Mapping, Optional, Tuple
 
 import dpath
-from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
-from langchain.utils import stringify_dict
 from langchain_core.documents.base import Document
+from langchain_core.utils.strings import stringify_dict
+from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 
 from airbyte_cdk.destinations.vector_db_based.config import (
     ProcessingConfigModel,

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/destinations/vector_db_based/embedder.py

@@ -7,10 +7,12 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import List, Optional, Union, cast
 
-from langchain.embeddings.cohere import CohereEmbeddings
-from langchain.embeddings.fake import FakeEmbeddings
-from langchain.embeddings.localai import LocalAIEmbeddings
-from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain_community.embeddings import (
+    CohereEmbeddings,
+    FakeEmbeddings,
+    LocalAIEmbeddings,
+    OpenAIEmbeddings,
+)
 
 from airbyte_cdk.destinations.vector_db_based.config import (
     AzureOpenAIEmbeddingConfigModel,
@@ -140,7 +142,9 @@ class CohereEmbedder(Embedder):
         super().__init__()
         # Client is set internally
         self.embeddings = CohereEmbeddings(
-            cohere_api_key=config.cohere_key, model="embed-english-light-v2.0"
+            cohere_api_key=config.cohere_key,
+            model="embed-english-light-v2.0",
+            user_agent="airbyte-cdk",
         )  # type: ignore
 
     def check(self) -> Optional[str]:

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/manifest_server/Dockerfile

@@ -5,7 +5,7 @@
 # Example:
 # docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
 
-FROM python:3.12-slim-bookworm
+FROM python:3.13.9-slim-bookworm@sha256:4c9fe962f6ce46ecf3633a7e9d0a9fb7f5622121ee00d628eff206da024147c9
 
 # Install git (needed for dynamic versioning) and poetry
 RUN apt-get update && \

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -3353,7 +3353,7 @@ definitions:
       - ["code", "type"]
   PropertiesFromEndpoint:
     title: Properties from Endpoint
-    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records.
+    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records. Note that stream_slices can't be interpolated from this retriever.
     type: object
     required:
       - type

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -551,9 +551,13 @@ from airbyte_cdk.sources.declarative.schema import (
     DynamicSchemaLoader,
     InlineSchemaLoader,
     JsonFileSchemaLoader,
+    SchemaLoader,
     SchemaTypeIdentifier,
     TypesMap,
 )
+from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import (
+    CachingSchemaLoaderDecorator,
+)
 from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader
 from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec
 from airbyte_cdk.sources.declarative.stream_slicers import (
@@ -2095,13 +2099,7 @@ class ModelToComponentFactory:
         if isinstance(retriever, AsyncRetriever):
             stream_slicer = retriever.stream_slicer
 
-        schema_loader: Union[
-            CompositeSchemaLoader,
-            DefaultSchemaLoader,
-            DynamicSchemaLoader,
-            InlineSchemaLoader,
-            JsonFileSchemaLoader,
-        ]
+        schema_loader: SchemaLoader
         if model.schema_loader and isinstance(model.schema_loader, list):
             nested_schema_loaders = [
                 self._create_component_from_model(model=nested_schema_loader, config=config)
@@ -2120,6 +2118,7 @@ class ModelToComponentFactory:
             if "name" not in options:
                 options["name"] = model.name
             schema_loader = DefaultSchemaLoader(config=config, parameters=options)
+        schema_loader = CachingSchemaLoaderDecorator(schema_loader)
 
         stream_name = model.name or ""
         return DefaultStream(

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py

@@ -22,19 +22,27 @@ class PropertiesFromEndpoint:
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
+    _cached_properties: Optional[List[str]] = None
+
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._property_field_path = [
             InterpolatedString(string=property_field, parameters=parameters)
             for property_field in self.property_field_path
         ]
 
-    def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> Iterable[str]:
-        response_properties = self.retriever.read_records(
-            records_schema={}, stream_slice=stream_slice
-        )
-        for property_obj in response_properties:
-            path = [
-                node.eval(self.config) if not isinstance(node, str) else node
-                for node in self._property_field_path
-            ]
-            yield dpath.get(property_obj, path, default=[])  # type: ignore # extracted will be a MutableMapping, given input data structure
+    def get_properties_from_endpoint(self) -> List[str]:
+        if self._cached_properties is None:
+            self._cached_properties = list(
+                map(
+                    self._get_property,  # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records?
+                    self.retriever.read_records(records_schema={}, stream_slice=None),
+                )
+            )
+        return self._cached_properties
+
+    def _get_property(self, property_obj: Mapping[str, Any]) -> str:
+        path = [
+            node.eval(self.config) if not isinstance(node, str) else node
+            for node in self._property_field_path
+        ]
+        return str(dpath.get(property_obj, path, default=[]))  # type: ignore # extracted will be a MutableMapping, given input data structure

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py

@@ -41,7 +41,7 @@ class PropertyChunking:
 
     def get_request_property_chunks(
         self,
-        property_fields: Iterable[str],
+        property_fields: List[str],
         always_include_properties: Optional[List[str]],
         configured_properties: Optional[Set[str]],
     ) -> Iterable[List[str]]:

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py

@@ -30,23 +30,16 @@ class QueryProperties:
     config: Config
     parameters: InitVar[Mapping[str, Any]]
 
-    def get_request_property_chunks(
-        self,
-        stream_slice: Optional[StreamSlice] = None,
-    ) -> Iterable[List[str]]:
+    def get_request_property_chunks(self) -> Iterable[List[str]]:
         """
         Uses the defined property_list to fetch the total set of properties dynamically or from a static list
         and based on the resulting properties, performs property chunking if applicable.
-        :param stream_slice: The StreamSlice of the current partition being processed during the sync. This is included
-        because subcomponents of QueryProperties can make use of interpolation of the top-level StreamSlice object
-        :param configured_stream: The customer configured stream being synced which is needed to identify which
-        record fields to query for and emit.
         """
+        fields: List[str]
         configured_properties = self.property_selector.select() if self.property_selector else None
 
-        fields: Union[Iterable[str], List[str]]
         if isinstance(self.property_list, PropertiesFromEndpoint):
-            fields = self.property_list.get_properties_from_endpoint(stream_slice=stream_slice)
+            fields = self.property_list.get_properties_from_endpoint()
         else:
             fields = self.property_list if self.property_list else []
 

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -385,9 +385,9 @@ class SimpleRetriever(Retriever):
             response = None
             try:
                 if self.additional_query_properties:
-                    for properties in self.additional_query_properties.get_request_property_chunks(
-                        stream_slice=stream_slice,
-                    ):
+                    for (
+                        properties
+                    ) in self.additional_query_properties.get_request_property_chunks():
                         stream_slice = StreamSlice(
                             partition=stream_slice.partition or {},
                             cursor_slice=stream_slice.cursor_slice or {},
@@ -523,7 +523,6 @@ class SimpleRetriever(Retriever):
         """
         _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
 
-        most_recent_record_from_slice = None
         record_generator = partial(
             self._parse_records,
             stream_slice=stream_slice,

airbyte_cdk-7.4.1/airbyte_cdk/sources/declarative/schema/caching_schema_loader_decorator.py

@@ -0,0 +1,15 @@
+from typing import Any, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.schema import SchemaLoader
+
+
+class CachingSchemaLoaderDecorator(SchemaLoader):
+    def __init__(self, schema_loader: SchemaLoader):
+        self._decorated = schema_loader
+        self._loaded_schema: Optional[Mapping[str, Any]] = None
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        if self._loaded_schema is None:
+            self._loaded_schema = self._decorated.get_json_schema()
+
+        return self._loaded_schema  # type: ignore  # at that point, we assume the schema will be populated

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -31,18 +31,6 @@ class RecordCounter:
         return self.total_record_counter
 
 
-class SchemaLoaderCachingDecorator(SchemaLoader):
-    def __init__(self, schema_loader: SchemaLoader):
-        self._decorated = schema_loader
-        self._loaded_schema: Optional[Mapping[str, Any]] = None
-
-    def get_json_schema(self) -> Mapping[str, Any]:
-        if self._loaded_schema is None:
-            self._loaded_schema = self._decorated.get_json_schema()
-
-        return self._loaded_schema  # type: ignore  # at that point, we assume the schema will be populated
-
-
 class DeclarativePartitionFactory:
     def __init__(
         self,
@@ -58,7 +46,7 @@ class DeclarativePartitionFactory:
         In order to avoid these problems, we will create one retriever per thread which should make the processing thread-safe.
         """
         self._stream_name = stream_name
-        self._schema_loader = SchemaLoaderCachingDecorator(schema_loader)
+        self._schema_loader = schema_loader
         self._retriever = retriever
         self._message_repository = message_repository
         self._max_records_limit = max_records_limit

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/airbyte_cdk/sources/utils/transform.py

@@ -3,20 +3,11 @@
 #
 
 import logging
-from copy import deepcopy
 from enum import Flag, auto
-from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Mapping, Optional, cast
+from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
 
 from jsonschema import Draft7Validator, ValidationError, validators
 from jsonschema.protocols import Validator
-from referencing import Registry, Resource
-from referencing._core import Resolver
-from referencing.exceptions import Unresolvable
-from referencing.jsonschema import DRAFT7
-
-from airbyte_cdk.sources.utils.schema_helpers import expand_refs
-
-from .schema_helpers import get_ref_resolver_registry
 
 MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
@@ -201,15 +192,6 @@ class TypeTransformer:
             validators parameter for detailed description.
             :
             """
-            # Very first step is to expand $refs in the schema itself:
-            expand_refs(schema)
-
-            # Now we can expand $refs in the property value:
-            if isinstance(property_value, dict):
-                expand_refs(property_value)
-
-            # Now we can validate and normalize the values:
-
             # Transform object and array values before running json schema type checking for each element.
             # Recursively normalize every value of the "instance" sub-object,
             # if "instance" is an incorrect type - skip recursive normalization of "instance"

{airbyte_cdk-7.4.0 → airbyte_cdk-7.4.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "7.4.0"  # Version will be calculated dynamically.
+version = "7.4.1"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -64,8 +64,9 @@ wcmatch = "10.0"
 avro = { version = ">=1.11.2,<1.13.0", optional = true }  # TODO: Move into dev dependencies if only used in tests
 cohere = { version = ">=4.21,<6.0.0", optional = true }
 fastavro = { version = ">=1.11.0,<2.0.0", optional = true }
-langchain = { version = "0.1.16", optional = true }
-langchain_core = { version = "0.1.42", optional = true }
+langchain_community = { version = "^0.4", optional = true }
+langchain_core = { version = "^1.0.0", optional = true }
+langchain_text_splitters = { version = "^1.0.0", optional = true }
 markdown = { version = "*", optional = true }  # TODO: Remove if unused
 openai = { version = "0.27.9", extras = ["embeddings"], optional = true }  # Used indirectly by langchain library
 pdf2image = { version = "1.16.3", optional = true }
@@ -121,7 +122,7 @@ dagger-io = "0.19.0"
 
 [tool.poetry.extras]
 file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
-vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
+vector-db-based = ["langchain_community", "langchain_core", "langchain_text_splitters", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 dev = ["pytest"]
 manifest-server = ["fastapi", "uvicorn", "ddtrace"]