airbyte-cdk 6.9.0.dev0__tar.gz → 6.9.1.dev0__tar.gz

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.9.0.dev0
+Version: 6.9.1.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -18,12 +18,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: file-based
-Provides-Extra: sphinx-docs
 Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: Sphinx (>=4.2,<4.3) ; extra == "sphinx-docs"
 Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
 Requires-Dist: backoff
@@ -63,7 +61,6 @@ Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
 Requires-Dist: requests
 Requires-Dist: requests_cache
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
-Requires-Dist: sphinx-rtd-theme (>=1.0,<1.1) ; extra == "sphinx-docs"
 Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
 Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
 Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -86,23 +86,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             component_factory=component_factory,
         )
 
+        # todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
+        #  no longer needs to store the original incoming state. But maybe there's an edge case?
         self._state = state
 
-        self._concurrent_streams: Optional[List[AbstractStream]]
-        self._synchronous_streams: Optional[List[Stream]]
-
-        # If the connector command was SPEC, there is no incoming config, and we cannot instantiate streams because
-        # they might depend on it. Ideally we want to have a static method on this class to get the spec without
-        # any other arguments, but the existing entrypoint.py isn't designed to support this. Just noting this
-        # for our future improvements to the CDK.
-        if config:
-            self._concurrent_streams, self._synchronous_streams = self._group_streams(
-                config=config or {}
-            )
-        else:
-            self._concurrent_streams = None
-            self._synchronous_streams = None
-
         concurrency_level_from_manifest = self._source_config.get("concurrency_level")
         if concurrency_level_from_manifest:
             concurrency_level_component = self._constructor.create_component(
@@ -136,17 +123,20 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         logger: logging.Logger,
         config: Mapping[str, Any],
         catalog: ConfiguredAirbyteCatalog,
-        state: Optional[Union[List[AirbyteStateMessage]]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
     ) -> Iterator[AirbyteMessage]:
-        # ConcurrentReadProcessor pops streams that are finished being read so before syncing, the names of the concurrent
-        # streams must be saved so that they can be removed from the catalog before starting synchronous streams
-        if self._concurrent_streams:
+        concurrent_streams, _ = self._group_streams(config=config)
+
+        # ConcurrentReadProcessor pops streams that are finished being read so before syncing, the names of
+        # the concurrent streams must be saved so that they can be removed from the catalog before starting
+        # synchronous streams
+        if len(concurrent_streams) > 0:
             concurrent_stream_names = set(
-                [concurrent_stream.name for concurrent_stream in self._concurrent_streams]
+                [concurrent_stream.name for concurrent_stream in concurrent_streams]
             )
 
             selected_concurrent_streams = self._select_streams(
-                streams=self._concurrent_streams, configured_catalog=catalog
+                streams=concurrent_streams, configured_catalog=catalog
             )
             # It would appear that passing in an empty set of streams causes an infinite loop in ConcurrentReadProcessor.
             # This is also evident in concurrent_source_adapter.py so I'll leave this out of scope to fix for now
@@ -165,8 +155,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         yield from super().read(logger, config, filtered_catalog, state)
 
     def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
-        concurrent_streams = self._concurrent_streams or []
-        synchronous_streams = self._synchronous_streams or []
+        concurrent_streams, synchronous_streams = self._group_streams(config=config)
         return AirbyteCatalog(
             streams=[
                 stream.as_airbyte_stream() for stream in concurrent_streams + synchronous_streams
@@ -206,7 +195,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # so we need to treat them as synchronous
             if (
                 isinstance(declarative_stream, DeclarativeStream)
-                and name_to_stream_mapping[declarative_stream.name].get("retriever")["type"]
+                and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
                 == "SimpleRetriever"
             ):
                 incremental_sync_component_definition = name_to_stream_mapping[
@@ -215,7 +204,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
                 partition_router_component_definition = (
                     name_to_stream_mapping[declarative_stream.name]
-                    .get("retriever")
+                    .get("retriever", {})
                     .get("partition_router")
                 )
                 is_without_partition_router_or_cursor = not bool(
@@ -237,7 +226,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     cursor = self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
                         state_manager=state_manager,
                         model_type=DatetimeBasedCursorModel,
-                        component_definition=incremental_sync_component_definition,
+                        component_definition=incremental_sync_component_definition,  # type: ignore  # Not None because of the if condition above
                         stream_name=declarative_stream.name,
                         stream_namespace=declarative_stream.namespace,
                         config=config or {},
@@ -320,10 +309,11 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
     def _is_datetime_incremental_without_partition_routing(
         self,
         declarative_stream: DeclarativeStream,
-        incremental_sync_component_definition: Mapping[str, Any],
+        incremental_sync_component_definition: Mapping[str, Any] | None,
     ) -> bool:
         return (
-            bool(incremental_sync_component_definition)
+            incremental_sync_component_definition is not None
+            and bool(incremental_sync_component_definition)
             and incremental_sync_component_definition.get("type", "")
             == DatetimeBasedCursorModel.__name__
             and self._stream_supports_concurrent_partition_processing(

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1684,92 +1684,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  TypesMap:
-    title: Types Map
-    description: (This component is experimental. Use at your own risk.) Represents a mapping between a current type and its corresponding target type.
-    type: object
-    required:
-      - target_type
-      - current_type
-    properties:
-      target_type:
-        anyOf:
-          - type: string
-          - type: array
-            items:
-              type: string
-      current_type:
-        anyOf:
-          - type: string
-          - type: array
-            items:
-              type: string
-  SchemaTypeIdentifier:
-    title: Schema Type Identifier
-    description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing.
-    type: object
-    required:
-      - key_pointer
-    properties:
-      type:
-        type: string
-        enum: [SchemaTypeIdentifier]
-      schema_pointer:
-        title: Schema Path
-        description: List of nested fields defining the schema field path to extract. Defaults to [].
-        type: array
-        default: []
-        items:
-          - type: string
-        interpolation_content:
-          - config
-      key_pointer:
-        title: Key Path
-        description: List of potentially nested fields describing the full path of the field key to extract.
-        type: array
-        items:
-          - type: string
-        interpolation_content:
-          - config
-      type_pointer:
-        title: Type Path
-        description: List of potentially nested fields describing the full path of the field type to extract.
-        type: array
-        items:
-          - type: string
-        interpolation_content:
-          - config
-      types_mapping:
-        type: array
-        items:
-          - "$ref": "#/definitions/TypesMap"
-      $parameters:
-        type: object
-        additionalProperties: true
-  DynamicSchemaLoader:
-    title: Dynamic Schema Loader
-    description: (This component is experimental. Use at your own risk.) Loads a schema by extracting data from retrieved records.
-    type: object
-    required:
-      - type
-      - retriever
-      - schema_type_identifier
-    properties:
-      type:
-        type: string
-        enum: [DynamicSchemaLoader]
-      retriever:
-        title: Retriever
-        description: Component used to coordinate how records are extracted across stream slices and request pages.
-        anyOf:
-          - "$ref": "#/definitions/AsyncRetriever"
-          - "$ref": "#/definitions/CustomRetriever"
-          - "$ref": "#/definitions/SimpleRetriever"
-      schema_type_identifier:
-        "$ref": "#/definitions/SchemaTypeIdentifier"
-      $parameters:
-        type: object
-        additionalProperties: true
   InlineSchemaLoader:
     title: Inline Schema Loader
     description: Loads a schema that is defined directly in the manifest file.

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -650,32 +650,6 @@ class HttpResponseFilter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class TypesMap(BaseModel):
-    target_type: Union[str, List[str]]
-    current_type: Union[str, List[str]]
-
-
-class SchemaTypeIdentifier(BaseModel):
-    type: Optional[Literal["SchemaTypeIdentifier"]] = None
-    schema_pointer: Optional[List[str]] = Field(
-        [],
-        description="List of nested fields defining the schema field path to extract. Defaults to [].",
-        title="Schema Path",
-    )
-    key_pointer: List[str] = Field(
-        ...,
-        description="List of potentially nested fields describing the full path of the field key to extract.",
-        title="Key Path",
-    )
-    type_pointer: Optional[List[str]] = Field(
-        None,
-        description="List of potentially nested fields describing the full path of the field type to extract.",
-        title="Type Path",
-    )
-    types_mapping: Optional[List[TypesMap]] = None
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class InlineSchemaLoader(BaseModel):
     type: Literal["InlineSchemaLoader"]
     schema_: Optional[Dict[str, Any]] = Field(
@@ -848,13 +822,13 @@ class OauthConnectorInputSpecification(BaseModel):
     )
     extract_output: List[str] = Field(
         ...,
-        description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
+        description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.                ",
         examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
         title="DeclarativeOAuth Extract Output",
     )
     state: Optional[State] = Field(
         None,
-        description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
+        description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.                ",
         examples=[{"state": {"min": 7, "max": 128}}],
         title="(Optional) DeclarativeOAuth Configurable State Query Param",
     )
@@ -878,13 +852,13 @@ class OauthConnectorInputSpecification(BaseModel):
     )
     state_key: Optional[str] = Field(
         None,
-        description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
+        description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.                ",
         examples=[{"state_key": "my_custom_state_key_key_name"}],
         title="(Optional) DeclarativeOAuth State Key Override",
     )
     auth_code_key: Optional[str] = Field(
         None,
-        description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
+        description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.                ",
         examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
         title="(Optional) DeclarativeOAuth Auth Code Key Override",
     )
@@ -1800,17 +1774,6 @@ class HttpRequester(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class DynamicSchemaLoader(BaseModel):
-    type: Literal["DynamicSchemaLoader"]
-    retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
-        ...,
-        description="Component used to coordinate how records are extracted across stream slices and request pages.",
-        title="Retriever",
-    )
-    schema_type_identifier: SchemaTypeIdentifier
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class ParentStreamConfig(BaseModel):
     type: Literal["ParentStreamConfig"]
     parent_key: str = Field(
@@ -2018,6 +1981,5 @@ DeclarativeSource2.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
-DynamicSchemaLoader.update_forward_refs()
 SimpleRetriever.update_forward_refs()
 AsyncRetriever.update_forward_refs()

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -64,10 +64,6 @@ DEFAULT_MODEL_TYPES: Mapping[str, str] = {
     "AddFields.fields": "AddedFieldDefinition",
     # CustomPartitionRouter
     "CustomPartitionRouter.parent_stream_configs": "ParentStreamConfig",
-    # DynamicSchemaLoader
-    "DynamicSchemaLoader.retriever": "SimpleRetriever",
-    # SchemaTypeIdentifier
-    "SchemaTypeIdentifier.types_map": "TypesMap",
 }
 
 # We retain a separate registry for custom components to automatically insert the type if it is missing. This is intended to

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -188,9 +188,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DpathExtractor as DpathExtractorModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    DynamicSchemaLoader as DynamicSchemaLoaderModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
@@ -281,9 +278,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ResponseToFileExtractor as ResponseToFileExtractorModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    SchemaTypeIdentifier as SchemaTypeIdentifierModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SelectiveAuthenticator as SelectiveAuthenticatorModel,
 )
@@ -297,9 +291,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SubstreamPartitionRouter as SubstreamPartitionRouterModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    TypesMap as TypesMapModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     WaitTimeFromHeader as WaitTimeFromHeaderModel,
@@ -365,11 +356,8 @@ from airbyte_cdk.sources.declarative.retrievers import (
 )
 from airbyte_cdk.sources.declarative.schema import (
     DefaultSchemaLoader,
-    DynamicSchemaLoader,
     InlineSchemaLoader,
     JsonFileSchemaLoader,
-    SchemaTypeIdentifier,
-    TypesMap,
 )
 from airbyte_cdk.sources.declarative.spec import Spec
 from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
@@ -467,9 +455,6 @@ class ModelToComponentFactory:
             IterableDecoderModel: self.create_iterable_decoder,
             XmlDecoderModel: self.create_xml_decoder,
             JsonFileSchemaLoaderModel: self.create_json_file_schema_loader,
-            DynamicSchemaLoaderModel: self.create_dynamic_schema_loader,
-            SchemaTypeIdentifierModel: self.create_schema_type_identifier,
-            TypesMapModel: self.create_types_map,
             JwtAuthenticatorModel: self.create_jwt_authenticator,
             LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration,
             ListPartitionRouterModel: self.create_list_partition_router,
@@ -1589,63 +1574,6 @@ class ModelToComponentFactory:
     ) -> InlineSchemaLoader:
         return InlineSchemaLoader(schema=model.schema_ or {}, parameters={})
 
-    @staticmethod
-    def create_types_map(model: TypesMapModel, **kwargs: Any) -> TypesMap:
-        return TypesMap(target_type=model.target_type, current_type=model.current_type)
-
-    def create_schema_type_identifier(
-        self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any
-    ) -> SchemaTypeIdentifier:
-        types_mapping = []
-        if model.types_mapping:
-            types_mapping.extend(
-                [
-                    self._create_component_from_model(types_map, config=config)
-                    for types_map in model.types_mapping
-                ]
-            )
-        model_schema_pointer: List[Union[InterpolatedString, str]] = (
-            [x for x in model.schema_pointer] if model.schema_pointer else []
-        )
-        model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer]
-        model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = (
-            [x for x in model.type_pointer] if model.type_pointer else None
-        )
-
-        return SchemaTypeIdentifier(
-            schema_pointer=model_schema_pointer,
-            key_pointer=model_key_pointer,
-            type_pointer=model_type_pointer,
-            types_mapping=types_mapping,
-            parameters=model.parameters or {},
-        )
-
-    def create_dynamic_schema_loader(
-        self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any
-    ) -> DynamicSchemaLoader:
-        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
-        combined_slicers = self._build_resumable_cursor_from_paginator(
-            model.retriever, stream_slicer
-        )
-
-        retriever = self._create_component_from_model(
-            model=model.retriever,
-            config=config,
-            name="",
-            primary_key=None,
-            stream_slicer=combined_slicers,
-            transformations=[],
-        )
-        schema_type_identifier = self._create_component_from_model(
-            model.schema_type_identifier, config=config, parameters=model.parameters or {}
-        )
-        return DynamicSchemaLoader(
-            retriever=retriever,
-            config=config,
-            schema_type_identifier=schema_type_identifier,
-            parameters=model.parameters or {},
-        )
-
     @staticmethod
     def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder:
         return JsonDecoder(parameters={})

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/airbyte_cdk/sources/declarative/schema/__init__.py

@@ -6,6 +6,5 @@ from airbyte_cdk.sources.declarative.schema.default_schema_loader import Default
 from airbyte_cdk.sources.declarative.schema.inline_schema_loader import InlineSchemaLoader
 from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
-from airbyte_cdk.sources.declarative.schema.dynamic_schema_loader import DynamicSchemaLoader, TypesMap, SchemaTypeIdentifier
 
-__all__ = ["JsonFileSchemaLoader", "DefaultSchemaLoader", "SchemaLoader", "InlineSchemaLoader", "DynamicSchemaLoader", "TypesMap", "SchemaTypeIdentifier"]
+__all__ = ["JsonFileSchemaLoader", "DefaultSchemaLoader", "SchemaLoader", "InlineSchemaLoader"]

{airbyte_cdk-6.9.0.dev0 → airbyte_cdk-6.9.1.dev0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.9.0dev0"  # Version will be calculated dynamically.
+version = "6.9.1dev0"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -66,8 +66,6 @@ pyarrow = { version = "~15.0.0", optional = true }
 pytesseract = { version = "0.3.10", optional = true }
 python-calamine = { version = "0.2.3", optional = true }
 python-snappy = { version = "0.7.3", optional = true }
-Sphinx = { version = "~4.2", optional = true }
-sphinx-rtd-theme = { version = "~1.0", optional = true }
 tiktoken = { version = "0.8.0", optional = true }
 nltk = { version = "3.9.1", optional = true }
 # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
@@ -106,7 +104,6 @@ types-cachetools = "^5.5.0.20240820"
 
 [tool.poetry.extras]
 file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
-sphinx-docs = ["Sphinx", "sphinx-rtd-theme"]
 vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 
@@ -198,6 +195,6 @@ filterwarnings = [
 [tool.airbyte_ci]
 python_versions = ["3.10", "3.11"]
 optional_poetry_groups = ["dev"]
-poetry_extras = ["file-based", "sphinx-docs", "vector-db-based"]
+poetry_extras = ["file-based", "vector-db-based"]
 poe_tasks = ["check-ci"]
 mount_docker_socket = true