airbyte-cdk 0.71.0__py3-none-any.whl → 0.72.0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -602,6 +602,27 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  CustomSchemaLoader:
+    title: Custom Schema Loader
+    description: Schema Loader component whose behavior is derived from a custom code implementation of the connector.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomSchemaLoader]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        examples:
+          - "source_railz.components.MyCustomSchemaLoader"
+      $parameters:
+        type: object
+        additionalProperties: true
   CustomTransformation:
     title: Custom Transformation
     description: Transformation component whose behavior is derived from a custom code implementation of the connector.
@@ -948,6 +969,7 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/InlineSchemaLoader"
           - "$ref": "#/definitions/JsonFileSchemaLoader"
+          - "$ref": "#/definitions/CustomSchemaLoader"
       # TODO we have move the transformation to the RecordSelector level in the code but kept this here for
       # compatibility reason. We should eventually move this to align with the code.
       transformations:

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -208,6 +208,20 @@ class CustomPartitionRouter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
 
 
+class CustomSchemaLoader(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal['CustomSchemaLoader']
+    class_name: str = Field(
+        ...,
+        description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.',
+        examples=['source_railz.components.MyCustomSchemaLoader'],
+        title='Class Name',
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
+
+
 class CustomTransformation(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1161,7 +1175,9 @@ class DeclarativeStream(BaseModel):
     primary_key: Optional[PrimaryKey] = Field(
         '', description='The primary key of the stream.', title='Primary Key'
     )
-    schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field(
+    schema_loader: Optional[
+        Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]
+    ] = Field(
         None,
         description='Component used to retrieve the schema for the current stream.',
         title='Schema Loader',

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -49,6 +49,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordFilter as CustomRecordFilterModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRequester as CustomRequesterModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRetriever as CustomRetrieverModel
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomSchemaLoader as CustomSchemaLoader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomTransformation as CustomTransformationModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import DatetimeBasedCursor as DatetimeBasedCursorModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import DeclarativeStream as DeclarativeStreamModel
@@ -165,6 +166,7 @@ class ModelToComponentFactory:
             CustomRecordFilterModel: self.create_custom_component,
             CustomRequesterModel: self.create_custom_component,
             CustomRetrieverModel: self.create_custom_component,
+            CustomSchemaLoader: self.create_custom_component,
             CustomPaginationStrategyModel: self.create_custom_component,
             CustomPartitionRouterModel: self.create_custom_component,
             CustomTransformationModel: self.create_custom_component,

{airbyte_cdk-0.71.0.dist-info → airbyte_cdk-0.72.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.71.0
+Version: 0.72.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.71.0.dist-info → airbyte_cdk-0.72.0.dist-info}/RECORD

@@ -38,7 +38,7 @@ airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py
 airbyte_cdk/sources/concurrent_source/thread_pool_manager.py,sha256=hFj5rsRtORurl3fwH8GC9h6Uz2wbzBFOLWUxJ-YJ7J8,4801
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=An4UXx4e_GodeVd0bSQTv1G_Z1yjUcb7NbOmcC9-i9I,89327
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=netrMub3A9k9wk5VWx8vqDWhfeLk_sviHHJ8NXnH2OA,90111
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=U2As9PDKmcWDgbsWUo-RetJ9fxQOBlwntWZ0NOgs5Ac,1453
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=9nBjSBilzH2aeJsUEqOLyc4G2RRjlPZCapHDMv4jnOU,6691
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
@@ -80,14 +80,14 @@ airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=dyIM-bzh54
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=hOLBs9VaaE5xsT2wY2VxSrISE165bu_Egb83ordG4XI,5379
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=V6WGKJ9cXX1rjuM4bK3Cs9xEryMlkY2U3FMsSBhrgC8,3098
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=EiYnzwCHZV7EYqMJqcy6xKSeHvTKZBsQndjbEwmiTW4,93
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=u6i4nYufXZe27bIED7uhsvfhghOMN4NdWoGLEeTjzwk,61032
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=1a67m2fGAdBRf7rOzvk5SIluQHIWL4SPLnrjsnrnm_s,61574
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/class_types_registry.py,sha256=5vOvMuyWlpALrOq2ehLxa7wO6tlFIlgUNtMYrMCKIjE,6092
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=y7_G5mM07zxT5YG975kdC2PAja-Uc83pYp8WrV3GNdo,522
 airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py,sha256=W8BcK4KOg4ifNXgsdeIoV4oneHjXBKcPHEZHIC4r-hM,3801
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=i2yUdrdlPUHI0dSQX0zBT8WSg912SMiCF8qkQ8VvlA4,8287
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=6ukHx0bBrCJm9rek1l_MEfS3U_gdJcM4pJRyifJEOp0,6412
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=_jX1sPW5SA51nHoxpoVktC3jdO8PR4cGrQfZftcVSLo,59316
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=1zI1Mk9k_3p_TMz6LsgV7U54CJ6etl88q8WOv4AZO-w,59499
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=27sOWhw2LBQs62HchURakHQ2M_mtnOatNgU6q8RUtpU,476
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=L22D-up7W2HahZZo2dA-IbRSs7qJEahU6O6bU0eiIt8,4324
 airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=cl-TQdu_m6_IF20gdD1jll0SpejIyMZHvyGXx2NafuI,1611
@@ -329,7 +329,7 @@ unit_tests/sources/declarative/interpolation/test_macros.py,sha256=vEZmHQ0KsfQUz
 unit_tests/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py,sha256=egePHWYcXprfPtoHhiquWAXuJkDr-DB_RakKhdyaoHs,14316
 unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py,sha256=K3q9eyx-sJFQ8nGYjAgS7fxau4sX_FlNreEAjiCYOeE,5306
-unit_tests/sources/declarative/parsers/test_model_to_component_factory.py,sha256=WIUygDJvNGEIzNdlarkYymTTgRtiXlR6IOjwnsqlC3E,75683
+unit_tests/sources/declarative/parsers/test_model_to_component_factory.py,sha256=cDlDoNutC6JMGdyvkYMteiHtVrpQ_cKnRE_yn6dWui0,76426
 unit_tests/sources/declarative/parsers/testing_components.py,sha256=_yUijmYRM-yYHPGDB2JsfEiOuVrgexGW9QwHf1xxNW8,1326
 unit_tests/sources/declarative/partition_routers/__init__.py,sha256=O8MZg4Bv_DghdRy9BoJCPIqdV75VtiUrhEkExQgb2nE,61
 unit_tests/sources/declarative/partition_routers/test_list_partition_router.py,sha256=WKdbAQCHfCVOyoAFM_kbHsbqAF_e5FX5Zvou5ARsJZ4,6572
@@ -459,8 +459,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=CdKK8A2-5XVxbXVtX22FK9dwwMeP5KNqDH6luWRXSNw,5256
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.71.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.71.0.dist-info/METADATA,sha256=IEujsMxKiLNDhn0U2HOCIBvKGSrEZ0fx51DA8KAR9l8,11074
-airbyte_cdk-0.71.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-airbyte_cdk-0.71.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.71.0.dist-info/RECORD,,
+airbyte_cdk-0.72.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.72.0.dist-info/METADATA,sha256=lDN4hbkJHUsXxycTKKTDAgCzzo72JYBIzlCOZzuU0nM,11074
+airbyte_cdk-0.72.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+airbyte_cdk-0.72.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.72.0.dist-info/RECORD,,

unit_tests/sources/declarative/parsers/test_model_to_component_factory.py

@@ -5,6 +5,7 @@
 # mypy: ignore-errors
 
 import datetime
+from typing import Any, Mapping
 
 import pytest
 from airbyte_cdk.models import Level
@@ -27,6 +28,7 @@ from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamMod
 from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel
 from airbyte_cdk.sources.declarative.models import CustomErrorHandler as CustomErrorHandlerModel
 from airbyte_cdk.sources.declarative.models import CustomPartitionRouter as CustomPartitionRouterModel
+from airbyte_cdk.sources.declarative.models import CustomSchemaLoader as CustomSchemaLoaderModel
 from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
 from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
 from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
@@ -66,6 +68,7 @@ from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
 from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator
 from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
+from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.spec import Spec
 from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
@@ -1820,3 +1823,19 @@ def test_create_offset_increment():
     assert strategy.page_size == expected_strategy.page_size
     assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
     assert strategy.config == input_config
+
+
+class MyCustomSchemaLoader(SchemaLoader):
+    def get_json_schema(self) -> Mapping[str, Any]:
+        """Returns a mapping describing the stream's schema"""
+        return {}
+
+
+def test_create_custom_schema_loader():
+
+    definition = {
+        "type": "CustomSchemaLoader",
+        "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader"
+    }
+    component = factory.create_component(CustomSchemaLoaderModel, definition, {})
+    assert isinstance(component, MyCustomSchemaLoader)