airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/custom_exceptions.py

@@ -8,7 +8,7 @@ class CircularReferenceException(Exception):
     Raised when a circular reference is detected in a manifest.
     """
 
-    def __init__(self, reference):
+    def __init__(self, reference: str) -> None:
         super().__init__(f"Circular reference found: {reference}")
 
 
@@ -17,5 +17,5 @@ class UndefinedReferenceException(Exception):
     Raised when refering to an undefined reference.
     """
 
-    def __init__(self, path, reference):
+    def __init__(self, path: str, reference: str) -> None:
         super().__init__(f"Undefined reference {reference} from {path}")

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -31,6 +31,15 @@ DEFAULT_MODEL_TYPES: Mapping[str, str] = {
     # DeclarativeStream
     "DeclarativeStream.retriever": "SimpleRetriever",
     "DeclarativeStream.schema_loader": "JsonFileSchemaLoader",
+    # DynamicDeclarativeStream
+    "DynamicDeclarativeStream.stream_template": "DeclarativeStream",
+    "DynamicDeclarativeStream.components_resolver": "ConfigComponentResolver",
+    # HttpComponentsResolver
+    "HttpComponentsResolver.retriever": "SimpleRetriever",
+    "HttpComponentsResolver.components_mapping": "ComponentMappingDefinition",
+    # ConfigComponentResolver
+    "ConfigComponentsResolver.stream_config": "StreamConfig",
+    "ConfigComponentsResolver.components_mapping": "ComponentMappingDefinition",
     # DefaultErrorHandler
     "DefaultErrorHandler.response_filters": "HttpResponseFilter",
     # DefaultPaginator
@@ -58,6 +67,10 @@ DEFAULT_MODEL_TYPES: Mapping[str, str] = {
     "AddFields.fields": "AddedFieldDefinition",
     # CustomPartitionRouter
     "CustomPartitionRouter.parent_stream_configs": "ParentStreamConfig",
+    # DynamicSchemaLoader
+    "DynamicSchemaLoader.retriever": "SimpleRetriever",
+    # SchemaTypeIdentifier
+    "SchemaTypeIdentifier.types_map": "TypesMap",
 }
 
 # We retain a separate registry for custom components to automatically insert the type if it is missing. This is intended to
@@ -77,7 +90,10 @@ CUSTOM_COMPONENTS_MAPPING: Mapping[str, str] = {
 
 class ManifestComponentTransformer:
     def propagate_types_and_parameters(
-        self, parent_field_identifier: str, declarative_component: Mapping[str, Any], parent_parameters: Mapping[str, Any]
+        self,
+        parent_field_identifier: str,
+        declarative_component: Mapping[str, Any],
+        parent_parameters: Mapping[str, Any],
     ) -> Mapping[str, Any]:
         """
         Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
@@ -119,7 +135,9 @@ class ManifestComponentTransformer:
         # Parameters should be applied to the current component fields with the existing field taking precedence over parameters if
         # both exist
         for parameter_key, parameter_value in current_parameters.items():
-            propagated_component[parameter_key] = propagated_component.get(parameter_key) or parameter_value
+            propagated_component[parameter_key] = (
+                propagated_component.get(parameter_key) or parameter_value
+            )
 
         for field_name, field_value in propagated_component.items():
             if isinstance(field_value, dict):
@@ -136,8 +154,12 @@ class ManifestComponentTransformer:
                 excluded_parameter = current_parameters.pop(field_name, None)
                 for i, element in enumerate(field_value):
                     if isinstance(element, dict):
-                        parent_type_field_identifier = f"{propagated_component.get('type')}.{field_name}"
-                        field_value[i] = self.propagate_types_and_parameters(parent_type_field_identifier, element, current_parameters)
+                        parent_type_field_identifier = (
+                            f"{propagated_component.get('type')}.{field_name}"
+                        )
+                        field_value[i] = self.propagate_types_and_parameters(
+                            parent_type_field_identifier, element, current_parameters
+                        )
                 if excluded_parameter:
                     current_parameters[field_name] = excluded_parameter
 

airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py

@@ -5,7 +5,10 @@
 import re
 from typing import Any, Mapping, Set, Tuple, Union
 
-from airbyte_cdk.sources.declarative.parsers.custom_exceptions import CircularReferenceException, UndefinedReferenceException
+from airbyte_cdk.sources.declarative.parsers.custom_exceptions import (
+    CircularReferenceException,
+    UndefinedReferenceException,
+)
 
 REF_TAG = "$ref"
 
@@ -101,11 +104,15 @@ class ManifestReferenceResolver:
         :param manifest: incoming manifest that could have references to previously defined components
         :return:
         """
-        return self._evaluate_node(manifest, manifest, set())
+        return self._evaluate_node(manifest, manifest, set())  # type: ignore[no-any-return]
 
-    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set) -> Any:
+    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:
         if isinstance(node, dict):
-            evaluated_dict = {k: self._evaluate_node(v, manifest, visited) for k, v in node.items() if not self._is_ref_key(k)}
+            evaluated_dict = {
+                k: self._evaluate_node(v, manifest, visited)
+                for k, v in node.items()
+                if not self._is_ref_key(k)
+            }
             if REF_TAG in node:
                 # The node includes a $ref key, so we splat the referenced value(s) into the evaluated dict
                 evaluated_ref = self._evaluate_node(node[REF_TAG], manifest, visited)
@@ -129,10 +136,11 @@ class ManifestReferenceResolver:
             return node
 
     def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
-        path = re.match(r"#/(.*)", ref).groups()[0]
-        if not path:
-            raise UndefinedReferenceException(path, ref)
+        ref_match = re.match(r"#/(.*)", ref)
+        if not ref_match:
+            raise ValueError(f"Invalid reference format {ref}")
         try:
+            path = ref_match.groups()[0]
             return self._read_ref_value(path, manifest)
         except (AttributeError, KeyError, IndexError):
             raise UndefinedReferenceException(path, ref)
@@ -142,8 +150,8 @@ class ManifestReferenceResolver:
         return isinstance(node, str) and node.startswith("#/")
 
     @staticmethod
-    def _is_ref_key(key) -> bool:
-        return key == REF_TAG
+    def _is_ref_key(key: str) -> bool:
+        return bool(key == REF_TAG)
 
     @staticmethod
     def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:
@@ -173,7 +181,7 @@ class ManifestReferenceResolver:
                 return manifest_node[ref]
             except (KeyError, TypeError):
                 head, ref = _parse_path(ref)
-                manifest_node = manifest_node[head]
+                manifest_node = manifest_node[head]  # type: ignore # Couldn't figure out how to fix this since manifest_node can get reassigned into other types like lists
         return manifest_node
 
 
@@ -195,8 +203,11 @@ def _parse_path(ref: str) -> Tuple[Union[str, int], str]:
     "8foo", "bar"
     """
     match = re.match(r"([^/]*)/?(.*)", ref)
-    first, rest = match.groups()
-    try:
-        return int(first), rest
-    except ValueError:
-        return first, rest
+    if match:
+        first, rest = match.groups()
+        try:
+            return int(first), rest
+        except ValueError:
+            return first, rest
+    else:
+        raise ValueError(f"Invalid path {ref} specified")