airbyte-cdk 6.56.6__py3-none-any.whl → 6.56.7__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py

@@ -108,8 +108,7 @@ class ManifestNormalizer:
             ManifestNormalizationException: Caught internally and handled by returning the original manifest.
         """
         try:
-            self._deduplicate_minifest()
-            self._reference_schemas()
+            self._deduplicate_manifest()
 
             return self._normalized_manifest
         except ManifestNormalizationException:
@@ -131,7 +130,7 @@ class ManifestNormalizer:
 
         yield from []
 
-    def _deduplicate_minifest(self) -> None:
+    def _deduplicate_manifest(self) -> None:
         """
         Find commonalities in the input JSON structure and refactor it to avoid redundancy.
         """
@@ -141,9 +140,117 @@ class ManifestNormalizer:
             self._prepare_definitions()
             # replace duplicates with references, if any
             self._handle_duplicates(self._collect_duplicates())
+            # replace parent streams with $refs
+            self._replace_parent_streams_with_refs()
+            # clean dangling fields after resolving $refs
+            self._clean_dangling_fields()
         except Exception as e:
             raise ManifestNormalizationException(str(e))
 
+    def _replace_parent_streams_with_refs(self) -> None:
+        """
+        For each stream in the manifest, if it has a retriever.partition_router with parent_stream_configs,
+        replace any 'stream' fields in those configs that are dicts and deeply equal to another stream object
+        with a $ref to the correct stream index.
+        """
+        streams = self._normalized_manifest.get(STREAMS_TAG, [])
+
+        # Build a hash-to-index mapping for O(1) lookups
+        stream_hash_to_index = {}
+        for idx, stream in enumerate(streams):
+            stream_hash = self._hash_object(stream)
+            stream_hash_to_index[stream_hash] = idx
+
+        for idx, stream in enumerate(streams):
+            retriever = stream.get("retriever")
+            if not retriever:
+                continue
+            partition_router = retriever.get("partition_router")
+            routers = (
+                partition_router
+                if isinstance(partition_router, list)
+                else [partition_router]
+                if partition_router
+                else []
+            )
+            for router in routers:
+                if not isinstance(router, dict):
+                    continue
+                if router.get("type") != "SubstreamPartitionRouter":
+                    continue
+                parent_stream_configs = router.get("parent_stream_configs", [])
+                for parent_config in parent_stream_configs:
+                    if not isinstance(parent_config, dict):
+                        continue
+                    stream_ref = parent_config.get("stream")
+                    # Only replace if it's a dict and matches any stream in the manifest
+                    if stream_ref is not None and isinstance(stream_ref, dict):
+                        stream_ref_hash = self._hash_object(stream_ref)
+                        if stream_ref_hash in stream_hash_to_index:
+                            parent_config["stream"] = {
+                                "$ref": f"#/streams/{stream_hash_to_index[stream_ref_hash]}"
+                            }
+
+    def _clean_dangling_fields(self) -> None:
+        """
+        Clean the manifest by removing unused definitions and schemas.
+        This method removes any definitions or schemas that are not referenced by any $ref in the manifest.
+        """
+
+        def find_all_refs(obj: Dict[str, Any], refs: List[str]) -> None:
+            """
+            Recursively find all $ref paths in the object.
+
+            Args:
+                obj: The object to search through
+                refs: List to store found reference paths
+            """
+            if not isinstance(obj, dict):
+                return
+
+            for key, value in obj.items():
+                if key == "$ref" and isinstance(value, str):
+                    # Remove the leading #/ from the ref path
+                    refs.append(value[2:])
+                elif isinstance(value, dict):
+                    find_all_refs(value, refs)
+                elif isinstance(value, list):
+                    for item in value:
+                        if isinstance(item, dict):
+                            find_all_refs(item, refs)
+
+        def clean_section(section: Dict[str, Any], section_path: str) -> None:
+            """
+            Clean a section by removing unreferenced fields.
+
+            Args:
+                section: The section to clean
+                section_path: The path to this section in the manifest
+            """
+            for key in list(section.keys()):
+                current_path = f"{section_path}/{key}"
+                # Check if this path is referenced or is a parent of a referenced path
+                if not any(ref.startswith(current_path) for ref in all_refs):
+                    del section[key]
+
+        # Find all references in the manifest
+        all_refs: List[str] = []
+        find_all_refs(self._normalized_manifest, all_refs)
+
+        # Clean definitions
+        if DEF_TAG in self._normalized_manifest:
+            clean_section(self._normalized_manifest[DEF_TAG], DEF_TAG)
+            # Remove empty definitions section
+            if not self._normalized_manifest[DEF_TAG]:
+                del self._normalized_manifest[DEF_TAG]
+
+        # Clean schemas
+        if SCHEMAS_TAG in self._normalized_manifest:
+            clean_section(self._normalized_manifest[SCHEMAS_TAG], SCHEMAS_TAG)
+            # Remove empty schemas section
+            if not self._normalized_manifest[SCHEMAS_TAG]:
+                del self._normalized_manifest[SCHEMAS_TAG]
+
     def _prepare_definitions(self) -> None:
         """
         Clean the definitions in the manifest by removing unnecessary properties.
@@ -163,43 +270,6 @@ class ManifestNormalizer:
             if key != LINKED_TAG:
                 self._normalized_manifest[DEF_TAG].pop(key, None)
 
-    def _extract_stream_schema(self, stream: Dict[str, Any]) -> None:
-        """
-        Extract the schema from the stream and add it to the `schemas` tag.
-        """
-
-        stream_name = stream["name"]
-        # copy the value of the SCHEMA_TAG to the SCHEMAS_TAG with the stream name as key
-        schema = stream.get(SCHEMA_LOADER_TAG, {}).get(SCHEMA_TAG)
-        if not SCHEMAS_TAG in self._normalized_manifest.keys():
-            self._normalized_manifest[SCHEMAS_TAG] = {}
-        # add stream schema to the SCHEMAS_TAG
-        if not stream_name in self._normalized_manifest[SCHEMAS_TAG].keys():
-            # add the schema to the SCHEMAS_TAG with the stream name as key
-            self._normalized_manifest[SCHEMAS_TAG][stream_name] = schema
-
-    def _reference_schemas(self) -> None:
-        """
-        Set the schema reference for the given stream in the manifest.
-        This function modifies the manifest in place.
-        """
-
-        # reference the stream schema for the stream to where it's stored
-        if SCHEMAS_TAG in self._normalized_manifest.keys():
-            for stream in self._get_manifest_streams():
-                self._extract_stream_schema(stream)
-                self._set_stream_schema_ref(stream)
-
-    def _set_stream_schema_ref(self, stream: Dict[str, Any]) -> None:
-        """
-        Set the schema reference for the given stream in the manifest.
-        This function modifies the manifest in place.
-        """
-        stream_name = stream["name"]
-        if SCHEMAS_TAG in self._normalized_manifest.keys():
-            if stream_name in self._normalized_manifest[SCHEMAS_TAG]:
-                stream[SCHEMA_LOADER_TAG][SCHEMA_TAG] = self._create_schema_ref(stream_name)
-
     def _replace_duplicates_with_refs(self, duplicates: DuplicatesType) -> None:
         """
         Process duplicate objects and replace them with references.
@@ -447,16 +517,3 @@ class ManifestNormalizer:
         """
 
         return {"$ref": f"#/{DEF_TAG}/{LINKED_TAG}/{type_key}/{key}"}
-
-    def _create_schema_ref(self, key: str) -> Dict[str, str]:
-        """
-        Create a reference object for stream schema using the specified key.
-
-        Args:
-            key: The reference key to use
-
-        Returns:
-            A reference object in the proper format
-        """
-
-        return {"$ref": f"#/{SCHEMAS_TAG}/{key}"}

airbyte_cdk/test/entrypoint_wrapper.py

@@ -373,6 +373,7 @@ def read(
     expecting_exception: bool | None = None,  # Deprecated, use `expected_outcome` instead.
     *,
     expected_outcome: ExpectedOutcome | None = None,
+    debug: bool = False,
 ) -> EntrypointOutput:
     """
     config and state must be json serializable
@@ -394,6 +395,8 @@ def read(
             "--catalog",
             catalog_file,
         ]
+        if debug:
+            args.append("--debug")
         if state is not None:
             args.extend(
                 [

{airbyte_cdk-6.56.6.dist-info → airbyte_cdk-6.56.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.56.6
+Version: 6.56.7
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.56.6.dist-info → airbyte_cdk-6.56.7.dist-info}/RECORD

@@ -138,7 +138,7 @@ airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQ
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
-airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
+airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=EtKjS9c94yNp3AwQC8KUCQaAYW5T3zvFYxoWYjc_buI,19729
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
 airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=d1GXdgEwolqQSfkMvREvIesUhMgMOMlxoXj4ctPCBwI,176855
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
@@ -379,7 +379,7 @@ airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_Mahi
 airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
 airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
 airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
-airbyte_cdk/test/entrypoint_wrapper.py,sha256=Lj1E00iTen1PZKLQ63mLCK5cLKFLmm65lqahNJDXxK4,16197
+airbyte_cdk/test/entrypoint_wrapper.py,sha256=GVddh4-mDdPWDd-Qua85aAUAsOdf6uhdf1zkPL926Gg,16275
 airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
 airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
 airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
@@ -423,9 +423,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.56.6.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.56.6.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.56.6.dist-info/METADATA,sha256=UIEu7OLuNVAhPHDB7oKrCBEvNdXnGz6-oVgzQor4n5o,6392
-airbyte_cdk-6.56.6.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.56.6.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.56.6.dist-info/RECORD,,
+airbyte_cdk-6.56.7.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.56.7.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.56.7.dist-info/METADATA,sha256=KqCnSx1P31HADnmA2gRYE6bTkOp0_55QBtNd_Zs-QBk,6392
+airbyte_cdk-6.56.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.56.7.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.56.7.dist-info/RECORD,,