collate-dbt-artifacts-parser 0.1__tar.gz → 0.1.2__tar.gz

{collate_dbt_artifacts_parser-0.1 → collate_dbt_artifacts_parser-0.1.2}/.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 *.so
 
 # Distribution / packaging
+.DS_Store
 .Python
 build/
 develop-eggs/
@@ -111,6 +112,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+test_parser/
 
 # Spyder project settings
 .spyderproject

{collate_dbt_artifacts_parser-0.1 → collate_dbt_artifacts_parser-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: collate-dbt-artifacts-parser
-Version: 0.1
+Version: 0.1.2
 Summary: A dbt artifacts parser in python
 Project-URL: Home, https://github.com/open-metadata/collate-dbt-artifacts-parser
 Author: open-metadata
@@ -214,6 +214,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -221,7 +222,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
-Requires-Python: >=3.9.0
+Requires-Python: >=3.8.0
 Requires-Dist: pydantic<3.0,>=2.0
 Provides-Extra: dev
 Requires-Dist: black==24.8.0; extra == 'dev'

{collate_dbt_artifacts_parser-0.1 → collate_dbt_artifacts_parser-0.1.2}/collate_dbt_artifacts_parser/__init__.py

@@ -18,4 +18,4 @@
 A dbt artifacts parser in python
 """
 
-__version__ = "0.1"
+__version__ = "0.1.2"

{collate_dbt_artifacts_parser-0.1 → collate_dbt_artifacts_parser-0.1.2}/collate_dbt_artifacts_parser/parser.py

@@ -18,9 +18,15 @@ from typing import Union
 
 from pydantic import ValidationError
 
-from collate_dbt_artifacts_parser.parsers.catalog.catalog_cloud import CatalogCLOUD
+from collate_dbt_artifacts_parser.parsers.catalog.catalog_cloud_v1 import CatalogCLOUDV1
+from collate_dbt_artifacts_parser.parsers.catalog.catalog_cloud_v2 import CatalogCLOUDV2
 from collate_dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogV1
-from collate_dbt_artifacts_parser.parsers.manifest.manifest_cloud import ManifestCLOUD
+from collate_dbt_artifacts_parser.parsers.manifest.manifest_cloud_v1 import (
+    ManifestCLOUDV1,
+)
+from collate_dbt_artifacts_parser.parsers.manifest.manifest_cloud_v2 import (
+    ManifestCLOUDV2,
+)
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v1 import ManifestV1
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v2 import ManifestV2
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v3 import ManifestV3
@@ -33,8 +39,11 @@ from collate_dbt_artifacts_parser.parsers.manifest.manifest_v9 import ManifestV9
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
 from collate_dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
-from collate_dbt_artifacts_parser.parsers.run_results.run_results_cloud import (
-    RunResultsCLOUD,
+from collate_dbt_artifacts_parser.parsers.run_results.run_results_cloud_v1 import (
+    RunResultsCLOUDV1,
+)
+from collate_dbt_artifacts_parser.parsers.run_results.run_results_cloud_v2 import (
+    RunResultsCLOUDV2,
 )
 from collate_dbt_artifacts_parser.parsers.run_results.run_results_v1 import RunResultsV1
 from collate_dbt_artifacts_parser.parsers.run_results.run_results_v2 import RunResultsV2
@@ -42,7 +51,8 @@ from collate_dbt_artifacts_parser.parsers.run_results.run_results_v3 import RunR
 from collate_dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4
 from collate_dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5
 from collate_dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6
-from collate_dbt_artifacts_parser.parsers.sources.sources_cloud import SourcesCLOUD
+from collate_dbt_artifacts_parser.parsers.sources.sources_cloud_v1 import SourcesCLOUDV1
+from collate_dbt_artifacts_parser.parsers.sources.sources_cloud_v2 import SourcesCLOUDV2
 from collate_dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1
 from collate_dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2
 from collate_dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3
@@ -53,7 +63,8 @@ from collate_dbt_artifacts_parser.parsers.version_map import ArtifactTypes
 #
 # catalog
 #
-def parse_catalog(catalog: dict) -> Union[CatalogV1, CatalogCLOUD]:
+def parse_catalog(
+        catalog: dict) -> Union[CatalogV1, CatalogCLOUDV1, CatalogCLOUDV2]:
     """Parse catalog.json
 
     Args:
@@ -67,40 +78,56 @@ def parse_catalog(catalog: dict) -> Union[CatalogV1, CatalogCLOUD]:
         try:
             return CatalogV1(**catalog)
         except ValidationError:
-            return CatalogCLOUD(**catalog)
+            try:
+                return CatalogCLOUDV1(**catalog)
+            except ValidationError:
+                return CatalogCLOUDV2(**catalog)
     raise ValueError("Not a catalog.json")
 
 
-def parse_catalog_v1(catalog: dict) -> Union[CatalogV1, CatalogCLOUD]:
+def parse_catalog_v1(catalog: dict) -> Union[CatalogV1, CatalogCLOUDV1]:
     """Parse catalog.json v1"""
     dbt_schema_version = get_dbt_schema_version(artifact_json=catalog)
     if dbt_schema_version == ArtifactTypes.CATALOG_V1.value.dbt_schema_version:
         try:
             return CatalogV1(**catalog)
         except ValidationError:
-            return CatalogCLOUD(**catalog)
+            return CatalogCLOUDV1(**catalog)
     raise ValueError("Not a catalog.json v1")
 
 
 #
 # manifest
 #
+def _try_parsers(data: dict, parsers: list):
+    """Helper function to try multiple parsers in sequence.
+    Returns the first successful parse result or raises the last ValidationError."""
+    last_error = None
+    for parser in parsers:
+        try:
+            return parser(**data)
+        except ValidationError as e:
+            last_error = e
+    raise last_error
+
+
 def parse_manifest(
     manifest: dict,
 ) -> Union[
-    ManifestV1,
-    ManifestV2,
-    ManifestV3,
-    ManifestV4,
-    ManifestV5,
-    ManifestV6,
-    ManifestV7,
-    ManifestV8,
-    ManifestV9,
-    ManifestV10,
-    ManifestV11,
-    ManifestV12,
-    ManifestCLOUD,
+        ManifestV1,
+        ManifestV2,
+        ManifestV3,
+        ManifestV4,
+        ManifestV5,
+        ManifestV6,
+        ManifestV7,
+        ManifestV8,
+        ManifestV9,
+        ManifestV10,
+        ManifestV11,
+        ManifestV12,
+        ManifestCLOUDV1,
+        ManifestCLOUDV2,
 ]:
     """Parse manifest.json
 
@@ -138,10 +165,8 @@ def parse_manifest(
     elif dbt_schema_version == ArtifactTypes.MANIFEST_V11.value.dbt_schema_version:
         return ManifestV11(**manifest)
     elif dbt_schema_version == ArtifactTypes.MANIFEST_V12.value.dbt_schema_version:
-        try:
-            return ManifestV12(**manifest)
-        except ValidationError:
-            return ManifestCLOUD(**manifest)
+        return _try_parsers(manifest,
+                            [ManifestV12, ManifestCLOUDV1, ManifestCLOUDV2])
     raise ValueError("Not a manifest.json")
 
 
@@ -233,14 +258,13 @@ def parse_manifest_v11(manifest: dict) -> ManifestV11:
     raise ValueError("Not a manifest.json v11")
 
 
-def parse_manifest_v12(manifest: dict) -> Union[ManifestV12, ManifestCLOUD]:
+def parse_manifest_v12(
+        manifest: dict) -> Union[ManifestV12, ManifestCLOUDV1, ManifestCLOUDV2]:
     """Parse manifest.json ver.12"""
     dbt_schema_version = get_dbt_schema_version(artifact_json=manifest)
     if dbt_schema_version == ArtifactTypes.MANIFEST_V12.value.dbt_schema_version:
-        try:
-            return ManifestV12(**manifest)
-        except ValidationError:
-            return ManifestCLOUD(**manifest)
+        return _try_parsers(manifest,
+                            [ManifestV12, ManifestCLOUDV1, ManifestCLOUDV2])
     raise ValueError("Not a manifest.json v12")
 
 
@@ -250,13 +274,14 @@ def parse_manifest_v12(manifest: dict) -> Union[ManifestV12, ManifestCLOUD]:
 def parse_run_results(
     run_results: dict,
 ) -> Union[
-    RunResultsV1,
-    RunResultsV2,
-    RunResultsV3,
-    RunResultsV4,
-    RunResultsV5,
-    RunResultsV6,
-    RunResultsCLOUD,
+        RunResultsV1,
+        RunResultsV2,
+        RunResultsV3,
+        RunResultsV4,
+        RunResultsV5,
+        RunResultsV6,
+        RunResultsCLOUDV1,
+        RunResultsCLOUDV2,
 ]:
     """Parse run-results.json
 
@@ -278,10 +303,8 @@ def parse_run_results(
     elif dbt_schema_version == ArtifactTypes.RUN_RESULTS_V5.value.dbt_schema_version:
         return RunResultsV5(**run_results)
     elif dbt_schema_version == ArtifactTypes.RUN_RESULTS_V6.value.dbt_schema_version:
-        try:
-            return RunResultsV6(**run_results)
-        except ValidationError:
-            return RunResultsCLOUD(**run_results)
+        return _try_parsers(
+            run_results, [RunResultsV6, RunResultsCLOUDV1, RunResultsCLOUDV2])
     raise ValueError("Not a run_results.json")
 
 
@@ -325,14 +348,14 @@ def parse_run_results_v5(run_results: dict) -> RunResultsV5:
     raise ValueError("Not a run-results.json v5")
 
 
-def parse_run_results_v6(run_results: dict) -> Union[RunResultsV6, RunResultsCLOUD]:
+def parse_run_results_v6(
+    run_results: dict
+) -> Union[RunResultsV6, RunResultsCLOUDV1, RunResultsCLOUDV2]:
     """Parse run-results.json v6"""
     dbt_schema_version = get_dbt_schema_version(artifact_json=run_results)
     if dbt_schema_version == ArtifactTypes.RUN_RESULTS_V6.value.dbt_schema_version:
-        try:
-            return RunResultsV6(**run_results)
-        except ValidationError:
-            return RunResultsCLOUD(**run_results)
+        return _try_parsers(
+            run_results, [RunResultsV6, RunResultsCLOUDV1, RunResultsCLOUDV2])
     raise ValueError("Not a run-results.json v6")
 
 
@@ -341,7 +364,7 @@ def parse_run_results_v6(run_results: dict) -> Union[RunResultsV6, RunResultsCLO
 #
 def parse_sources(
     sources: dict,
-) -> Union[SourcesV1, SourcesV2, SourcesV3, SourcesCLOUD]:
+) -> Union[SourcesV1, SourcesV2, SourcesV3, SourcesCLOUDV1, SourcesCLOUDV2]:
     """Parse sources.json
 
     Args:
@@ -356,10 +379,8 @@ def parse_sources(
     elif dbt_schema_version == ArtifactTypes.SOURCES_V2.value.dbt_schema_version:
         return SourcesV2(**sources)
     elif dbt_schema_version == ArtifactTypes.SOURCES_V3.value.dbt_schema_version:
-        try:
-            return SourcesV3(**sources)
-        except ValidationError:
-            return SourcesCLOUD(**sources)
+        return _try_parsers(sources,
+                            [SourcesV3, SourcesCLOUDV1, SourcesCLOUDV2])
     raise ValueError("Not a sources.json")
 
 
@@ -379,12 +400,11 @@ def parse_sources_v2(sources: dict) -> SourcesV2:
     raise ValueError("Not a sources.json v2")
 
 
-def parse_sources_v3(sources: dict) -> Union[SourcesV3, SourcesCLOUD]:
+def parse_sources_v3(
+        sources: dict) -> Union[SourcesV3, SourcesCLOUDV1, SourcesCLOUDV2]:
     """Parse sources.json v3"""
     dbt_schema_version = get_dbt_schema_version(artifact_json=sources)
     if dbt_schema_version == ArtifactTypes.SOURCES_V3.value.dbt_schema_version:
-        try:
-            return SourcesV3(**sources)
-        except ValidationError:
-            return SourcesCLOUD(**sources)
+        return _try_parsers(sources,
+                            [SourcesV3, SourcesCLOUDV1, SourcesCLOUDV2])
     raise ValueError("Not a sources.json v3")

collate_dbt_artifacts_parser-0.1.2/collate_dbt_artifacts_parser/parsers/catalog/catalog_cloud_v1.py

@@ -0,0 +1,71 @@
+# generated by datamodel-codegen:
+#   filename:  catalog_cloud_v1.json
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import ConfigDict, Field
+
+from collate_dbt_artifacts_parser.parsers.base import BaseParserModel
+
+
+class Metadata(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    dbt_schema_version: Optional[str] = None
+    dbt_version: Optional[str] = '1.10.0a1'
+    generated_at: Optional[str] = None
+    invocation_id: Optional[str] = None
+    env: Optional[Dict[str, str]] = None
+
+
+class Metadata1(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    type: str
+    schema_: str = Field(..., alias='schema')
+    name: str
+    database: Optional[str] = None
+    comment: Optional[str] = None
+    owner: Optional[str] = None
+
+
+class Columns(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+class Stats(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    id: str
+    label: str
+    value: Optional[Union[bool, str, float]] = None
+    include: bool
+    description: Optional[str] = None
+
+
+class Nodes(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata1 = Field(..., title='TableMetadata')
+    columns: Dict[str, Columns]
+    stats: Dict[str, Stats]
+    unique_id: Optional[str] = None
+
+
+class Sources(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata1 = Field(..., title='TableMetadata')
+    columns: Dict[str, Columns]
+    stats: Dict[str, Stats]
+    unique_id: Optional[str] = None
+
+
+class CatalogCLOUDV1(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata = Field(..., title='CatalogMetadata')
+    nodes: Dict[str, Nodes]
+    sources: Dict[str, Sources]
+    errors: Optional[List[str]] = None
+    field_compile_results: Any = Field(None, alias='_compile_results')

collate_dbt_artifacts_parser-0.1.2/collate_dbt_artifacts_parser/parsers/catalog/catalog_cloud_v2.py

@@ -0,0 +1,72 @@
+# generated by datamodel-codegen:
+#   filename:  catalog_cloud_v2.json
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import ConfigDict, Field
+
+from collate_dbt_artifacts_parser.parsers.base import BaseParserModel
+
+
+class Metadata(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    dbt_schema_version: Optional[str] = None
+    dbt_version: Optional[str] = '1.10.0a1'
+    generated_at: Optional[str] = None
+    invocation_id: Optional[str] = None
+    invocation_started_at: Optional[str] = None
+    env: Optional[Dict[str, str]] = None
+
+
+class Metadata1(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    type: str
+    schema_: str = Field(..., alias='schema')
+    name: str
+    database: Optional[str] = None
+    comment: Optional[str] = None
+    owner: Optional[str] = None
+
+
+class Columns(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+class Stats(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    id: str
+    label: str
+    value: Optional[Union[bool, str, float]] = None
+    include: bool
+    description: Optional[str] = None
+
+
+class Nodes(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata1 = Field(..., title='TableMetadata')
+    columns: Dict[str, Columns]
+    stats: Dict[str, Stats]
+    unique_id: Optional[str] = None
+
+
+class Sources(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata1 = Field(..., title='TableMetadata')
+    columns: Dict[str, Columns]
+    stats: Dict[str, Stats]
+    unique_id: Optional[str] = None
+
+
+class CatalogCLOUDV2(BaseParserModel):
+    model_config = ConfigDict(extra='forbid',)
+    metadata: Metadata = Field(..., title='CatalogMetadata')
+    nodes: Dict[str, Nodes]
+    sources: Dict[str, Sources]
+    errors: Optional[List[str]] = None
+    field_compile_results: Any = Field(None, alias='_compile_results')