altimate-datapilot-cli 0.0.11__tar.gz → 0.0.13__tar.gz

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.11
+current_version = 0.0.13
 commit = True
 tag = True
 

{altimate-datapilot-cli-0.0.11/src/altimate_datapilot_cli.egg-info → altimate-datapilot-cli-0.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: altimate-datapilot-cli
-Version: 0.0.11
+Version: 0.0.13
 Summary: Assistant for Data Teams
 Home-page: https://github.com/AltimateAI/datapilot-cli
 Author: Altimate Inc

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/docs/conf.py

@@ -15,7 +15,7 @@ project = "datapilot-cli"
 year = "2024"
 author = "Altimate Inc."
 copyright = f"{year}, {author}"
-version = release = "0.0.11"
+version = release = "0.0.13"
 
 pygments_style = "trac"
 templates_path = ["."]

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/setup.py

@@ -13,7 +13,7 @@ def read(*names, **kwargs):
 
 setup(
     name="altimate-datapilot-cli",
-    version="0.0.11",
+    version="0.0.13",
     license="MIT",
     description="Assistant for Data Teams",
     long_description="{}\n{}".format(
@@ -63,7 +63,7 @@ setup(
     python_requires=">=3.8",
     install_requires=[
         "click==8.1.7",
-        "dbt-artifacts-parser==0.5.1",
+        "dbt-artifacts-parser==0.6.0",
         "ruamel.yaml==0.18.6",
         "tabulate==0.9.0",
         "requests==2.31.0",

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13/src/altimate_datapilot_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: altimate-datapilot-cli
-Version: 0.0.11
+Version: 0.0.13
 Summary: Assistant for Data Teams
 Home-page: https://github.com/AltimateAI/datapilot-cli
 Author: Altimate Inc

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/altimate_datapilot_cli.egg-info/SOURCES.txt

@@ -168,6 +168,9 @@ src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py
 src/datapilot/core/platforms/dbt/wrappers/manifest/v11/__init__.py
 src/datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py
 src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py
+src/datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py
+src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py
+src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py
 src/datapilot/core/platforms/dbt/wrappers/run_results/__init__.py
 src/datapilot/core/platforms/dbt/wrappers/run_results/run_results.py
 src/datapilot/exceptions/__init__.py
@@ -186,11 +189,13 @@ tests/core/platform/dbt/__init__.py
 tests/core/platform/dbt/test_cli.py
 tests/core/platform/dbt/test_utils.py
 tests/data/catalog_v1.json
+tests/data/catalog_v12.json
 tests/data/config.yml
 tests/data/manifest_v10.json
 tests/data/manifest_v10macroargs.json
 tests/data/manifest_v11.json
 tests/data/manifest_v11macroargs.json
+tests/data/manifest_v12.json
 tests/data/manifests/manifest_js.json
 tests/data/manifests/manifest_js2.json
 tests/data/manifests/manifest_js3.json

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/altimate_datapilot_cli.egg-info/requires.txt

@@ -1,5 +1,5 @@
 click==8.1.7
-dbt-artifacts-parser==0.5.1
+dbt-artifacts-parser==0.6.0
 ruamel.yaml==0.18.6
 tabulate==0.9.0
 requests==2.31.0

altimate-datapilot-cli-0.0.13/src/datapilot/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.13"

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/datapilot/clients/altimate/client.py

@@ -62,7 +62,7 @@ class APIClient:
         response = requests.post(url, headers=headers, json=data, timeout=timeout)
         self.logger.debug(f"Received POST response with status: {response.status_code }")
 
-        return response
+        return response.json()
 
     def put(self, endpoint, data, timeout=None):
         url = f"{self.base_url}{endpoint}"
@@ -83,3 +83,7 @@ class APIClient:
     def validate_credentials(self):
         endpoint = "/dbt/v3/validate-credentials"
         return self.get(endpoint)
+
+    def start_dbt_ingestion(self, params=None):
+        endpoint = "/dbt/v1/start_dbt_ingestion"
+        return self.post(endpoint, data=params)

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/datapilot/clients/altimate/utils.py

@@ -46,10 +46,14 @@ def validate_credentials(
     return api_client.validate_credentials()
 
 
-def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> Dict:
+def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
     api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
 
-    params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"}
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+        "file_type": file_type,
+    }
     signed_url_data = api_client.get_signed_url(params)
     if signed_url_data:
         signed_url = signed_url_data.get("url")
@@ -57,7 +61,7 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path,
         api_client.log(f"Received signed URL: {signed_url}")
         api_client.log(f"Received File ID: {file_id}")
 
-        upload_response = upload_content_to_signed_url(manifest_path, signed_url)
+        upload_response = upload_content_to_signed_url(file_path, signed_url)
 
         if upload_response:
             verify_params = {"dbt_core_integration_file_id": file_id}
@@ -73,3 +77,20 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path,
             "ok": False,
             "message": "Error in uploading the manifest.                                                                                                                              ",
         }
+
+
+def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, backend_url):
+    api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+    }
+    data = api_client.start_dbt_ingestion(params)
+    if data and data.get("ok"):
+        return {"ok": True}
+    else:
+        api_client.log("Error starting dbt ingestion worker")
+        return {
+            "ok": False,
+            "message": "Error starting dbt ingestion worker.                                                                                                                              ",
+        }

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/datapilot/core/platforms/dbt/cli/cli.py

@@ -3,7 +3,8 @@ import logging
 import click
 
 from datapilot.clients.altimate.utils import check_token_and_instance
-from datapilot.clients.altimate.utils import onboard_manifest
+from datapilot.clients.altimate.utils import onboard_file
+from datapilot.clients.altimate.utils import start_dbt_ingestion
 from datapilot.clients.altimate.utils import validate_credentials
 from datapilot.config.config import load_config
 from datapilot.core.platforms.dbt.constants import MODEL
@@ -14,6 +15,7 @@ from datapilot.core.platforms.dbt.formatting import generate_project_insights_ta
 from datapilot.core.platforms.dbt.utils import load_catalog
 from datapilot.core.platforms.dbt.utils import load_manifest
 from datapilot.utils.formatting.utils import tabulate_data
+from datapilot.utils.utils import map_url_to_instance
 
 logging.basicConfig(level=logging.INFO)
 
@@ -87,9 +89,21 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
 @click.option("--token", prompt="API Token", help="Your API token for authentication.")
 @click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.")
 @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID")
+@click.option(
+    "--dbt_core_integration_environment", default="PROD", prompt="DBT Core Integration Environment", help="DBT Core Integration Environment"
+)
 @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.")
+@click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.")
 @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
-def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None):
+def onboard(
+    token,
+    instance_name,
+    dbt_core_integration_id,
+    dbt_core_integration_environment,
+    manifest_path,
+    catalog_path,
+    backend_url="https://api.myaltimate.com",
+):
     """Onboard a manifest file to DBT."""
     check_token_and_instance(token, instance_name)
 
@@ -104,9 +118,32 @@ def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backen
         click.echo(f"Error: {e}")
         return
 
-    response = onboard_manifest(token, instance_name, dbt_core_integration_id, manifest_path, backend_url)
-
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "manifest", manifest_path, backend_url
+    )
     if response["ok"]:
         click.echo("Manifest onboarded successfully!")
     else:
         click.echo(f"{response['message']}")
+
+    if not catalog_path:
+        return
+
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "catalog", catalog_path, backend_url
+    )
+    if response["ok"]:
+        click.echo("Catalog onboarded successfully!")
+    else:
+        click.echo(f"{response['message']}")
+
+    response = start_dbt_ingestion(token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, backend_url)
+    if response["ok"]:
+        url = map_url_to_instance(backend_url, instance_name)
+        if not url:
+            click.echo("Manifest and catalog ingestion has started.")
+        else:
+            url = f"{url}/settings/integrations/{dbt_core_integration_id}/{dbt_core_integration_environment}"
+            click.echo(f"Manifest and catalog ingestion has started. You can check the status at {url}")
+    else:
+        click.echo(f"{response['message']}")

{altimate-datapilot-cli-0.0.11 → altimate-datapilot-cli-0.0.13}/src/datapilot/core/platforms/dbt/factory.py

@@ -1,18 +1,22 @@
 from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogV1
 from dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10
 from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
 
 from datapilot.core.platforms.dbt.schemas.manifest import Catalog
 from datapilot.core.platforms.dbt.schemas.manifest import Manifest
 from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import CatalogV1Wrapper
 from datapilot.core.platforms.dbt.wrappers.manifest.v10.wrapper import ManifestV10Wrapper
 from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import ManifestV11Wrapper
+from datapilot.core.platforms.dbt.wrappers.manifest.v12.wrapper import ManifestV12Wrapper
 from datapilot.exceptions.exceptions import AltimateNotSupportedError
 
 
 class DBTFactory:
     @classmethod
     def get_manifest_wrapper(cls, manifest: Manifest):
+        if isinstance(manifest, ManifestV12):
+            return ManifestV12Wrapper(manifest)
         if isinstance(manifest, ManifestV11):
             return ManifestV11Wrapper(manifest)
         if isinstance(manifest, ManifestV10):

altimate-datapilot-cli-0.0.13/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py

@@ -0,0 +1,36 @@
+from typing import Dict
+from typing import Type
+from typing import Union
+
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Exposures
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Macros
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node1
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node3
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node4
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node5
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node7
+from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Sources
+
+from datapilot.core.platforms.dbt.constants import GENERIC
+from datapilot.core.platforms.dbt.constants import SINGULAR
+
+ManifestNode = Union[Node, Node1, Node2, Node3, Node4, Node5, Node6, Node7]
+
+SourceNode = Sources
+
+ExposureNode = Exposures
+
+TestNode = Union[Node6, Node2]
+
+MacroNode = Macros
+
+TEST_TYPE_TO_NODE_MAP: Dict[str, Type] = {
+    GENERIC: [Node6],
+    SINGULAR: [Node2],
+}
+
+
+SeedNodeMap = Node