altimate-datapilot-cli 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl

{altimate_datapilot_cli-0.0.10.dist-info → altimate_datapilot_cli-0.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: altimate-datapilot-cli
-Version: 0.0.10
+Version: 0.0.12
 Summary: Assistant for Data Teams
 Home-page: https://github.com/AltimateAI/datapilot-cli
 Author: Altimate Inc
@@ -31,7 +31,6 @@ Requires-Python: >=3.8
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: click (==8.1.7)
-Requires-Dist: sqlglot (==18.3.0)
 Requires-Dist: dbt-artifacts-parser (==0.5.1)
 Requires-Dist: ruamel.yaml (==0.18.6)
 Requires-Dist: tabulate (==0.9.0)

{altimate_datapilot_cli-0.0.10.dist-info → altimate_datapilot_cli-0.0.12.dist-info}/RECORD

@@ -1,11 +1,11 @@
-datapilot/__init__.py,sha256=-nNlMKS9nph3FR78_ZG9RGKrbxseeNp2K6nMr0pVGaU,23
+datapilot/__init__.py,sha256=1SFj-qMOaeO0fOKQaFmYhAKiRwwNda5uWKpYtIcken8,23
 datapilot/__main__.py,sha256=I9USmeNnK-cAHb6LZfydJC0LeNSE8enieeY55wpR6uw,380
 datapilot/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapilot/cli/main.py,sha256=VSdqlkCiu8GSG9qQh8q0BzyocsQc4lKWxZAPEsjXF18,181
 datapilot/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapilot/clients/altimate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapilot/clients/altimate/client.py,sha256=ayTvegsQKOj0EocIWeB03ys1PlGLJFB5k8zDs8oxPBo,3107
-datapilot/clients/altimate/utils.py,sha256=cDYR-6J4SDaOBMOtkOlaP_j0TkCgCyYLFVDjVCEiCLY,2629
+datapilot/clients/altimate/client.py,sha256=DHPG2y7r1gFph80TVp7zozd_Hl05mzxtNFQ4JYEN1Jk,3260
+datapilot/clients/altimate/utils.py,sha256=IyS4iY5nE5KF9bivHKVzioUitQnmgVVIZjqPNFoeUwY,3547
 datapilot/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapilot/config/config.py,sha256=kyj53Qsb85V4iGQsX0vSwULOjscMOSFrJDJ3tnagJpo,403
 datapilot/config/utils.py,sha256=DIAVX-OZ5Lc0Ky_A7dvdbPcD1QSg2DRxZcuaIIZ2rhw,1146
@@ -30,7 +30,7 @@ datapilot/core/platforms/dbt/factory.py,sha256=fsk8XWXiSq22t5BH5jGQMASWIJ-eUNqOh
 datapilot/core/platforms/dbt/formatting.py,sha256=bpfa7XmVghTq4WnGDGYC6DruwOwH8YmjFHghoo5cPD8,1638
 datapilot/core/platforms/dbt/utils.py,sha256=ozFHprR6LTLXQdrGyaRoyIBTua4P1NkP8T7LGgN-9c0,18577
 datapilot/core/platforms/dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapilot/core/platforms/dbt/cli/cli.py,sha256=sEPOoYL7lVy4G5Y4HNi6yeQqMtknxIh1EzobiisBL6c,4146
+datapilot/core/platforms/dbt/cli/cli.py,sha256=qzeJU8ALoo-VykKeqPT0O5pheMMH9noBh7S42099p7w,5563
 datapilot/core/platforms/dbt/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapilot/core/platforms/dbt/hooks/executor_hook.py,sha256=gSM50vAO7C-f1rdnHogWbqc87aCXPXysZepjp5L2qzw,2966
 datapilot/core/platforms/dbt/insights/__init__.py,sha256=vnNOqP6lJ-r1SqmxQbeTEphL-omj466OanqQY1WNrUA,7547
@@ -130,13 +130,13 @@ datapilot/schemas/constants.py,sha256=-eyLQvpB3ut2cQ8Cx-xGpCa9lOHBUATA8s-FXEkBoF
 datapilot/schemas/nodes.py,sha256=bm2ui0rx46UTzlI-Fbvh0BkGWNqnUBPBz-uNL5XAzPI,319
 datapilot/schemas/sql.py,sha256=swvQKX0E2t9K48zB7NiqmPl8yZgfzILMHwBdzAwe1hQ,193
 datapilot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapilot/utils/utils.py,sha256=XGi0jn030M2Cp_WJ-cIOZzdzRewEOQrPldzcaDj7uPw,11150
+datapilot/utils/utils.py,sha256=MY8q6ZBJ0hkrTuH7gWMxAlEAQGrajXFMabEhtGtT7sc,11524
 datapilot/utils/formatting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapilot/utils/formatting/utils.py,sha256=rAVmIYuldvw9VvCSwG2kMTEgiT7cEconp_F1sAWVyCo,1377
-altimate_datapilot_cli-0.0.10.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
-altimate_datapilot_cli-0.0.10.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
-altimate_datapilot_cli-0.0.10.dist-info/METADATA,sha256=sVAlnm0mvyJ-cSbcc25WA14HNwuWZp5myRiUXDJ3Dc8,2399
-altimate_datapilot_cli-0.0.10.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-altimate_datapilot_cli-0.0.10.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
-altimate_datapilot_cli-0.0.10.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
-altimate_datapilot_cli-0.0.10.dist-info/RECORD,,
+altimate_datapilot_cli-0.0.12.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
+altimate_datapilot_cli-0.0.12.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
+altimate_datapilot_cli-0.0.12.dist-info/METADATA,sha256=XCKSmeQwcKzUJqlhoa9F3wsArWyp2FYUSZoJow4R2yk,2365
+altimate_datapilot_cli-0.0.12.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+altimate_datapilot_cli-0.0.12.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
+altimate_datapilot_cli-0.0.12.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
+altimate_datapilot_cli-0.0.12.dist-info/RECORD,,

datapilot/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.10"
+__version__ = "0.0.12"

datapilot/clients/altimate/client.py

@@ -62,7 +62,7 @@ class APIClient:
         response = requests.post(url, headers=headers, json=data, timeout=timeout)
         self.logger.debug(f"Received POST response with status: {response.status_code }")
 
-        return response
+        return response.json()
 
     def put(self, endpoint, data, timeout=None):
         url = f"{self.base_url}{endpoint}"
@@ -83,3 +83,7 @@ class APIClient:
     def validate_credentials(self):
         endpoint = "/dbt/v3/validate-credentials"
         return self.get(endpoint)
+
+    def start_dbt_ingestion(self, params=None):
+        endpoint = "/dbt/v1/start_dbt_ingestion"
+        return self.post(endpoint, data=params)

datapilot/clients/altimate/utils.py

@@ -46,10 +46,14 @@ def validate_credentials(
     return api_client.validate_credentials()
 
 
-def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path, backend_url) -> Dict:
+def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
     api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
 
-    params = {"dbt_core_integration_id": dbt_core_integration_id, "file_type": "manifest"}
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+        "file_type": file_type,
+    }
     signed_url_data = api_client.get_signed_url(params)
     if signed_url_data:
         signed_url = signed_url_data.get("url")
@@ -57,7 +61,7 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path,
         api_client.log(f"Received signed URL: {signed_url}")
         api_client.log(f"Received File ID: {file_id}")
 
-        upload_response = upload_content_to_signed_url(manifest_path, signed_url)
+        upload_response = upload_content_to_signed_url(file_path, signed_url)
 
         if upload_response:
             verify_params = {"dbt_core_integration_file_id": file_id}
@@ -73,3 +77,20 @@ def onboard_manifest(api_token, tenant, dbt_core_integration_id, manifest_path,
             "ok": False,
             "message": "Error in uploading the manifest.                                                                                                                              ",
         }
+
+
+def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, backend_url):
+    api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
+    params = {
+        "dbt_core_integration_id": dbt_core_integration_id,
+        "dbt_core_integration_environment_type": dbt_core_integration_environment,
+    }
+    data = api_client.start_dbt_ingestion(params)
+    if data and data.get("ok"):
+        return {"ok": True}
+    else:
+        api_client.log("Error starting dbt ingestion worker")
+        return {
+            "ok": False,
+            "message": "Error starting dbt ingestion worker.                                                                                                                              ",
+        }

datapilot/core/platforms/dbt/cli/cli.py

@@ -3,7 +3,8 @@ import logging
 import click
 
 from datapilot.clients.altimate.utils import check_token_and_instance
-from datapilot.clients.altimate.utils import onboard_manifest
+from datapilot.clients.altimate.utils import onboard_file
+from datapilot.clients.altimate.utils import start_dbt_ingestion
 from datapilot.clients.altimate.utils import validate_credentials
 from datapilot.config.config import load_config
 from datapilot.core.platforms.dbt.constants import MODEL
@@ -14,6 +15,7 @@ from datapilot.core.platforms.dbt.formatting import generate_project_insights_ta
 from datapilot.core.platforms.dbt.utils import load_catalog
 from datapilot.core.platforms.dbt.utils import load_manifest
 from datapilot.utils.formatting.utils import tabulate_data
+from datapilot.utils.utils import map_url_to_instance
 
 logging.basicConfig(level=logging.INFO)
 
@@ -87,9 +89,21 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
 @click.option("--token", prompt="API Token", help="Your API token for authentication.")
 @click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.")
 @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID")
+@click.option(
+    "--dbt_core_integration_environment", default="PROD", prompt="DBT Core Integration Environment", help="DBT Core Integration Environment"
+)
 @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.")
+@click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.")
 @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
-def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backend_url="https://api.myaltimate.com", env=None):
+def onboard(
+    token,
+    instance_name,
+    dbt_core_integration_id,
+    dbt_core_integration_environment,
+    manifest_path,
+    catalog_path,
+    backend_url="https://api.myaltimate.com",
+):
     """Onboard a manifest file to DBT."""
     check_token_and_instance(token, instance_name)
 
@@ -104,9 +118,32 @@ def onboard(token, instance_name, dbt_core_integration_id, manifest_path, backen
         click.echo(f"Error: {e}")
         return
 
-    response = onboard_manifest(token, instance_name, dbt_core_integration_id, manifest_path, backend_url)
-
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "manifest", manifest_path, backend_url
+    )
     if response["ok"]:
         click.echo("Manifest onboarded successfully!")
     else:
         click.echo(f"{response['message']}")
+
+    if not catalog_path:
+        return
+
+    response = onboard_file(
+        token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, "catalog", catalog_path, backend_url
+    )
+    if response["ok"]:
+        click.echo("Catalog onboarded successfully!")
+    else:
+        click.echo(f"{response['message']}")
+
+    response = start_dbt_ingestion(token, instance_name, dbt_core_integration_id, dbt_core_integration_environment, backend_url)
+    if response["ok"]:
+        url = map_url_to_instance(backend_url, instance_name)
+        if not url:
+            click.echo("Manifest and catalog ingestion has started.")
+        else:
+            url = f"{url}/settings/integrations/{dbt_core_integration_id}/{dbt_core_integration_environment}"
+            click.echo(f"Manifest and catalog ingestion has started. You can check the status at {url}")
+    else:
+        click.echo(f"{response['message']}")

datapilot/utils/utils.py

@@ -311,7 +311,14 @@ def generate_partial_manifest_catalog(changed_files, base_path: str = "./"):
         raise Exception("Unable to generate partial manifest and catalog") from e
 
 
-if __name__ == "__main__":
-    print("Running main")
-    print(generate_partial_manifest_catalog([], "/Users/gaurp/Desktop/manifest.json", ""))
-    print("Done running main")
+def map_url_to_instance(url, instance):
+    # Base URLs and their corresponding patterns
+    url_mapping = {
+        "https://api.tryaltimate.com": f"https://{instance}.demo.tryaltimate.com",
+        "https://api.myaltimate.com": f"https://{instance}.app.myaltimate.com",
+        "https://api.getaltimate.com": f"https://{instance}.app.getaltimate.com",
+        "http://localhost:8000": f"http://{instance}.localhost:3000",
+    }
+
+    # Check if the URL is in the dictionary and return the corresponding instance URL
+    return url_mapping.get(url)