bigeye-cli 0.3.60__tar.gz → 0.3.62__tar.gz

{bigeye_cli-0.3.60 → bigeye_cli-0.3.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigeye-cli
-Version: 0.3.60
+Version: 0.3.62
 Summary: Bigeye CLI offers developer tools for maintaining your developer workspace.
 Home-page: https://docs.bigeye.com/docs
 License: Proprietary
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: PyGithub (>=1.57,<2.0)
-Requires-Dist: bigeye-sdk (>=0.4.80,<0.5.0)
+Requires-Dist: bigeye-sdk (>=0.4.83,<0.5.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
 Requires-Dist: rich (>=12.4.4,<13.0.0)
 Description-Content-Type: text/markdown

bigeye_cli-0.3.62/bigeye_cli/bigconfig/bigconfig_commands.py

@@ -0,0 +1,198 @@
+import os
+from pathlib import Path
+from typing import Optional, List
+
+import typer
+
+from bigeye_cli.functions import cli_client_factory
+from bigeye_cli import global_options
+from bigeye_cli.bigconfig import bigconfig_options
+from bigeye_cli.model.github_report import GitHubReport
+from bigeye_sdk.controller.metric_controller import MetricController
+from bigeye_sdk.controller.metric_suite_controller import MetricSuiteController
+from bigeye_sdk.generated.com.bigeye.models.generated import MetricInfoList
+from bigeye_sdk.model.protobuf_message_facade import SimpleCollection
+from bigeye_sdk.log import get_logger
+
+log = get_logger(__file__)
+
+app = typer.Typer(no_args_is_help=True, help="Bigconfig Commands for Bigeye CLI")
+
+"""
+File should contain commands relating to deploying Bigconfig files.
+"""
+
+
+@app.command()
+def plan(
+    bigeye_conf: str = global_options.bigeye_conf,
+    config_file: str = global_options.config_file,
+    workspace: str = global_options.workspace,
+    input_path: Optional[List[str]] = bigconfig_options.input_path,
+    output_path: str = bigconfig_options.output_path,
+    purge_source_names: Optional[List[str]] = bigconfig_options.purge_source_names,
+    purge_all_sources: bool = bigconfig_options.purge_all_sources,
+    no_queue: bool = bigconfig_options.no_queue,
+    recursive: bool = bigconfig_options.recursive,
+    strict_mode: bool = bigconfig_options.strict_mode,
+    cicd: bool = typer.Option(
+        False,
+        "--cicd_report",
+        "-cicd",
+        help="Add the report to a pull request. (Github only)",
+    ),
+):
+    """Executes a plan for purging sources or processing bigconfig files in the input path/current
+    working directory."""
+
+    client = cli_client_factory(bigeye_conf, config_file, workspace)
+    mc = MetricSuiteController(client=client)
+
+    # Resolve args to first true or first non None values
+    if not input_path:
+        input_path = client.config.bigconfig_input_path or [Path.cwd()]
+    if not output_path:
+        output_path = client.config.bigconfig_output_path or Path.cwd()
+    strict_mode = strict_mode or client.config.bigconfig_strict_mode
+
+    if purge_source_names or purge_all_sources:
+        report = mc.execute_purge(
+            purge_source_names=purge_source_names,
+            purge_all_sources=purge_all_sources,
+            output_path=output_path,
+            apply=False,
+        )
+    else:
+        report = mc.execute_bigconfig(
+            input_path=input_path,
+            output_path=output_path,
+            apply=False,
+            recursive=recursive,
+            no_queue=no_queue,
+            strict_mode=strict_mode,
+        )
+    if cicd:
+        gh_report = GitHubReport(github_token=os.environ["GITHUB_TOKEN"])
+        gh_report.publish_bigconfig_plan(plan=report)
+
+
+@app.command()
+def apply(
+    bigeye_conf: str = global_options.bigeye_conf,
+    config_file: str = global_options.config_file,
+    workspace: str = global_options.workspace,
+    input_path: Optional[List[str]] = bigconfig_options.input_path,
+    output_path: str = bigconfig_options.output_path,
+    purge_source_names: Optional[List[str]] = bigconfig_options.purge_source_names,
+    purge_all_sources: bool = bigconfig_options.purge_all_sources,
+    no_queue: bool = bigconfig_options.no_queue,
+    recursive: bool = bigconfig_options.recursive,
+    strict_mode: bool = bigconfig_options.strict_mode,
+    auto_approve: bool = bigconfig_options.auto_approve,
+):
+    """Applies a purge of deployed metrics or applies Bigconfig files from the input path/current working directory to
+    the Bigeye workspace."""
+
+    client = cli_client_factory(bigeye_conf, config_file, workspace)
+    mc = MetricSuiteController(client=client)
+
+    # Resolve args to first true or first non None values
+    if not input_path:
+        input_path = client.config.bigconfig_input_path or [Path.cwd()]
+    if not output_path:
+        output_path = client.config.bigconfig_output_path or Path.cwd()
+    strict_mode = strict_mode or client.config.bigconfig_strict_mode
+    auto_approve = auto_approve or client.config.bigconfig_auto_approve
+
+    if purge_source_names or purge_all_sources:
+        mc.execute_purge(
+            purge_source_names=purge_source_names,
+            purge_all_sources=purge_all_sources,
+            output_path=output_path,
+            apply=True,
+        )
+    else:
+        mc.execute_bigconfig(
+            input_path=input_path,
+            output_path=output_path,
+            apply=True,
+            no_queue=no_queue,
+            recursive=recursive,
+            strict_mode=strict_mode,
+            auto_approve=auto_approve,
+        )
+
+
+@app.command()
+def export(
+    bigeye_conf: str = global_options.bigeye_conf,
+    config_file: str = global_options.config_file,
+    workspace: str = global_options.workspace,
+    output_path: str = bigconfig_options.output_path,
+    collection_id: int = typer.Option(
+        None, "--collection_id", "-cid", help="Collection ID of metrics to be exported."
+    ),
+    table_id: int = typer.Option(
+        None, "--table_id", "-tid", help="Table ID of metrics to be exported."
+    ),
+    metric_ids: List[int] = typer.Option(
+        None, "--metric_id", "-mid", help="Metric ID of metrics to be exported."
+    ),
+):
+    """Exports existing metrics into a valid Bigconfig file."""
+
+    client = cli_client_factory(bigeye_conf, config_file, workspace)
+    mc = MetricController(client=client)
+    collection: SimpleCollection = SimpleCollection(
+        name="Sample", description="SDK Generated"
+    )
+
+    # Require at least 1 option to be specified
+    if not collection_id and not table_id and not metric_ids:
+        raise typer.BadParameter(
+            "At least 1 option must be provided for bigconfig export. Please provide the "
+            "collection id (-cid), table id (-tid), or list of metric ids (-mid). "
+        )
+
+    # Get existing collection if passed
+    if collection_id:
+        collection: SimpleCollection = SimpleCollection.from_datawatch_object(
+            client.get_collection(collection_id=collection_id).collection
+        )
+        metric_ids.extend(collection.metric_ids)
+
+    # Get metrics from table if passed
+    if table_id:
+        metrics_from_table = client.get_metric_info_batch_post(
+            table_ids=[table_id]
+        ).metrics
+        metric_ids.extend([m.metric_configuration.id for m in metrics_from_table])
+        if not metrics_from_table:
+            raise typer.BadParameter(
+                f"No metrics found for given table id. Please check arguments and try again."
+            )
+        if not collection_id:
+            collection.name = metrics_from_table[0].metric_metadata.dataset_name
+
+    # Pull metric information
+    metrics: MetricInfoList = client.get_metric_info_batch_post(metric_ids=metric_ids)
+
+    if not output_path:
+        output_path = client.config.bigconfig_output_path or Path.cwd()
+
+    if not metrics.metrics:
+        raise typer.BadParameter(
+            "No metrics found for given arguments. Please check again."
+        )
+
+    bigconfig = mc.metric_info_to_bigconfig(metric_info=metrics, collection=collection)
+
+    file_name = collection.name.lower().replace(" ", "_")
+    output = bigconfig.save(
+        output_path=output_path,
+        default_file_name=f"{file_name}.bigconfig",
+        custom_formatter=bigconfig.format_bigconfig_export,
+    )
+    typer.secho(
+        f"\nSample bigconfig file generated at: {output}\n", fg="green", bold=True
+    )

{bigeye_cli-0.3.60 → bigeye_cli-0.3.62}/bigeye_cli/catalog/catalog_commands.py

@@ -1,23 +1,37 @@
 import json
 import logging
+import os
+import sys
+import time
+from collections import defaultdict
+from datetime import datetime
 from pathlib import Path
 from typing import Optional, List, Dict
 
 import typer
+import yaml
 from rich.progress import track
 
+from bigeye_cli.exceptions.exceptions import MissingArgumentException
 from bigeye_sdk.client.enum import Method
 
 from bigeye_cli.enums import OPS_METRICS
 from bigeye_cli import global_options
 from bigeye_sdk.functions.dbt import parse_dbt_manifest
 from bigeye_sdk.functions.file_functs import read_json_file
+from bigeye_sdk.functions.notifications import (
+    send_email,
+    post_slack_snippet,
+    post_webhook_request,
+)
 from bigeye_sdk.functions.search_and_match_functions import fuzzy_match
 from bigeye_sdk.model.dbt_manifest import DbtManifest
 from bigeye_sdk.model.protobuf_enum_facade import (
     MetricStatus,
     SimpleMetricCategory,
     SimpleMetricTemplateFieldType,
+    SimpleSchemaChangeOperation,
+    SimpleCatalogEntityType,
 )
 from bigeye_sdk.functions.aws import get_secret
 from bigeye_sdk.generated.com.bigeye.models.generated import (
@@ -33,6 +47,8 @@ from bigeye_sdk.generated.com.bigeye.models.generated import (
     Table,
     User,
     OwnableType,
+    SchemaChangeOperation,
+    SchemaChange,
 )
 from bigeye_sdk.log import get_logger
 from bigeye_cli.functions import (
@@ -44,6 +60,7 @@ from bigeye_cli.functions import (
 )
 from bigeye_sdk.functions.metric_functions import has_auto_threshold
 from bigeye_sdk.functions.core_py_functs import int_enum_enum_list_joined
+from bigeye_sdk.model.schema_change import SourceSchemaChange
 
 # create logger
 log = get_logger(__file__)
@@ -430,11 +447,11 @@ def add_source(
 
     if secret_name:
         secret = get_secret(secret_name, secret_region_name)
-        source_config = json.loads(secret["SecretString"])
+        source_config = yaml.safe_load(secret["SecretString"])
     else:
         log.info(f"Loading source config file: {source_catalog_config_file}")
-        with open(source_catalog_config_file) as json_file:
-            source_config = json.load(json_file)
+        with open(source_catalog_config_file) as fin:
+            source_config = yaml.safe_load(fin)
 
     db_type = source_config["database_type"].upper()
     source_config["database_type"] = WarehouseType[f"DATABASE_TYPE_{db_type}"]
@@ -803,26 +820,26 @@ def upsert_template(
     config_file: str = global_options.config_file,
     workspace: str = global_options.workspace,
     source_name: str = typer.Option(
-        ...,
+        None,
         "--source_name",
         "-sn",
         help="The name of the source where the template will be defined",
     ),
     template_name: str = typer.Option(
-        ..., "--name", "-tn", help="The user defined name of the template"
+        None, "--name", "-tn", help="The user defined name of the template"
     ),
     template_body: str = typer.Option(
-        ..., "--body", "-tb", help="The string to define the template"
+        None, "--body", "-tb", help="The string to define the template"
     ),
     return_type: SimpleMetricTemplateFieldType = typer.Option(
-        ...,
+        None,
         "--returns",
         "-rt",
         help="The data type returned by the template; i.e. NUMERIC, BOOLEAN",
     ),
     # TODO: Update to List[Tuple[str, SimpleMetricTemplateParameterType]] when supported by typer.
     parameters: List[str] = typer.Option(
-        ...,
+        None,
         "--params",
         "-p",
         help="A list of key/value pairs for parameters; ex. -p column=COLUMN_REFERENCE -p table=STRING",
@@ -833,11 +850,27 @@ def upsert_template(
         "-id",
         help="The template ID (Only required when updating a template).",
     ),
+    template_file: Optional[str] = typer.Option(
+        None,
+        "--template_file",
+        "-tf",
+        help="A file containing a template definition.",
+    )
 ):
     """Create or update a template for a source."""
     # TODO: Lookup template ID, once template names are unique
     client = cli_client_factory(bigeye_conf, config_file, workspace)
 
+    if template_file:
+        with open(template_file) as fin:
+            template = yaml.safe_load(stream=fin)
+        source_name = template["source_name"]
+        template_name = template["name"]
+        template_body = template["body"]
+        return_type = template["return_type"]
+        parameters = template["parameters"]
+
+
     source: Source = [s for s in client.get_sources().sources if s.name == source_name][
         0
     ]
@@ -1089,7 +1122,10 @@ def ingest_dbt_core_run_info(
         help="The path to the target directory created with dbt run, build, and test commands.",
     ),
     project_name: Optional[str] = typer.Option(
-        None, "--project_name", "-pn", help="The name of the dbt project. Will attempt to parse from metadata section of manifest.json, if not present"
+        None,
+        "--project_name",
+        "-pn",
+        help="The name of the dbt project. Will attempt to parse from metadata section of manifest.json, if not present",
     ),
     job_name: Optional[str] = typer.Option(
         None, "--job_name", "-jn", help="The name of the dbt job."
@@ -1122,9 +1158,192 @@ def ingest_dbt_core_run_info(
         run_results_json=json.dumps(run_results_json),
         project_url=project_url,
         job_url=job_url,
-        job_run_url=job_run_url
+        job_run_url=job_run_url,
     )
 
     log.info(
         f"The following response was returned:\n\n\t{response.to_json(indent=True)}"
     )
+
+
+@app.command()
+def track_schema_changes(
+    bigeye_conf: str = global_options.bigeye_conf,
+    config_file: str = global_options.config_file,
+    workspace: str = global_options.workspace,
+    source_names: List[str] = typer.Option(
+        [],
+        "--source",
+        "-src",
+        help="A list of source names to track. (None will track changes from all sources)",
+    ),
+    since: Optional[str] = typer.Option(
+        "1970-01-01",
+        "--since",
+        "-s",
+        help="Track schema changes from this date. (Formatted as yyyy-MM-dd)",
+    ),
+    frequency: Optional[int] = typer.Option(
+        0,
+        "--frequency",
+        "-f",
+        help="Every n hours to check for schema changes. (None will only run the check once)",
+    ),
+    change_types: List[SimpleSchemaChangeOperation] = typer.Option(
+        [
+            SimpleSchemaChangeOperation.DELETED,
+            SimpleSchemaChangeOperation.CREATED,
+            SimpleSchemaChangeOperation.TYPE_CHANGED,
+        ],
+        "--change_type",
+        "-ct",
+        help="The types of changes to track; i.e additions, changes, deletions. (None will track all types)",
+    ),
+    notification_channels: List[str] = typer.Option(
+        [],
+        "--channel",
+        "-nc",
+        help="A list of notification channels; e.g. -nc email@mydomain.com -nc #slack_channel -nc https://www.webhook.com",
+    ),
+    smtp_server: Optional[str] = typer.Option(
+        None, "--server", "-smtp", help="The server to send emails."
+    ),
+    smtp_port: Optional[int] = typer.Option(
+        587, "--port", "-p", help="The server port to send emails."
+    ),
+    slack_token: Optional[str] = typer.Option(
+        None,
+        "--token",
+        "-t",
+        help="The token to utilize the Slack web client. (Will attempt to use environment variable SLACK_BOT_TOKEN, if not set)",
+    ),
+):
+    """Track schema changes for sources on demand or every n hour(s) and notify specified channels.
+    Environment variables SMTP_USER and SMTP_PASSWORD MUST be set if using email notification.
+    """
+
+    hour_as_secs = 3600
+    no_reply_at_bigeye = "noreply@bigeye.com"
+
+    try:
+        date_epoch = datetime.strptime(since, "%Y-%m-%d").timestamp()
+    except ValueError as e:
+        log.error(
+            f"Error converting date {since}. Please ensure it formatted as yyyy-MM-dd."
+        )
+        log.info("The process will exit.")
+        raise e
+
+    emails = []
+    slack_channels = []
+    webhooks = []
+    for nc in notification_channels:
+        if "@" in nc:
+            emails.append(nc)
+        elif nc.startswith("#"):
+            slack_channels.append(nc)
+        elif nc.startswith("http"):
+            webhooks.append(nc)
+        else:
+            log.error(f"Unrecognized notification channel: {nc}")
+            log.warning(f"Schema changes will not be sent to {nc}")
+
+    if emails:
+        if not smtp_server:
+            raise MissingArgumentException(
+                "Email listed as notification channel but no email server specified to send one."
+            )
+        if not os.environ.get("SMTP_USER", None):
+            raise MissingArgumentException(
+                "Email listed as notification channel but no value set for environment variable SMTP_USER."
+            )
+        if not os.environ.get("SMTP_PASSWORD", None):
+            raise MissingArgumentException(
+                "Email listed as notification channel but no value set for environment variable SMTP_PASSWORD."
+            )
+
+    if slack_channels:
+        if not slack_token and not os.environ.get("SLACK_BOT_TOKEN", None):
+            raise MissingArgumentException(
+                "Slack listed as notification channel but no value set for slack token or environment variable SLACK_BOT_TOKEN."
+            )
+        if not slack_token:
+            slack_token = os.environ.get("SLACK_BOT_TOKEN")
+
+    change_types = [ct.to_datawatch_object() for ct in change_types]
+
+    client = cli_client_factory(bigeye_conf, config_file, workspace)
+
+    sources = client.get_sources_by_name(source_names=source_names)
+    while True:
+        changes_by_source: Dict[str, List[SchemaChange]] = {}
+        for s in sources.values():
+
+            changes = []
+            since_time = datetime.fromtimestamp(timestamp=date_epoch).strftime(
+                "%m/%d/%Y, %H:%M"
+            )
+            log.info(f"Retrieving changes for source {s.name} since {since_time}")
+            changes.extend(
+                [
+                    c
+                    for c in client.fetch_schema_changes(source_id=s.id, page_size=1000)
+                    if c.change_type in change_types and c.detected_at >= date_epoch
+                ]
+            )
+            if changes:
+                changes_by_source[s.name] = changes
+            else:
+                continue
+
+        source_changes = [
+            SourceSchemaChange(name=k, changes=v) for k, v in changes_by_source.items()
+        ]
+
+        all_alerts = ""
+        alert_webhook = {}
+
+        if source_changes:
+
+            for change in source_changes:
+                all_alerts += f"<br><br>{change.format_message()}</br></br>"
+                alert_webhook.update(change.format_webhook())
+
+
+            if emails:
+                send_email(
+                    server_name=smtp_server,
+                    port=smtp_port,
+                    user_name=os.environ.get("SMTP_USER"),
+                    password=os.environ.get("SMTP_PASSWORD"),
+                    sender=no_reply_at_bigeye,
+                    subject=f"Schema changes detected in Bigeye since {since_time}",
+                    recipient=emails,
+                    body=all_alerts,
+                )
+
+            if slack_channels:
+                post_slack_snippet(
+                    slack_token=slack_token,
+                    slack_channel=slack_channels,
+                    body=alert_webhook,
+                    text=f"*Bigeye schema changes since {since_time}*",
+                    title="Bigeye Schema Changes",
+                )
+
+            if webhooks:
+                data = {
+                    "name": "Bigeye Schema Changes",
+                    "since": since_time,
+                    "alerts": alert_webhook,
+                }
+                post_webhook_request(webhook_url=webhooks, data=data)
+
+        if frequency:
+            log.info(f"Continuing to track schema changes.")
+            log.info(f"The process will run again in {frequency} hours.")
+            date_epoch = datetime.now().timestamp()
+            time.sleep(frequency * hour_as_secs)
+        else:
+            log.info("No frequency specified. The process will exit.")
+            sys.exit(0)