bigeye-cli 0.3.61__tar.gz → 0.3.63__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigeye_cli-0.3.61/bigeye_cli/CLI_DOCS.md → bigeye_cli-0.3.63/PKG-INFO +34 -59
- bigeye_cli-0.3.61/PKG-INFO → bigeye_cli-0.3.63/bigeye_cli/CLI_DOCS.md +12 -81
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/bigconfig/bigconfig_commands.py +9 -3
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/configure/configure_commands.py +23 -13
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/deltas/deltas_commands.py +37 -16
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/lineage/lineage_commands.py +55 -11
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/model/github_report.py +21 -17
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/model/vendor_report.py +13 -15
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/pyproject.toml +2 -2
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/LICENSE +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/__main__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/__version__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/bigconfig/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/bigconfig/bigconfig_options.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/catalog/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/catalog/catalog_commands.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/collections/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/collections/collection_commands.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/configure/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/deltas/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/enums.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/exceptions/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/exceptions/exceptions.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/functions.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/global_options.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/issues/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/issues/issue_commands.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/lineage/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/metric/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/metric/metric_commands.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/model/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/model/cicd_conf.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/workspace/__init__.py +0 -0
- {bigeye_cli-0.3.61 → bigeye_cli-0.3.63}/bigeye_cli/workspace/workspace_commands.py +0 -0
|
@@ -1,5 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: bigeye-cli
|
|
3
|
+
Version: 0.3.63
|
|
4
|
+
Summary: Bigeye CLI offers developer tools for maintaining your developer workspace.
|
|
5
|
+
Home-page: https://docs.bigeye.com/docs
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Author: Bigeye
|
|
8
|
+
Author-email: support@bigeye.com
|
|
9
|
+
Requires-Python: >=3.9.0,<4.0.0
|
|
10
|
+
Classifier: License :: Other/Proprietary License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Requires-Dist: PyGithub (>=1.57,<2.0)
|
|
17
|
+
Requires-Dist: bigeye-sdk (>=0.4.84,<0.5.0)
|
|
18
|
+
Requires-Dist: click (>=8.0.4,<9.0.0)
|
|
19
|
+
Requires-Dist: rich (>=12.4.4,<13.0.0)
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
1
22
|
# **Bigeye CLI**
|
|
2
|
-
Version: 0.3.
|
|
23
|
+
Version: 0.3.55
|
|
3
24
|
## Installation
|
|
4
25
|
|
|
5
26
|
**Setting up your local/build environment**
|
|
@@ -171,7 +192,9 @@ password = fakepassword1234
|
|
|
171
192
|
|
|
172
193
|
**Note:** Most use cases will only require you to have a single `DEFAULT` credential that all workspaces can utilize. Running the `bigeye configure -w <workspace-name>` command for a non-default workspace will prompt you to inherit the credentials from default or create a new set of credentials. A workspace credential is only required if the `use_default_credential` setting is set to False in the configuration of the corresponding workspace.
|
|
173
194
|
|
|
174
|
-
# CLI Documentation
|
|
195
|
+
# CLI Documentation
|
|
196
|
+
|
|
197
|
+
# `bigeye configure`
|
|
175
198
|
|
|
176
199
|
Configuration Commands for Bigeye CLI
|
|
177
200
|
|
|
@@ -285,7 +308,6 @@ $ bigeye workspace [OPTIONS] COMMAND [ARGS]...
|
|
|
285
308
|
* `schedule-all-metrics`: Schedule all metrics in a workspace.
|
|
286
309
|
* `unschedule-all-metrics`: Unschedule all metrics in a workspace.
|
|
287
310
|
* `upsert-group`: Create or update a group.
|
|
288
|
-
* `upsert-named-schedule`: Create a named, cron based schedule
|
|
289
311
|
* `upsert-workspace`: Create or update a workspace.
|
|
290
312
|
|
|
291
313
|
## `bigeye workspace create-named-schedule`
|
|
@@ -512,25 +534,6 @@ $ bigeye workspace upsert-group [OPTIONS]
|
|
|
512
534
|
* `-id, --group_id INTEGER`: Group ID.
|
|
513
535
|
* `--help`: Show this message and exit.
|
|
514
536
|
|
|
515
|
-
## `bigeye workspace upsert-named-schedule`
|
|
516
|
-
|
|
517
|
-
Create a named, cron based schedule
|
|
518
|
-
|
|
519
|
-
**Usage**:
|
|
520
|
-
|
|
521
|
-
```console
|
|
522
|
-
$ bigeye workspace upsert-named-schedule [OPTIONS]
|
|
523
|
-
```
|
|
524
|
-
|
|
525
|
-
**Options**:
|
|
526
|
-
|
|
527
|
-
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
528
|
-
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
529
|
-
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
530
|
-
* `-sn, --name TEXT`: The user defined name of the schedule [required]
|
|
531
|
-
* `-c, --cron TEXT`: The cron string to define the schedule [required]
|
|
532
|
-
* `--help`: Show this message and exit.
|
|
533
|
-
|
|
534
537
|
## `bigeye workspace upsert-workspace`
|
|
535
538
|
|
|
536
539
|
Create or update a workspace.
|
|
@@ -642,7 +645,6 @@ $ bigeye bigconfig plan [OPTIONS]
|
|
|
642
645
|
* `-nq, --no_queue`: Don't submit to queue: True or False.
|
|
643
646
|
* `-r, --recursive`: Search all input directories recursively.
|
|
644
647
|
* `-strict, --strict_mode`: API errors cause an exception if True. (Validation errors still cause an exception)
|
|
645
|
-
* `-cicd, --cicd_report`: Add the report to a pull request. (Github only)
|
|
646
648
|
* `--help`: Show this message and exit.
|
|
647
649
|
|
|
648
650
|
# `bigeye catalog`
|
|
@@ -682,7 +684,6 @@ $ bigeye catalog [OPTIONS] COMMAND [ARGS]...
|
|
|
682
684
|
* `run-metrics`: Runs metrics by warehouse id, schema...
|
|
683
685
|
* `schedule-all-metrics`: Updates schedule for all metrics in a...
|
|
684
686
|
* `set-metric-time`: Sets metric times from a list of possible...
|
|
685
|
-
* `track-schema-changes`: Track schema changes for sources on demand...
|
|
686
687
|
* `unschedule-all-metrics`: Unschedule all metrics by warehouse,...
|
|
687
688
|
* `unset-metric-time`: Unsets metric times for whole warehouse or...
|
|
688
689
|
* `upsert-template`: Create or update a template for a source.
|
|
@@ -770,7 +771,7 @@ $ bigeye catalog delete-metrics [OPTIONS]
|
|
|
770
771
|
* `-wid, --warehouse_id INTEGER`: Warehouse ID
|
|
771
772
|
* `-sn, --schema_name TEXT`: List of Schema Names. E.g. -sn schema_1 -sn schema_2.
|
|
772
773
|
* `-tid, --table_id INTEGER`: Table IDs. E.g. -tid 123 -tid 124
|
|
773
|
-
* `-m, --metric_type TEXT`: Delete by name of the metric type.UNDEFINED_PREDEFINED_METRIC_NAME, PERCENT_NULL, COUNT_NULL, PERCENT_EMPTY_STRING, COUNT_EMPTY_STRING, PERCENT_UNIQUE, PERCENT_VALUE_IN_LIST, AVERAGE, MIN, MAX, SUM, COUNT_ROWS, COUNT_DISTINCT, HOURS_SINCE_MAX_DATE, HOURS_SINCE_MAX_TIMESTAMP, COUNT_TRUE, PERCENT_TRUE, COUNT_FALSE, PERCENT_FALSE, COUNT_USA_PHONE, PERCENT_USA_PHONE, COUNT_USA_ZIP_CODE, PERCENT_USA_ZIP_CODE, PERCENT_UUID, COUNT_TIMESTAMP_STRING, PERCENT_TIMESTAMP_STRING, COUNT_DUPLICATES, COUNT_USA_STATE_CODE, PERCENT_USA_STATE_CODE, VARIANCE, SKEW, KURTOSIS, GEOMETRIC_MEAN, HARMONIC_MEAN, COUNT_UUID, COUNT_CUSIP, PERCENT_CUSIP, COUNT_SEDOL, PERCENT_SEDOL, COUNT_ISIN, PERCENT_ISIN, COUNT_LEI, PERCENT_LEI, COUNT_FIGI, PERCENT_FIGI, COUNT_PERM_ID, PERCENT_PERM_ID, COUNT_NAN, PERCENT_NAN, COUNT_LONGITUDE, PERCENT_LONGITUDE, COUNT_LATITUDE, PERCENT_LATITUDE, COUNT_NOT_IN_FUTURE, PERCENT_NOT_IN_FUTURE, COUNT_DATE_NOT_IN_FUTURE, PERCENT_DATE_NOT_IN_FUTURE, MEDIAN, PERCENTILE, COUNT_NOT_NULL, STRING_LENGTH_AVERAGE, STRING_LENGTH_MIN, STRING_LENGTH_MAX, COUNT_SSN, PERCENT_SSN, COUNT_EMAIL, PERCENT_EMAIL, ROWS_INSERTED, HOURS_SINCE_LAST_LOAD, COUNT_READ_QUERIES, PERCENT_NOT_NULL, FRESHNESS, VOLUME, FRESHNESS_DATA, VOLUME_DATA
|
|
774
|
+
* `-m, --metric_type TEXT`: Delete by name of the metric type.UNDEFINED_PREDEFINED_METRIC_NAME, PERCENT_NULL, COUNT_NULL, PERCENT_EMPTY_STRING, COUNT_EMPTY_STRING, PERCENT_UNIQUE, PERCENT_VALUE_IN_LIST, AVERAGE, MIN, MAX, SUM, COUNT_ROWS, COUNT_DISTINCT, HOURS_SINCE_MAX_DATE, HOURS_SINCE_MAX_TIMESTAMP, COUNT_TRUE, PERCENT_TRUE, COUNT_FALSE, PERCENT_FALSE, COUNT_USA_PHONE, PERCENT_USA_PHONE, COUNT_USA_ZIP_CODE, PERCENT_USA_ZIP_CODE, PERCENT_UUID, COUNT_TIMESTAMP_STRING, PERCENT_TIMESTAMP_STRING, COUNT_DUPLICATES, COUNT_USA_STATE_CODE, PERCENT_USA_STATE_CODE, VARIANCE, SKEW, KURTOSIS, GEOMETRIC_MEAN, HARMONIC_MEAN, COUNT_UUID, COUNT_CUSIP, PERCENT_CUSIP, COUNT_SEDOL, PERCENT_SEDOL, COUNT_ISIN, PERCENT_ISIN, COUNT_LEI, PERCENT_LEI, COUNT_FIGI, PERCENT_FIGI, COUNT_PERM_ID, PERCENT_PERM_ID, COUNT_NAN, PERCENT_NAN, COUNT_LONGITUDE, PERCENT_LONGITUDE, COUNT_LATITUDE, PERCENT_LATITUDE, COUNT_NOT_IN_FUTURE, PERCENT_NOT_IN_FUTURE, COUNT_DATE_NOT_IN_FUTURE, PERCENT_DATE_NOT_IN_FUTURE, MEDIAN, PERCENTILE, COUNT_NOT_NULL, STRING_LENGTH_AVERAGE, STRING_LENGTH_MIN, STRING_LENGTH_MAX, COUNT_SSN, PERCENT_SSN, COUNT_EMAIL, PERCENT_EMAIL, ROWS_INSERTED, HOURS_SINCE_LAST_LOAD, COUNT_READ_QUERIES, PERCENT_NOT_NULL, FRESHNESS, VOLUME, FRESHNESS_DATA, VOLUME_DATA
|
|
774
775
|
* `--help`: Show this message and exit.
|
|
775
776
|
|
|
776
777
|
## `bigeye catalog delete-source`
|
|
@@ -1093,32 +1094,6 @@ $ bigeye catalog set-metric-time [OPTIONS]
|
|
|
1093
1094
|
* `-r, --replace`: replace metric times if already present on tables? Default is false.
|
|
1094
1095
|
* `--help`: Show this message and exit.
|
|
1095
1096
|
|
|
1096
|
-
## `bigeye catalog track-schema-changes`
|
|
1097
|
-
|
|
1098
|
-
Track schema changes for sources on demand or every n hour(s) and notify specified channels.
|
|
1099
|
-
Environment variables SMTP_USER and SMTP_PASSWORD MUST be set if using email notification.
|
|
1100
|
-
|
|
1101
|
-
**Usage**:
|
|
1102
|
-
|
|
1103
|
-
```console
|
|
1104
|
-
$ bigeye catalog track-schema-changes [OPTIONS]
|
|
1105
|
-
```
|
|
1106
|
-
|
|
1107
|
-
**Options**:
|
|
1108
|
-
|
|
1109
|
-
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
1110
|
-
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
1111
|
-
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1112
|
-
* `-src, --source TEXT`: A list of source names to track. (None will track changes from all sources)
|
|
1113
|
-
* `-s, --since TEXT`: Track schema changes from this date. (Formatted as yyyy-MM-dd) [default: 1970-01-01]
|
|
1114
|
-
* `-f, --frequency INTEGER`: Every n hours to check for schema changes. (None will only run the check once) [default: 0]
|
|
1115
|
-
* `-ct, --change_type [CREATED|DELETED|TYPE_CHANGED]`: The types of changes to track; i.e additions, changes, deletions. (None will track all types) [default: DELETED, CREATED, TYPE_CHANGED]
|
|
1116
|
-
* `-nc, --channel TEXT`: A list of notification channels; e.g. -nc email@mydomain.com -nc #slack_channel -nc https://www.webhook.com
|
|
1117
|
-
* `-smtp, --server TEXT`: The server to send emails.
|
|
1118
|
-
* `-p, --port INTEGER`: The server port to send emails. [default: 587]
|
|
1119
|
-
* `-t, --token TEXT`: The token to utilize the Slack web client. (Will attempt to use environment variable SLACK_BOT_TOKEN, if not set)
|
|
1120
|
-
* `--help`: Show this message and exit.
|
|
1121
|
-
|
|
1122
1097
|
## `bigeye catalog unschedule-all-metrics`
|
|
1123
1098
|
|
|
1124
1099
|
Unschedule all metrics by warehouse, schema or tables.
|
|
@@ -1173,13 +1148,12 @@ $ bigeye catalog upsert-template [OPTIONS]
|
|
|
1173
1148
|
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
1174
1149
|
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
1175
1150
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1176
|
-
* `-sn, --source_name TEXT`: The name of the source where the template will be defined
|
|
1177
|
-
* `-tn, --name TEXT`: The user defined name of the template
|
|
1178
|
-
* `-tb, --body TEXT`: The string to define the template
|
|
1179
|
-
* `-rt, --returns [BOOLEAN|NUMERIC]`: The data type returned by the template; i.e. NUMERIC, BOOLEAN
|
|
1180
|
-
* `-p, --params TEXT`: A list of key/value pairs for parameters; ex. -p column=COLUMN_REFERENCE -p table=STRING
|
|
1151
|
+
* `-sn, --source_name TEXT`: The name of the source where the template will be defined [required]
|
|
1152
|
+
* `-tn, --name TEXT`: The user defined name of the template [required]
|
|
1153
|
+
* `-tb, --body TEXT`: The string to define the template [required]
|
|
1154
|
+
* `-rt, --returns [BOOLEAN|NUMERIC]`: The data type returned by the template; i.e. NUMERIC, BOOLEAN [required]
|
|
1155
|
+
* `-p, --params TEXT`: A list of key/value pairs for parameters; ex. -p column=COLUMN_REFERENCE -p table=STRING [required]
|
|
1181
1156
|
* `-id, --template_id INTEGER`: The template ID (Only required when updating a template). [default: 0]
|
|
1182
|
-
* `-tf, --template_file TEXT`: A file containing a template definition.
|
|
1183
1157
|
* `--help`: Show this message and exit.
|
|
1184
1158
|
|
|
1185
1159
|
## `bigeye catalog upsert-virtual-table`
|
|
@@ -1537,7 +1511,7 @@ $ bigeye deltas suggest-deltas [OPTIONS]
|
|
|
1537
1511
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1538
1512
|
* `-source, --source_selector TEXT`: The pattern of tables in the source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_1.*.* would be all schemas in source_1. [required]
|
|
1539
1513
|
* `-target, --target_selector TEXT`: The pattern of tables in the target to select. Wildcard (*) indicates all tables or all schemas, e.g. source_2.*.* would be all schemas in source_2. [required]
|
|
1540
|
-
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default:
|
|
1514
|
+
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default: MatchType.STRICT]
|
|
1541
1515
|
* `-op, --output_path TEXT`: File to write the delta configuration. [required]
|
|
1542
1516
|
* `--help`: Show this message and exit.
|
|
1543
1517
|
|
|
@@ -1804,6 +1778,7 @@ $ bigeye lineage infer-relations [OPTIONS]
|
|
|
1804
1778
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1805
1779
|
* `-upstream, --upstream_selector TEXT`: The pattern of tables in the upstream source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_1.*.* would be all schemas in source_1.
|
|
1806
1780
|
* `-downstream, --downstream_selector TEXT`: The pattern of tables in the downstream source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_2.*.* would be all schemas in source_2.
|
|
1807
|
-
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default:
|
|
1781
|
+
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default: MatchType.STRICT]
|
|
1808
1782
|
* `--help`: Show this message and exit.
|
|
1809
1783
|
|
|
1784
|
+
|
|
@@ -1,26 +1,5 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: bigeye-cli
|
|
3
|
-
Version: 0.3.61
|
|
4
|
-
Summary: Bigeye CLI offers developer tools for maintaining your developer workspace.
|
|
5
|
-
Home-page: https://docs.bigeye.com/docs
|
|
6
|
-
License: Proprietary
|
|
7
|
-
Author: Bigeye
|
|
8
|
-
Author-email: support@bigeye.com
|
|
9
|
-
Requires-Python: >=3.9.0,<4.0.0
|
|
10
|
-
Classifier: License :: Other/Proprietary License
|
|
11
|
-
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
-
Requires-Dist: PyGithub (>=1.57,<2.0)
|
|
17
|
-
Requires-Dist: bigeye-sdk (>=0.4.81,<0.5.0)
|
|
18
|
-
Requires-Dist: click (>=8.0.4,<9.0.0)
|
|
19
|
-
Requires-Dist: rich (>=12.4.4,<13.0.0)
|
|
20
|
-
Description-Content-Type: text/markdown
|
|
21
|
-
|
|
22
1
|
# **Bigeye CLI**
|
|
23
|
-
Version: 0.3.
|
|
2
|
+
Version: 0.3.55
|
|
24
3
|
## Installation
|
|
25
4
|
|
|
26
5
|
**Setting up your local/build environment**
|
|
@@ -192,7 +171,9 @@ password = fakepassword1234
|
|
|
192
171
|
|
|
193
172
|
**Note:** Most use cases will only require you to have a single `DEFAULT` credential that all workspaces can utilize. Running the `bigeye configure -w <workspace-name>` command for a non-default workspace will prompt you to inherit the credentials from default or create a new set of credentials. A workspace credential is only required if the `use_default_credential` setting is set to False in the configuration of the corresponding workspace.
|
|
194
173
|
|
|
195
|
-
# CLI Documentation
|
|
174
|
+
# CLI Documentation
|
|
175
|
+
|
|
176
|
+
# `bigeye configure`
|
|
196
177
|
|
|
197
178
|
Configuration Commands for Bigeye CLI
|
|
198
179
|
|
|
@@ -306,7 +287,6 @@ $ bigeye workspace [OPTIONS] COMMAND [ARGS]...
|
|
|
306
287
|
* `schedule-all-metrics`: Schedule all metrics in a workspace.
|
|
307
288
|
* `unschedule-all-metrics`: Unschedule all metrics in a workspace.
|
|
308
289
|
* `upsert-group`: Create or update a group.
|
|
309
|
-
* `upsert-named-schedule`: Create a named, cron based schedule
|
|
310
290
|
* `upsert-workspace`: Create or update a workspace.
|
|
311
291
|
|
|
312
292
|
## `bigeye workspace create-named-schedule`
|
|
@@ -533,25 +513,6 @@ $ bigeye workspace upsert-group [OPTIONS]
|
|
|
533
513
|
* `-id, --group_id INTEGER`: Group ID.
|
|
534
514
|
* `--help`: Show this message and exit.
|
|
535
515
|
|
|
536
|
-
## `bigeye workspace upsert-named-schedule`
|
|
537
|
-
|
|
538
|
-
Create a named, cron based schedule
|
|
539
|
-
|
|
540
|
-
**Usage**:
|
|
541
|
-
|
|
542
|
-
```console
|
|
543
|
-
$ bigeye workspace upsert-named-schedule [OPTIONS]
|
|
544
|
-
```
|
|
545
|
-
|
|
546
|
-
**Options**:
|
|
547
|
-
|
|
548
|
-
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
549
|
-
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
550
|
-
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
551
|
-
* `-sn, --name TEXT`: The user defined name of the schedule [required]
|
|
552
|
-
* `-c, --cron TEXT`: The cron string to define the schedule [required]
|
|
553
|
-
* `--help`: Show this message and exit.
|
|
554
|
-
|
|
555
516
|
## `bigeye workspace upsert-workspace`
|
|
556
517
|
|
|
557
518
|
Create or update a workspace.
|
|
@@ -663,7 +624,6 @@ $ bigeye bigconfig plan [OPTIONS]
|
|
|
663
624
|
* `-nq, --no_queue`: Don't submit to queue: True or False.
|
|
664
625
|
* `-r, --recursive`: Search all input directories recursively.
|
|
665
626
|
* `-strict, --strict_mode`: API errors cause an exception if True. (Validation errors still cause an exception)
|
|
666
|
-
* `-cicd, --cicd_report`: Add the report to a pull request. (Github only)
|
|
667
627
|
* `--help`: Show this message and exit.
|
|
668
628
|
|
|
669
629
|
# `bigeye catalog`
|
|
@@ -703,7 +663,6 @@ $ bigeye catalog [OPTIONS] COMMAND [ARGS]...
|
|
|
703
663
|
* `run-metrics`: Runs metrics by warehouse id, schema...
|
|
704
664
|
* `schedule-all-metrics`: Updates schedule for all metrics in a...
|
|
705
665
|
* `set-metric-time`: Sets metric times from a list of possible...
|
|
706
|
-
* `track-schema-changes`: Track schema changes for sources on demand...
|
|
707
666
|
* `unschedule-all-metrics`: Unschedule all metrics by warehouse,...
|
|
708
667
|
* `unset-metric-time`: Unsets metric times for whole warehouse or...
|
|
709
668
|
* `upsert-template`: Create or update a template for a source.
|
|
@@ -791,7 +750,7 @@ $ bigeye catalog delete-metrics [OPTIONS]
|
|
|
791
750
|
* `-wid, --warehouse_id INTEGER`: Warehouse ID
|
|
792
751
|
* `-sn, --schema_name TEXT`: List of Schema Names. E.g. -sn schema_1 -sn schema_2.
|
|
793
752
|
* `-tid, --table_id INTEGER`: Table IDs. E.g. -tid 123 -tid 124
|
|
794
|
-
* `-m, --metric_type TEXT`: Delete by name of the metric type.UNDEFINED_PREDEFINED_METRIC_NAME, PERCENT_NULL, COUNT_NULL, PERCENT_EMPTY_STRING, COUNT_EMPTY_STRING, PERCENT_UNIQUE, PERCENT_VALUE_IN_LIST, AVERAGE, MIN, MAX, SUM, COUNT_ROWS, COUNT_DISTINCT, HOURS_SINCE_MAX_DATE, HOURS_SINCE_MAX_TIMESTAMP, COUNT_TRUE, PERCENT_TRUE, COUNT_FALSE, PERCENT_FALSE, COUNT_USA_PHONE, PERCENT_USA_PHONE, COUNT_USA_ZIP_CODE, PERCENT_USA_ZIP_CODE, PERCENT_UUID, COUNT_TIMESTAMP_STRING, PERCENT_TIMESTAMP_STRING, COUNT_DUPLICATES, COUNT_USA_STATE_CODE, PERCENT_USA_STATE_CODE, VARIANCE, SKEW, KURTOSIS, GEOMETRIC_MEAN, HARMONIC_MEAN, COUNT_UUID, COUNT_CUSIP, PERCENT_CUSIP, COUNT_SEDOL, PERCENT_SEDOL, COUNT_ISIN, PERCENT_ISIN, COUNT_LEI, PERCENT_LEI, COUNT_FIGI, PERCENT_FIGI, COUNT_PERM_ID, PERCENT_PERM_ID, COUNT_NAN, PERCENT_NAN, COUNT_LONGITUDE, PERCENT_LONGITUDE, COUNT_LATITUDE, PERCENT_LATITUDE, COUNT_NOT_IN_FUTURE, PERCENT_NOT_IN_FUTURE, COUNT_DATE_NOT_IN_FUTURE, PERCENT_DATE_NOT_IN_FUTURE, MEDIAN, PERCENTILE, COUNT_NOT_NULL, STRING_LENGTH_AVERAGE, STRING_LENGTH_MIN, STRING_LENGTH_MAX, COUNT_SSN, PERCENT_SSN, COUNT_EMAIL, PERCENT_EMAIL, ROWS_INSERTED, HOURS_SINCE_LAST_LOAD, COUNT_READ_QUERIES, PERCENT_NOT_NULL, FRESHNESS, VOLUME, FRESHNESS_DATA, VOLUME_DATA
|
|
753
|
+
* `-m, --metric_type TEXT`: Delete by name of the metric type.UNDEFINED_PREDEFINED_METRIC_NAME, PERCENT_NULL, COUNT_NULL, PERCENT_EMPTY_STRING, COUNT_EMPTY_STRING, PERCENT_UNIQUE, PERCENT_VALUE_IN_LIST, AVERAGE, MIN, MAX, SUM, COUNT_ROWS, COUNT_DISTINCT, HOURS_SINCE_MAX_DATE, HOURS_SINCE_MAX_TIMESTAMP, COUNT_TRUE, PERCENT_TRUE, COUNT_FALSE, PERCENT_FALSE, COUNT_USA_PHONE, PERCENT_USA_PHONE, COUNT_USA_ZIP_CODE, PERCENT_USA_ZIP_CODE, PERCENT_UUID, COUNT_TIMESTAMP_STRING, PERCENT_TIMESTAMP_STRING, COUNT_DUPLICATES, COUNT_USA_STATE_CODE, PERCENT_USA_STATE_CODE, VARIANCE, SKEW, KURTOSIS, GEOMETRIC_MEAN, HARMONIC_MEAN, COUNT_UUID, COUNT_CUSIP, PERCENT_CUSIP, COUNT_SEDOL, PERCENT_SEDOL, COUNT_ISIN, PERCENT_ISIN, COUNT_LEI, PERCENT_LEI, COUNT_FIGI, PERCENT_FIGI, COUNT_PERM_ID, PERCENT_PERM_ID, COUNT_NAN, PERCENT_NAN, COUNT_LONGITUDE, PERCENT_LONGITUDE, COUNT_LATITUDE, PERCENT_LATITUDE, COUNT_NOT_IN_FUTURE, PERCENT_NOT_IN_FUTURE, COUNT_DATE_NOT_IN_FUTURE, PERCENT_DATE_NOT_IN_FUTURE, MEDIAN, PERCENTILE, COUNT_NOT_NULL, STRING_LENGTH_AVERAGE, STRING_LENGTH_MIN, STRING_LENGTH_MAX, COUNT_SSN, PERCENT_SSN, COUNT_EMAIL, PERCENT_EMAIL, ROWS_INSERTED, HOURS_SINCE_LAST_LOAD, COUNT_READ_QUERIES, PERCENT_NOT_NULL, FRESHNESS, VOLUME, FRESHNESS_DATA, VOLUME_DATA
|
|
795
754
|
* `--help`: Show this message and exit.
|
|
796
755
|
|
|
797
756
|
## `bigeye catalog delete-source`
|
|
@@ -1114,32 +1073,6 @@ $ bigeye catalog set-metric-time [OPTIONS]
|
|
|
1114
1073
|
* `-r, --replace`: replace metric times if already present on tables? Default is false.
|
|
1115
1074
|
* `--help`: Show this message and exit.
|
|
1116
1075
|
|
|
1117
|
-
## `bigeye catalog track-schema-changes`
|
|
1118
|
-
|
|
1119
|
-
Track schema changes for sources on demand or every n hour(s) and notify specified channels.
|
|
1120
|
-
Environment variables SMTP_USER and SMTP_PASSWORD MUST be set if using email notification.
|
|
1121
|
-
|
|
1122
|
-
**Usage**:
|
|
1123
|
-
|
|
1124
|
-
```console
|
|
1125
|
-
$ bigeye catalog track-schema-changes [OPTIONS]
|
|
1126
|
-
```
|
|
1127
|
-
|
|
1128
|
-
**Options**:
|
|
1129
|
-
|
|
1130
|
-
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
1131
|
-
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
1132
|
-
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1133
|
-
* `-src, --source TEXT`: A list of source names to track. (None will track changes from all sources)
|
|
1134
|
-
* `-s, --since TEXT`: Track schema changes from this date. (Formatted as yyyy-MM-dd) [default: 1970-01-01]
|
|
1135
|
-
* `-f, --frequency INTEGER`: Every n hours to check for schema changes. (None will only run the check once) [default: 0]
|
|
1136
|
-
* `-ct, --change_type [CREATED|DELETED|TYPE_CHANGED]`: The types of changes to track; i.e additions, changes, deletions. (None will track all types) [default: DELETED, CREATED, TYPE_CHANGED]
|
|
1137
|
-
* `-nc, --channel TEXT`: A list of notification channels; e.g. -nc email@mydomain.com -nc #slack_channel -nc https://www.webhook.com
|
|
1138
|
-
* `-smtp, --server TEXT`: The server to send emails.
|
|
1139
|
-
* `-p, --port INTEGER`: The server port to send emails. [default: 587]
|
|
1140
|
-
* `-t, --token TEXT`: The token to utilize the Slack web client. (Will attempt to use environment variable SLACK_BOT_TOKEN, if not set)
|
|
1141
|
-
* `--help`: Show this message and exit.
|
|
1142
|
-
|
|
1143
1076
|
## `bigeye catalog unschedule-all-metrics`
|
|
1144
1077
|
|
|
1145
1078
|
Unschedule all metrics by warehouse, schema or tables.
|
|
@@ -1194,13 +1127,12 @@ $ bigeye catalog upsert-template [OPTIONS]
|
|
|
1194
1127
|
* `-b, --bigeye_conf TEXT`: Bigeye Credential File Path [env var: BIGEYE_API_CRED_FILE]
|
|
1195
1128
|
* `-c, --config_file TEXT`: Bigeye CLI Configuration File Path [env var: BIGEYE_API_CONFIG_FILE]
|
|
1196
1129
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1197
|
-
* `-sn, --source_name TEXT`: The name of the source where the template will be defined
|
|
1198
|
-
* `-tn, --name TEXT`: The user defined name of the template
|
|
1199
|
-
* `-tb, --body TEXT`: The string to define the template
|
|
1200
|
-
* `-rt, --returns [BOOLEAN|NUMERIC]`: The data type returned by the template; i.e. NUMERIC, BOOLEAN
|
|
1201
|
-
* `-p, --params TEXT`: A list of key/value pairs for parameters; ex. -p column=COLUMN_REFERENCE -p table=STRING
|
|
1130
|
+
* `-sn, --source_name TEXT`: The name of the source where the template will be defined [required]
|
|
1131
|
+
* `-tn, --name TEXT`: The user defined name of the template [required]
|
|
1132
|
+
* `-tb, --body TEXT`: The string to define the template [required]
|
|
1133
|
+
* `-rt, --returns [BOOLEAN|NUMERIC]`: The data type returned by the template; i.e. NUMERIC, BOOLEAN [required]
|
|
1134
|
+
* `-p, --params TEXT`: A list of key/value pairs for parameters; ex. -p column=COLUMN_REFERENCE -p table=STRING [required]
|
|
1202
1135
|
* `-id, --template_id INTEGER`: The template ID (Only required when updating a template). [default: 0]
|
|
1203
|
-
* `-tf, --template_file TEXT`: A file containing a template definition.
|
|
1204
1136
|
* `--help`: Show this message and exit.
|
|
1205
1137
|
|
|
1206
1138
|
## `bigeye catalog upsert-virtual-table`
|
|
@@ -1558,7 +1490,7 @@ $ bigeye deltas suggest-deltas [OPTIONS]
|
|
|
1558
1490
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1559
1491
|
* `-source, --source_selector TEXT`: The pattern of tables in the source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_1.*.* would be all schemas in source_1. [required]
|
|
1560
1492
|
* `-target, --target_selector TEXT`: The pattern of tables in the target to select. Wildcard (*) indicates all tables or all schemas, e.g. source_2.*.* would be all schemas in source_2. [required]
|
|
1561
|
-
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default:
|
|
1493
|
+
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default: MatchType.STRICT]
|
|
1562
1494
|
* `-op, --output_path TEXT`: File to write the delta configuration. [required]
|
|
1563
1495
|
* `--help`: Show this message and exit.
|
|
1564
1496
|
|
|
@@ -1825,7 +1757,6 @@ $ bigeye lineage infer-relations [OPTIONS]
|
|
|
1825
1757
|
* `-w, --workspace TEXT`: Name of the workspace configuration. [env var: BIGEYE_WORKSPACE; default: DEFAULT]
|
|
1826
1758
|
* `-upstream, --upstream_selector TEXT`: The pattern of tables in the upstream source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_1.*.* would be all schemas in source_1.
|
|
1827
1759
|
* `-downstream, --downstream_selector TEXT`: The pattern of tables in the downstream source to select. Wildcard (*) indicates all tables or all schemas, e.g. source_2.*.* would be all schemas in source_2.
|
|
1828
|
-
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default:
|
|
1760
|
+
* `-mt, --match_type [strict|fuzzy]`: How to match tables between the source and target destinations. Strict will only create relations if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match. [default: MatchType.STRICT]
|
|
1829
1761
|
* `--help`: Show this message and exit.
|
|
1830
1762
|
|
|
1831
|
-
|
|
@@ -147,6 +147,8 @@ def export(
|
|
|
147
147
|
name="Sample", description="SDK Generated"
|
|
148
148
|
)
|
|
149
149
|
|
|
150
|
+
final_metric_ids: List[int] = []
|
|
151
|
+
|
|
150
152
|
# Require at least 1 option to be specified
|
|
151
153
|
if not collection_id and not table_id and not metric_ids:
|
|
152
154
|
raise typer.BadParameter(
|
|
@@ -154,19 +156,23 @@ def export(
|
|
|
154
156
|
"collection id (-cid), table id (-tid), or list of metric ids (-mid). "
|
|
155
157
|
)
|
|
156
158
|
|
|
159
|
+
# Add metric ids if passed
|
|
160
|
+
if metric_ids:
|
|
161
|
+
final_metric_ids.extend(metric_ids)
|
|
162
|
+
|
|
157
163
|
# Get existing collection if passed
|
|
158
164
|
if collection_id:
|
|
159
165
|
collection: SimpleCollection = SimpleCollection.from_datawatch_object(
|
|
160
166
|
client.get_collection(collection_id=collection_id).collection
|
|
161
167
|
)
|
|
162
|
-
|
|
168
|
+
final_metric_ids.extend(collection.metric_ids)
|
|
163
169
|
|
|
164
170
|
# Get metrics from table if passed
|
|
165
171
|
if table_id:
|
|
166
172
|
metrics_from_table = client.get_metric_info_batch_post(
|
|
167
173
|
table_ids=[table_id]
|
|
168
174
|
).metrics
|
|
169
|
-
|
|
175
|
+
final_metric_ids.extend([m.metric_configuration.id for m in metrics_from_table])
|
|
170
176
|
if not metrics_from_table:
|
|
171
177
|
raise typer.BadParameter(
|
|
172
178
|
f"No metrics found for given table id. Please check arguments and try again."
|
|
@@ -175,7 +181,7 @@ def export(
|
|
|
175
181
|
collection.name = metrics_from_table[0].metric_metadata.dataset_name
|
|
176
182
|
|
|
177
183
|
# Pull metric information
|
|
178
|
-
metrics: MetricInfoList = client.get_metric_info_batch_post(metric_ids=
|
|
184
|
+
metrics: MetricInfoList = client.get_metric_info_batch_post(metric_ids=final_metric_ids)
|
|
179
185
|
|
|
180
186
|
if not output_path:
|
|
181
187
|
output_path = client.config.bigconfig_output_path or Path.cwd()
|
|
@@ -8,10 +8,10 @@ from bigeye_cli.functions import run_enum_menu
|
|
|
8
8
|
from bigeye_cli import global_options
|
|
9
9
|
|
|
10
10
|
from bigeye_sdk.authentication.enums import AuthConfType, BrowserType
|
|
11
|
-
from bigeye_sdk.authentication.api_authentication import BasicAPIAuth, BrowserAPIAuth, ApiAuth
|
|
11
|
+
from bigeye_sdk.authentication.api_authentication import BasicAPIAuth, BrowserAPIAuth, ApiAuth, APIKeyAuth
|
|
12
12
|
from bigeye_sdk.authentication.config import Config
|
|
13
13
|
from bigeye_sdk.client.datawatch_client import get_user_auth
|
|
14
|
-
from bigeye_sdk.generated.com.bigeye.models.generated import Workspace
|
|
14
|
+
from bigeye_sdk.generated.com.bigeye.models.generated import Workspace, WorkspaceIdNameAndRole
|
|
15
15
|
from bigeye_sdk.exceptions.exceptions import WorkspaceNotSetException
|
|
16
16
|
|
|
17
17
|
|
|
@@ -42,13 +42,7 @@ def configure(
|
|
|
42
42
|
if workspace == 'DEFAULT' or not use_default_credential:
|
|
43
43
|
auth_type = run_enum_menu(enum_clz=AuthConfType, default=AuthConfType.BASIC_AUTH)
|
|
44
44
|
|
|
45
|
-
if auth_type == AuthConfType.
|
|
46
|
-
base_url = Prompt.ask("Enter the Bigeye URL", default="https://app.bigeye.com")
|
|
47
|
-
username = Prompt.ask("Enter the username")
|
|
48
|
-
password = Prompt.ask("Enter the password", password=True)
|
|
49
|
-
cred = BasicAPIAuth(base_url=base_url,
|
|
50
|
-
user=username,password=password)
|
|
51
|
-
else:
|
|
45
|
+
if auth_type == AuthConfType.BROWSER_AUTH:
|
|
52
46
|
browser_type = run_enum_menu(enum_clz=BrowserType, default=BrowserType.CHROME)
|
|
53
47
|
|
|
54
48
|
browser_profile_user_name = None
|
|
@@ -60,13 +54,25 @@ def configure(
|
|
|
60
54
|
cred = BrowserAPIAuth(browser=browser_type, base_url=base_url,
|
|
61
55
|
browser_profile_user_name=browser_profile_user_name)
|
|
62
56
|
|
|
57
|
+
elif auth_type == AuthConfType.API_KEY_AUTH:
|
|
58
|
+
base_url = Prompt.ask("Enter the Bigeye URL", default="https://app.bigeye.com")
|
|
59
|
+
api_key = Prompt.ask("Enter the API Key", password=True)
|
|
60
|
+
cred = APIKeyAuth(base_url=base_url, api_key=api_key)
|
|
61
|
+
|
|
62
|
+
else:
|
|
63
|
+
base_url = Prompt.ask("Enter the Bigeye URL", default="https://app.bigeye.com")
|
|
64
|
+
username = Prompt.ask("Enter the username")
|
|
65
|
+
password = Prompt.ask("Enter the password", password=True)
|
|
66
|
+
cred = BasicAPIAuth(base_url=base_url,
|
|
67
|
+
user=username,password=password)
|
|
68
|
+
|
|
63
69
|
cred.save_as_file(auth_filename=bigeye_conf,cred=cred,workspace=workspace)
|
|
64
70
|
# If inheriting from default credential then load default credential
|
|
65
71
|
else:
|
|
66
72
|
cred = ApiAuth.load(auth_file=bigeye_conf,workspace='DEFAULT')
|
|
67
73
|
|
|
68
74
|
# Only prompt user for workspace ID if they have access to multiple workspaces
|
|
69
|
-
accessible_workspaces: List[
|
|
75
|
+
accessible_workspaces: List[WorkspaceIdNameAndRole] = get_user_auth(cred).workspaces
|
|
70
76
|
if len(accessible_workspaces) > 1:
|
|
71
77
|
print(Text(text='\n** WORKSPACE OPTIONS **',style='bold green'))
|
|
72
78
|
for wk in accessible_workspaces:
|
|
@@ -121,6 +127,7 @@ def set(
|
|
|
121
127
|
f"Try one of the following: {str.join(', ',config_settings)}"
|
|
122
128
|
)
|
|
123
129
|
|
|
130
|
+
|
|
124
131
|
@app.command()
|
|
125
132
|
def get(
|
|
126
133
|
bigeye_conf: str = global_options.bigeye_conf,
|
|
@@ -153,6 +160,7 @@ def get(
|
|
|
153
160
|
f"Try one of the following: {str.join(', ',config_settings)}"
|
|
154
161
|
)
|
|
155
162
|
|
|
163
|
+
|
|
156
164
|
@app.command()
|
|
157
165
|
def list(
|
|
158
166
|
bigeye_conf: str = global_options.bigeye_conf,
|
|
@@ -181,12 +189,14 @@ def list(
|
|
|
181
189
|
table.add_column("Credential Type", justify="right", style="green")
|
|
182
190
|
for key, value in config.items():
|
|
183
191
|
cred_type = None
|
|
184
|
-
if auth.get(section=key,option='browser',fallback=None):
|
|
192
|
+
if auth.get(section=key, option='browser', fallback=None):
|
|
185
193
|
cred_type = 'Browser'
|
|
186
|
-
elif auth.get(section=key,option='user',fallback=None):
|
|
194
|
+
elif auth.get(section=key, option='user', fallback=None):
|
|
187
195
|
cred_type = 'Basic'
|
|
196
|
+
elif auth.get(section=key, option='api_key', fallback=None):
|
|
197
|
+
cred_type = 'Api Key'
|
|
188
198
|
else:
|
|
189
199
|
cred_type = 'Default'
|
|
190
200
|
table.add_row(key,value.get('workspace_id'),cred_type)
|
|
191
201
|
|
|
192
|
-
print(table)
|
|
202
|
+
print(table)
|
|
@@ -14,7 +14,7 @@ from bigeye_cli.functions import cli_client_factory
|
|
|
14
14
|
from bigeye_cli import global_options
|
|
15
15
|
from bigeye_sdk.controller.delta_controller import DeltaController
|
|
16
16
|
from bigeye_sdk.controller.lineage_controller import LineageController
|
|
17
|
-
from bigeye_sdk.functions.delta_functions import infer_column_mappings
|
|
17
|
+
from bigeye_sdk.functions.delta_functions import infer_column_mappings, build_ccm
|
|
18
18
|
from bigeye_sdk.functions.table_functions import fully_qualified_table_to_elements
|
|
19
19
|
from bigeye_sdk.generated.com.bigeye.models.generated import (
|
|
20
20
|
Delta,
|
|
@@ -355,13 +355,32 @@ def cicd(
|
|
|
355
355
|
|
|
356
356
|
log.info("Looking for existing Delta with same name.")
|
|
357
357
|
client.delete_deltas_by_name(delta_names=[delta_cicd.delta_name])
|
|
358
|
+
if not delta_cicd.delta_column_mapping:
|
|
359
|
+
source_metrics_types = client.get_delta_applicable_metric_types(
|
|
360
|
+
table_id=source.id
|
|
361
|
+
).metric_types
|
|
362
|
+
target_metric_types = client.get_delta_applicable_metric_types(
|
|
363
|
+
table_id=target.id
|
|
364
|
+
).metric_types
|
|
365
|
+
column_mappings = infer_column_mappings(
|
|
366
|
+
source_metric_types=source_metrics_types,
|
|
367
|
+
target_metric_types=target_metric_types,
|
|
368
|
+
)
|
|
369
|
+
else:
|
|
370
|
+
column_mappings = [
|
|
371
|
+
build_ccm(scm=cm, source_table=source, target_table=target) for cm in delta_cicd.delta_column_mapping
|
|
372
|
+
]
|
|
373
|
+
|
|
374
|
+
|
|
358
375
|
comparisons = [
|
|
359
|
-
ComparisonTableConfiguration(name=delta_cicd.delta_name,
|
|
376
|
+
ComparisonTableConfiguration(name=delta_cicd.delta_name,
|
|
377
|
+
source_table_id=source.id,
|
|
378
|
+
target_table_id=target.id,
|
|
379
|
+
column_mappings=column_mappings)
|
|
360
380
|
]
|
|
361
381
|
|
|
362
382
|
delta = Delta(
|
|
363
383
|
name=delta_cicd.delta_name,
|
|
364
|
-
named_schedule=delta_cicd.cron_schedule.to_datawatch_object(),
|
|
365
384
|
source_table=IdAndDisplayName(id=source.id, display_name=source.name),
|
|
366
385
|
comparison_table_configurations=comparisons,
|
|
367
386
|
notification_channels=[
|
|
@@ -372,10 +391,11 @@ def cicd(
|
|
|
372
391
|
response = client.upsert_delta(delta=delta)
|
|
373
392
|
delta_run = client.run_a_delta(delta_id=response.delta.id, await_results=True)
|
|
374
393
|
|
|
375
|
-
delta_info = client.
|
|
394
|
+
delta_info = client.get_delta_info(delta_id=delta_run.id)
|
|
376
395
|
|
|
396
|
+
base_url = f"{client.get_base_url()}/w/{client.config.workspace_id}/deltas"
|
|
377
397
|
exit_code = vendor_report.publish(
|
|
378
|
-
|
|
398
|
+
base_url, delta_cicd.fq_source_table_name, delta_cicd.fq_target_table_name, delta_info
|
|
379
399
|
)
|
|
380
400
|
|
|
381
401
|
if exit_code != 0 and delta_cicd.group_bys:
|
|
@@ -384,20 +404,21 @@ def cicd(
|
|
|
384
404
|
group_by_name = f"{delta_cicd.delta_name} -- GROUPED BY {group_by_columns}"
|
|
385
405
|
client.delete_deltas_by_name(delta_names=[group_by_name])
|
|
386
406
|
group_bys = [gbc.to_datawatch_object() for gbc in delta_cicd.group_bys]
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
)
|
|
407
|
+
gb_compare = comparisons[0]
|
|
408
|
+
gb_compare.group_bys = group_bys
|
|
409
|
+
delta.name = group_by_name
|
|
410
|
+
delta.comparison_table_configurations = [gb_compare]
|
|
411
|
+
|
|
393
412
|
|
|
394
|
-
client.
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
413
|
+
gb_response = client.upsert_delta(delta=delta)
|
|
414
|
+
gb_delta_run = client.run_a_delta(delta_id=gb_response.delta.id, await_results=True)
|
|
415
|
+
delta_gb_run = client.get_delta_info(
|
|
416
|
+
delta_id=gb_delta_run.id
|
|
417
|
+
)
|
|
398
418
|
|
|
399
|
-
if delta_gb_run.alerting_metric_count != 0:
|
|
419
|
+
if delta_gb_run.delta.alerting_metric_count != 0 or delta_gb_run.delta.failed_metric_count != 0:
|
|
400
420
|
vendor_report.publish_group_bys(
|
|
421
|
+
base_url,
|
|
401
422
|
delta_cicd.fq_source_table_name,
|
|
402
423
|
delta_cicd.fq_target_table_name,
|
|
403
424
|
delta_gb_run,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Optional
|
|
1
|
+
from typing import Optional, List
|
|
2
2
|
|
|
3
3
|
import typer
|
|
4
4
|
|
|
@@ -9,7 +9,8 @@ from bigeye_cli import global_options
|
|
|
9
9
|
from bigeye_sdk.controller.lineage_controller import LineageController
|
|
10
10
|
from bigeye_sdk.model.enums import MatchType
|
|
11
11
|
from bigeye_sdk.log import get_logger
|
|
12
|
-
|
|
12
|
+
from bigeye_sdk.model.lineage_facade import LineageConfigurationFile, SimpleLineageConfigurationFile, \
|
|
13
|
+
LineageColumnOverride
|
|
13
14
|
|
|
14
15
|
log = get_logger(__file__)
|
|
15
16
|
|
|
@@ -85,14 +86,34 @@ def create_relation(
|
|
|
85
86
|
, "--downstream"
|
|
86
87
|
, "-down"
|
|
87
88
|
, help="The fully qualified table name"
|
|
89
|
+
),
|
|
90
|
+
column_overrides: List[str] = typer.Option(
|
|
91
|
+
None
|
|
92
|
+
, "--column_overrides"
|
|
93
|
+
, "-co"
|
|
94
|
+
, help="The list of column overrides, formatted as upstream_column_name:downstream_column_name"
|
|
88
95
|
)
|
|
89
96
|
):
|
|
90
97
|
"""Create a lineage relationship for 2 entities"""
|
|
91
98
|
client = cli_client_factory(bigeye_conf, config_file, workspace)
|
|
92
99
|
lineage_controller = LineageController(client)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
100
|
+
|
|
101
|
+
c_overrides: List[LineageColumnOverride] = []
|
|
102
|
+
if column_overrides is not None:
|
|
103
|
+
for co in column_overrides:
|
|
104
|
+
split_by_colon = co.split(":")
|
|
105
|
+
c_overrides.append(
|
|
106
|
+
LineageColumnOverride(
|
|
107
|
+
upstream_column_name=split_by_colon[0],
|
|
108
|
+
downstream_column_name=split_by_colon[1]
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
lineage_controller.create_edges_from_table_names(
|
|
113
|
+
upstream_table_name=upstream_table_name,
|
|
114
|
+
downstream_table_name=downstream_table_name,
|
|
115
|
+
column_overrides=c_overrides
|
|
116
|
+
)
|
|
96
117
|
|
|
97
118
|
|
|
98
119
|
@app.command()
|
|
@@ -157,17 +178,40 @@ def infer_relations(
|
|
|
157
178
|
, help="How to match tables between the source and target destinations. Strict will only create relations "
|
|
158
179
|
"if table names match exactly, Fuzzy will attempt to create relations using a fuzzy match."
|
|
159
180
|
, case_sensitive=False
|
|
181
|
+
),
|
|
182
|
+
lineage_configuration_file: str = typer.Option(
|
|
183
|
+
None,
|
|
184
|
+
"--lineage_conf",
|
|
185
|
+
"-lc",
|
|
186
|
+
help="The path to a Simple Lineage configuration file."
|
|
187
|
+
),
|
|
188
|
+
purge_lineage: bool = typer.Option(
|
|
189
|
+
False,
|
|
190
|
+
"--purge_lineage",
|
|
191
|
+
"-purge",
|
|
192
|
+
help="Purge all lineage edges."
|
|
160
193
|
)
|
|
161
194
|
):
|
|
162
195
|
"""Given an upstream and downstream path, infers lineage based on table/column names and creates the edges."""
|
|
163
196
|
client = cli_client_factory(bigeye_conf, config_file, workspace)
|
|
164
197
|
lineage_controller = LineageController(client)
|
|
165
198
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
199
|
+
if lineage_configuration_file:
|
|
200
|
+
lc = SimpleLineageConfigurationFile.load(lineage_configuration_file)
|
|
201
|
+
lineage_controller.infer_column_level_lineage_from_file(
|
|
202
|
+
lineage_configuration_file=lc,
|
|
203
|
+
purge_lineage=purge_lineage
|
|
204
|
+
)
|
|
170
205
|
|
|
171
|
-
|
|
206
|
+
else:
|
|
207
|
+
# Get matching tables
|
|
208
|
+
matching_tables = lineage_controller.get_matching_tables_from_selectors(upstream_selector=upstream_selector,
|
|
209
|
+
downstream_selector=downstream_selector,
|
|
210
|
+
match_type=match_type)
|
|
172
211
|
|
|
173
|
-
|
|
212
|
+
log.info(f'Identified {len(matching_tables)} table relationships between upstream and downstream sources.')
|
|
213
|
+
|
|
214
|
+
lineage_controller.infer_column_level_lineage_from_tables(
|
|
215
|
+
tables=matching_tables,
|
|
216
|
+
purge_lineage=purge_lineage
|
|
217
|
+
)
|
|
@@ -7,7 +7,7 @@ from github.PullRequest import PullRequest
|
|
|
7
7
|
from github.Repository import Repository
|
|
8
8
|
|
|
9
9
|
from bigeye_sdk.log import get_logger
|
|
10
|
-
from bigeye_sdk.generated.com.bigeye.models.generated import
|
|
10
|
+
from bigeye_sdk.generated.com.bigeye.models.generated import DeltaInfo
|
|
11
11
|
from bigeye_cli.model.vendor_report import (
|
|
12
12
|
VendorReport,
|
|
13
13
|
_format_report,
|
|
@@ -31,40 +31,44 @@ class GitHubReport(VendorReport):
|
|
|
31
31
|
self.pr = self.repo.get_pull(int(os.environ["PR_NUMBER"]))
|
|
32
32
|
|
|
33
33
|
def publish(
|
|
34
|
-
self, source_table_name: str, target_table_name: str,
|
|
34
|
+
self, base_url: str, source_table_name: str, target_table_name: str, di: DeltaInfo
|
|
35
35
|
) -> int:
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
dti = di.comparison_target_infos[0]
|
|
38
|
+
url = f"{base_url}/{di.delta.id}"
|
|
39
|
+
if dti.alerting_metric_count != 0 or dti.failed_metric_count != 0:
|
|
39
40
|
log.info(
|
|
40
41
|
"Delta has alerting metrics. A grouped delta will run, if columns have been specified."
|
|
41
42
|
)
|
|
42
43
|
log.error("The process will exit as failed.")
|
|
43
|
-
report = _format_report(source_table_name, target_table_name,
|
|
44
|
-
|
|
45
|
-
self.issue = self.repo.create_issue(
|
|
46
|
-
|
|
47
|
-
)
|
|
44
|
+
report = _format_report(url, source_table_name, target_table_name, dti)
|
|
45
|
+
body = f"Bigeye Delta failure for PR: {self.pr.title}\n\n{report}"
|
|
46
|
+
# self.issue = self.repo.create_issue(
|
|
47
|
+
# title=failure_title, body=report, assignee=self.pr.user, labels=["bug"]
|
|
48
|
+
# )
|
|
49
|
+
self.pr.create_issue_comment(body=body)
|
|
48
50
|
return 1
|
|
49
|
-
elif
|
|
50
|
-
question_title = f"Schema mismatch detected in delta. Is this expected?"
|
|
51
|
+
elif dti.schema_match is False:
|
|
52
|
+
question_title = f"## Schema mismatch detected in delta. Is this expected?"
|
|
51
53
|
header = "#### Please confirm that the schemas are not supposed to match before approving Pull Request."
|
|
52
54
|
tables = (
|
|
53
55
|
f"Source Table: {source_table_name}\nTarget Table: {target_table_name}"
|
|
54
56
|
)
|
|
55
|
-
body = f"{header}\n{tables}"
|
|
57
|
+
body = f"{question_title}\n{header}\n{tables}"
|
|
56
58
|
|
|
57
|
-
self.
|
|
59
|
+
self.pr.create_issue_comment(body=body)
|
|
58
60
|
return 0
|
|
59
61
|
else:
|
|
60
62
|
return 0
|
|
61
63
|
|
|
62
64
|
def publish_group_bys(
|
|
63
|
-
self, source_table_name: str, target_table_name: str,
|
|
65
|
+
self, base_url: str, source_table_name: str, target_table_name: str, di: DeltaInfo
|
|
64
66
|
):
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
dti = di.comparison_target_infos[0]
|
|
68
|
+
group_bys = di.delta.comparison_table_configurations[0].group_bys
|
|
69
|
+
url = f"{base_url}/{di.delta.id}"
|
|
70
|
+
report = _format_group_report(url, source_table_name, target_table_name, dti, group_bys)
|
|
71
|
+
self.pr.create_issue_comment(body=report)
|
|
68
72
|
|
|
69
73
|
def publish_bigconfig_plan(self, plan: str):
|
|
70
74
|
self.pr.create_issue_comment(body=plan)
|
|
@@ -2,9 +2,7 @@ import abc
|
|
|
2
2
|
from typing import List
|
|
3
3
|
|
|
4
4
|
from bigeye_sdk.generated.com.bigeye.models.generated import ComparisonMetricInfo, ComparisonMetricStatus, \
|
|
5
|
-
PredefinedMetricName,
|
|
6
|
-
|
|
7
|
-
deltas_url = "https://app.bigeye.com/deltas"
|
|
5
|
+
PredefinedMetricName, DeltaTargetInfo, ColumnNamePair, Delta, DeltaInfo
|
|
8
6
|
|
|
9
7
|
# TODO: This can all be dynamically generated. If we want to allow users to provide the desired fields.
|
|
10
8
|
__delta_header = """| metric_name | source_column | target_column | source_value | target_value | difference |"""
|
|
@@ -52,12 +50,12 @@ def __format_delta_table(cmis: List[ComparisonMetricInfo]) -> str:
|
|
|
52
50
|
return __collapsed_section.format(details=table)
|
|
53
51
|
|
|
54
52
|
|
|
55
|
-
def _format_report(source_table: str, target_table: str,
|
|
56
|
-
link = f"[View Full Delta in Bigeye]({deltas_url}
|
|
57
|
-
schema_match = f"#### Source and Target schema match is {
|
|
53
|
+
def _format_report(deltas_url: str, source_table: str, target_table: str, dti: DeltaTargetInfo):
|
|
54
|
+
link = f"[View Full Delta in Bigeye]({deltas_url})"
|
|
55
|
+
schema_match = f"#### Source and Target schema match is {dti.schema_match}"
|
|
58
56
|
table_names = f"Source Table: {source_table}\nTarget Table: {target_table}"
|
|
59
|
-
failed_metric_ratio = f"{
|
|
60
|
-
report = f"{table_names}\n{__format_delta_table(
|
|
57
|
+
failed_metric_ratio = f"{dti.failed_metric_count} of {dti.metric_count} metrics have failed"
|
|
58
|
+
report = f"{table_names}\n{__format_delta_table(dti.comparison_metric_infos)}\n{failed_metric_ratio}"
|
|
61
59
|
|
|
62
60
|
return f"{report}\n{schema_match}\n{link}"
|
|
63
61
|
|
|
@@ -82,14 +80,14 @@ def __format_group_table(cmis: List[ComparisonMetricInfo]):
|
|
|
82
80
|
return __collapsed_section.format(details=table)
|
|
83
81
|
|
|
84
82
|
|
|
85
|
-
def _format_group_report(source_table: str, target_table: str,
|
|
83
|
+
def _format_group_report(deltas_url: str, source_table: str, target_table: str, dti: DeltaTargetInfo, group_bys: List[ColumnNamePair]):
|
|
86
84
|
title = "### Grouped Delta Summary"
|
|
87
85
|
table_names = f"Source Table: {source_table}\nTarget Table: {target_table}"
|
|
88
86
|
group_by_columns = "Grouped by: " + ", ".join(gb.source_column_name
|
|
89
|
-
for gb in
|
|
90
|
-
table = "\n".join([__format_group_table(
|
|
91
|
-
failed_metric_ratio = f"{
|
|
92
|
-
link = f"[View Full Delta in Bigeye]({deltas_url}
|
|
87
|
+
for gb in group_bys)
|
|
88
|
+
table = "\n".join([__format_group_table(dti.comparison_metric_infos)])
|
|
89
|
+
failed_metric_ratio = f"{dti.failed_metric_count} of {dti.metric_count} metrics have failed"
|
|
90
|
+
link = f"[View Full Delta in Bigeye]({deltas_url})"
|
|
93
91
|
|
|
94
92
|
return f"{title}\n{table_names}\n\n{group_by_columns}\n{table}\n{failed_metric_ratio}\n{link}"
|
|
95
93
|
|
|
@@ -97,11 +95,11 @@ def _format_group_report(source_table: str, target_table: str, cti: ComparisonTa
|
|
|
97
95
|
class VendorReport(abc.ABC):
|
|
98
96
|
|
|
99
97
|
@abc.abstractmethod
|
|
100
|
-
def publish(self, source_table_name: str, target_table_name: str,
|
|
98
|
+
def publish(self, base_url: str, source_table_name: str, target_table_name: str, di: DeltaInfo):
|
|
101
99
|
pass
|
|
102
100
|
|
|
103
101
|
@abc.abstractmethod
|
|
104
|
-
def publish_group_bys(self, source_table_name: str, target_table_name: str,
|
|
102
|
+
def publish_group_bys(self, base_url: str, source_table_name: str, target_table_name: str, di: DeltaInfo):
|
|
105
103
|
pass
|
|
106
104
|
|
|
107
105
|
@abc.abstractmethod
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "bigeye-cli"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.63"
|
|
4
4
|
description = "Bigeye CLI offers developer tools for maintaining your developer workspace."
|
|
5
5
|
license = "Proprietary"
|
|
6
6
|
authors = ["Bigeye <support@bigeye.com>"]
|
|
@@ -19,7 +19,7 @@ bigeye = "bigeye_cli.__main__:app"
|
|
|
19
19
|
[tool.poetry.dependencies]
|
|
20
20
|
python = "^3.9.0"
|
|
21
21
|
click = "^8.0.4"
|
|
22
|
-
bigeye-sdk = "^0.4.
|
|
22
|
+
bigeye-sdk = "^0.4.84"
|
|
23
23
|
rich = "^12.4.4"
|
|
24
24
|
PyGithub = '^1.57'
|
|
25
25
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|