validio-cli 0.13.1__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {validio_cli-0.13.1 → validio_cli-0.15.0}/PKG-INFO +2 -2
- {validio_cli-0.13.1 → validio_cli-0.15.0}/pyproject.toml +11 -15
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/__init__.py +1 -1
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/config.py +4 -1
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/dbt.py +4 -0
- validio_cli-0.15.0/validio_cli/bin/entities/incidents.py +155 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/notification_rules.py +0 -9
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/resources.py +3 -3
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/sources.py +159 -137
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/main.py +7 -5
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/components.py +0 -1
- validio_cli-0.13.1/validio_cli/bin/entities/incidents.py +0 -254
- {validio_cli-0.13.1 → validio_cli-0.15.0}/LICENSE +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/README_PUBLIC.md +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/channels.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/code.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/credentials.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/metrics.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/recommendations.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/segmentations.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/segments.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/users.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/validators.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/bin/entities/windows.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/metadata.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/namespace.py +0 -0
- {validio_cli-0.13.1 → validio_cli-0.15.0}/validio_cli/schema.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: validio-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: CLI tool to interact with the Validio platform
|
|
5
5
|
Home-page: https://validio.io/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
14
|
Requires-Dist: camel-converter (>=3.0.0,<4.0.0)
|
|
15
|
-
Requires-Dist: classdiff (
|
|
15
|
+
Requires-Dist: classdiff (==0.4.0)
|
|
16
16
|
Requires-Dist: prompt-toolkit (>=3.0.38,<4.0.0)
|
|
17
17
|
Requires-Dist: pydantic (==2.4.2)
|
|
18
18
|
Requires-Dist: tabulate (>=0.9.0,<0.10.0)
|
|
@@ -3,29 +3,28 @@ name = "validio-cli"
|
|
|
3
3
|
# This version does not represent the released version or any tag. For each
|
|
4
4
|
# release we automatically bump this before building and publishing so this
|
|
5
5
|
# should be kept at 0.0.1dev1
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.15.0"
|
|
7
7
|
description = "CLI tool to interact with the Validio platform"
|
|
8
8
|
authors = ["Validio <support@validio.io>"]
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
homepage = "https://validio.io/"
|
|
11
11
|
documentation = "https://docs.validio.io/"
|
|
12
|
-
packages = [{include = "validio_cli"}]
|
|
12
|
+
packages = [{ include = "validio_cli" }]
|
|
13
13
|
readme = "README_PUBLIC.md"
|
|
14
14
|
|
|
15
15
|
[tool.poetry.dependencies]
|
|
16
16
|
python = "^3.10"
|
|
17
17
|
camel-converter = "^3.0.0"
|
|
18
|
-
classdiff = "
|
|
18
|
+
classdiff = "0.4.0"
|
|
19
19
|
validio-sdk = "*"
|
|
20
|
-
typer = {extras = ["all"], version = "^0.7.0"}
|
|
20
|
+
typer = { extras = ["all"], version = "^0.7.0" }
|
|
21
21
|
prompt-toolkit = "^3.0.38"
|
|
22
22
|
tabulate = "^0.9.0"
|
|
23
23
|
types-tabulate = "^0.9.0.2"
|
|
24
24
|
pydantic = "2.4.2"
|
|
25
25
|
|
|
26
26
|
[tool.poetry.group.dev.dependencies]
|
|
27
|
-
ruff = "^0.
|
|
28
|
-
black = "^23.3.0"
|
|
27
|
+
ruff = "^0.1.11"
|
|
29
28
|
mypy = "^1.2.0"
|
|
30
29
|
validio-sdk = { path = "../validio-sdk", develop = true }
|
|
31
30
|
licensecheck = "^2023.1.1"
|
|
@@ -61,19 +60,16 @@ select = [
|
|
|
61
60
|
]
|
|
62
61
|
|
|
63
62
|
ignore = [
|
|
64
|
-
"D203",
|
|
65
|
-
"D212",
|
|
66
|
-
"PLR0913" # `too-many-arguments` - Not relevant with only 6
|
|
63
|
+
"D203", # `one-blank-line-before-class` Incompatible with D211
|
|
64
|
+
"D212", # `multi-line-summary-first-line` Incompatible with D213
|
|
65
|
+
"PLR0913", # `too-many-arguments` - Not relevant with only 6
|
|
67
66
|
]
|
|
68
67
|
|
|
69
68
|
[tool.ruff.per-file-ignores]
|
|
70
69
|
# No need for documentation in these files
|
|
71
|
-
"__init__.py" = [
|
|
72
|
-
"*/bin/*.py" = [
|
|
73
|
-
"examples/*.py" = [
|
|
74
|
-
|
|
75
|
-
[tool.black]
|
|
76
|
-
preview = true
|
|
70
|
+
"__init__.py" = ["D"]
|
|
71
|
+
"*/bin/*.py" = ["D"]
|
|
72
|
+
"examples/*.py" = ["D"]
|
|
77
73
|
|
|
78
74
|
[tool.mypy]
|
|
79
75
|
ignore_missing_imports = true
|
|
@@ -231,7 +231,7 @@ def _resource_filter(
|
|
|
231
231
|
if resource is None:
|
|
232
232
|
return False
|
|
233
233
|
|
|
234
|
-
return resource.id
|
|
234
|
+
return value in (resource.id, resource.resource_name)
|
|
235
235
|
|
|
236
236
|
|
|
237
237
|
def _fixed_width(message: str, width: int = 25) -> str:
|
|
@@ -77,6 +77,7 @@ def ns(
|
|
|
77
77
|
def get(
|
|
78
78
|
config_dir: str = ConfigDir,
|
|
79
79
|
output_format: OutputFormat = OutputFormatOption,
|
|
80
|
+
show_secrets: bool = False,
|
|
80
81
|
) -> None:
|
|
81
82
|
config_path = Path(config_dir) if config_dir is not None else None
|
|
82
83
|
|
|
@@ -109,7 +110,9 @@ def get(
|
|
|
109
110
|
"default_namespace": validio_config.default_namespace,
|
|
110
111
|
"endpoint": validio_config.endpoint,
|
|
111
112
|
"access_key": validio_config.access_key,
|
|
112
|
-
"access_secret": validio_config.
|
|
113
|
+
"access_secret": validio_config._access_secret
|
|
114
|
+
if show_secrets
|
|
115
|
+
else validio_config.access_secret,
|
|
113
116
|
}
|
|
114
117
|
|
|
115
118
|
if output_format == OutputFormat.JSON:
|
|
@@ -19,6 +19,9 @@ async def upload(
|
|
|
19
19
|
namespace: str = Namespace(),
|
|
20
20
|
credential_id: str = typer.Option(..., help="Credential name or ID"),
|
|
21
21
|
manifest: Path = typer.Option(..., help="Path to the manifest file"),
|
|
22
|
+
job_name: str = typer.Option(
|
|
23
|
+
..., help="The job that the dbt execution belongs to, e.g. `staging-pipeline`"
|
|
24
|
+
),
|
|
22
25
|
run_results: Path = typer.Option(help="Path to the run results file", default=None),
|
|
23
26
|
) -> None:
|
|
24
27
|
vc, cfg = await get_client_and_config(config_dir)
|
|
@@ -46,6 +49,7 @@ async def upload(
|
|
|
46
49
|
credential_id=resolved_credential_id,
|
|
47
50
|
manifest=json.dumps(trimmed_manifest),
|
|
48
51
|
run_results=run_results_content,
|
|
52
|
+
job_name=job_name,
|
|
49
53
|
)
|
|
50
54
|
)
|
|
51
55
|
return print("dbt artifact uploaded successfully")
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
from datetime import datetime, timedelta
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import typer
|
|
5
|
+
from validio_sdk import ValidioError
|
|
6
|
+
from validio_sdk.graphql_client import (
|
|
7
|
+
GetSourceIncidentsSource,
|
|
8
|
+
GetValidatorIncidentsValidator,
|
|
9
|
+
)
|
|
10
|
+
from validio_sdk.graphql_client.input_types import TimeRangeInput
|
|
11
|
+
|
|
12
|
+
from validio_cli import (
|
|
13
|
+
AsyncTyper,
|
|
14
|
+
ConfigDir,
|
|
15
|
+
Namespace,
|
|
16
|
+
OutputFormat,
|
|
17
|
+
OutputFormatOption,
|
|
18
|
+
OutputSettings,
|
|
19
|
+
_format_relative_time,
|
|
20
|
+
get_client_and_config,
|
|
21
|
+
output_json,
|
|
22
|
+
output_text,
|
|
23
|
+
)
|
|
24
|
+
from validio_cli.bin.entities import sources, validators
|
|
25
|
+
|
|
26
|
+
app = AsyncTyper(help="Incidents from validators")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@app.async_command(
|
|
30
|
+
help="""List all incidents.
|
|
31
|
+
|
|
32
|
+
By default you will get incidents from the last hour. You can specify a time
|
|
33
|
+
range for when the incident occurred by specifying when the incident ended.
|
|
34
|
+
|
|
35
|
+
You can list incidents in different ways:
|
|
36
|
+
|
|
37
|
+
* Listing all incidents
|
|
38
|
+
|
|
39
|
+
* Listing all source incidents for a specific source with --source
|
|
40
|
+
|
|
41
|
+
* Listing all incidents for a specific validator with --validator
|
|
42
|
+
|
|
43
|
+
* Listing all incidents for a specific segment with --segment
|
|
44
|
+
|
|
45
|
+
* Listing all incidents for a specific validator and segment with --validator
|
|
46
|
+
and --segment together
|
|
47
|
+
"""
|
|
48
|
+
)
|
|
49
|
+
async def get(
|
|
50
|
+
config_dir: str = ConfigDir,
|
|
51
|
+
output_format: OutputFormat = OutputFormatOption,
|
|
52
|
+
namespace: str = Namespace(),
|
|
53
|
+
ended_before: datetime = typer.Option(
|
|
54
|
+
datetime.utcnow(),
|
|
55
|
+
help="The incident ended before this timestamp",
|
|
56
|
+
),
|
|
57
|
+
ended_after: datetime = typer.Option(
|
|
58
|
+
(datetime.utcnow() - timedelta(hours=1)),
|
|
59
|
+
help="The incident ended after this timestamp",
|
|
60
|
+
),
|
|
61
|
+
validator: str = typer.Option(None, help="Validator to fetch incidents for"),
|
|
62
|
+
segment: str = typer.Option(None, help="Segment to fetch incidents for"),
|
|
63
|
+
source: str = typer.Option(None, help="Source to fetch incidents for"),
|
|
64
|
+
) -> None:
|
|
65
|
+
vc, cfg = await get_client_and_config(config_dir)
|
|
66
|
+
|
|
67
|
+
if validator is not None:
|
|
68
|
+
validator_id = await validators.get_validator_id(vc, cfg, validator, namespace)
|
|
69
|
+
if validator_id is None:
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
if source is not None:
|
|
73
|
+
source_id = await sources.get_source_id(vc, cfg, source, namespace)
|
|
74
|
+
if source_id is None:
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
# TODO(UI-2006): These should all support namespace
|
|
78
|
+
|
|
79
|
+
incidents: (None | GetValidatorIncidentsValidator | GetSourceIncidentsSource) = None
|
|
80
|
+
|
|
81
|
+
if validator:
|
|
82
|
+
incidents = await vc.get_validator_incidents(
|
|
83
|
+
id=validator_id,
|
|
84
|
+
range=TimeRangeInput(
|
|
85
|
+
start=ended_after,
|
|
86
|
+
end=ended_before,
|
|
87
|
+
),
|
|
88
|
+
segment_id=segment,
|
|
89
|
+
)
|
|
90
|
+
elif source:
|
|
91
|
+
incidents = await vc.get_source_incidents(
|
|
92
|
+
id=source_id,
|
|
93
|
+
range=TimeRangeInput(
|
|
94
|
+
start=ended_after,
|
|
95
|
+
end=ended_before,
|
|
96
|
+
),
|
|
97
|
+
)
|
|
98
|
+
else:
|
|
99
|
+
raise ValidioError(
|
|
100
|
+
"You need to specify one of --validator and/or --segment or --source"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
if not incidents:
|
|
104
|
+
return output_text(None, {})
|
|
105
|
+
|
|
106
|
+
if output_format == OutputFormat.JSON:
|
|
107
|
+
return output_json(incidents)
|
|
108
|
+
|
|
109
|
+
return output_text(
|
|
110
|
+
incidents.incidents,
|
|
111
|
+
fields={
|
|
112
|
+
"operator": OutputSettings(
|
|
113
|
+
attribute_name="metric",
|
|
114
|
+
reformat=calculate_operator,
|
|
115
|
+
),
|
|
116
|
+
"bound": OutputSettings(
|
|
117
|
+
attribute_name="metric",
|
|
118
|
+
reformat=calculate_bound,
|
|
119
|
+
),
|
|
120
|
+
"value": OutputSettings(
|
|
121
|
+
attribute_name="metric",
|
|
122
|
+
reformat=lambda x: x.value,
|
|
123
|
+
),
|
|
124
|
+
"age": OutputSettings(
|
|
125
|
+
attribute_name="metric",
|
|
126
|
+
reformat=lambda x: _format_relative_time(x.end_time),
|
|
127
|
+
),
|
|
128
|
+
},
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def calculate_operator(item: Any) -> str:
|
|
133
|
+
type_ = item.typename__[len("ValidatorMetricWith") :]
|
|
134
|
+
if type_ == "DynamicThreshold":
|
|
135
|
+
operator = item.decision_bounds_type
|
|
136
|
+
else:
|
|
137
|
+
operator = item.operator
|
|
138
|
+
|
|
139
|
+
return f"{type_}/{operator}"
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def calculate_bound(item: Any) -> str:
|
|
143
|
+
type_ = item.typename__[len("ValidatorMetricWith") :]
|
|
144
|
+
if type_ == "DynamicThreshold":
|
|
145
|
+
bound = f"{item.lower_bound:.2f} - {item.upper_bound:.2f}"
|
|
146
|
+
elif type_ == "FixedThreshold":
|
|
147
|
+
bound = item.bound
|
|
148
|
+
else:
|
|
149
|
+
bound = "-"
|
|
150
|
+
|
|
151
|
+
return bound
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
typer.run(app())
|
|
@@ -28,9 +28,6 @@ async def get(
|
|
|
28
28
|
output_format: OutputFormat = OutputFormatOption,
|
|
29
29
|
namespace: str = Namespace(),
|
|
30
30
|
identifier: str = Identifier,
|
|
31
|
-
source: str = typer.Option(
|
|
32
|
-
None, help="List notification rules for this source (ID)"
|
|
33
|
-
),
|
|
34
31
|
) -> None:
|
|
35
32
|
vc, cfg = await get_client_and_config(config_dir)
|
|
36
33
|
|
|
@@ -46,12 +43,6 @@ async def get(
|
|
|
46
43
|
filter=ResourceFilter(resource_namespace=get_namespace(namespace, cfg))
|
|
47
44
|
)
|
|
48
45
|
|
|
49
|
-
notification_rules = [
|
|
50
|
-
nr
|
|
51
|
-
for nr in notification_rules
|
|
52
|
-
if source is None or (nr is not None and source in nr.sources)
|
|
53
|
-
]
|
|
54
|
-
|
|
55
46
|
# TODO(UI-2311): Fully support list/get/get_by_resource_name
|
|
56
47
|
notification_rules = _single_resource_if_specified(notification_rules, identifier)
|
|
57
48
|
|
|
@@ -105,14 +105,14 @@ def _read_resource_file(file: pathlib.Path) -> dict:
|
|
|
105
105
|
raise ValidioError(f"'{file.absolute()}' is not a file")
|
|
106
106
|
|
|
107
107
|
obj = json.loads(file.read_text())
|
|
108
|
-
if
|
|
108
|
+
if not isinstance(obj, dict):
|
|
109
109
|
raise ValidioError(f"expected json object in resource file: got {type(obj)}")
|
|
110
110
|
|
|
111
111
|
if "resources" not in obj:
|
|
112
112
|
raise ValidioError("json object in file should contain a 'resources' key")
|
|
113
113
|
|
|
114
114
|
resources_obj_raw = obj["resources"]
|
|
115
|
-
if
|
|
115
|
+
if not isinstance(resources_obj_raw, dict):
|
|
116
116
|
raise ValidioError(
|
|
117
117
|
"'resources' key should contain a json object: got"
|
|
118
118
|
f" {type(resources_obj_raw)}"
|
|
@@ -123,7 +123,7 @@ def _read_resource_file(file: pathlib.Path) -> dict:
|
|
|
123
123
|
if resource_type not in resources_obj_raw:
|
|
124
124
|
continue
|
|
125
125
|
resource_names = resources_obj_raw[resource_type]
|
|
126
|
-
if
|
|
126
|
+
if not isinstance(resource_names, list):
|
|
127
127
|
raise ValidioError(
|
|
128
128
|
f"'{resource_type}' should contain a list of resource names: got"
|
|
129
129
|
f" {resource_names}"
|
|
@@ -291,11 +291,13 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
291
291
|
|
|
292
292
|
match inference_type:
|
|
293
293
|
case "databricks":
|
|
294
|
-
info = await _multip_prompt(
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
294
|
+
info = await _multip_prompt(
|
|
295
|
+
[
|
|
296
|
+
("Catalog", [], ""),
|
|
297
|
+
("Schema", [], ""),
|
|
298
|
+
("Table", [], ""),
|
|
299
|
+
]
|
|
300
|
+
)
|
|
299
301
|
await _infer_schema_databricks(
|
|
300
302
|
vc,
|
|
301
303
|
filename,
|
|
@@ -307,47 +309,49 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
307
309
|
case "demo":
|
|
308
310
|
await _infer_schema_demo(vc, filename)
|
|
309
311
|
case "kinesis":
|
|
310
|
-
info = await _multip_prompt(
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
312
|
+
info = await _multip_prompt(
|
|
313
|
+
[
|
|
314
|
+
(
|
|
315
|
+
"Region",
|
|
316
|
+
# https://docs.aws.amazon.com/general/latest/gr/ak.html
|
|
317
|
+
[
|
|
318
|
+
"us-east-2",
|
|
319
|
+
"us-east-1",
|
|
320
|
+
"us-west-1",
|
|
321
|
+
"us-west-2",
|
|
322
|
+
"af-south-1",
|
|
323
|
+
"ap-east-1",
|
|
324
|
+
"ap-south-2",
|
|
325
|
+
"ap-southeast-3",
|
|
326
|
+
"ap-southeast-4",
|
|
327
|
+
"ap-south-1",
|
|
328
|
+
"ap-northeast-3",
|
|
329
|
+
"ap-northeast-2",
|
|
330
|
+
"ap-southeast-1",
|
|
331
|
+
"ap-southeast-2",
|
|
332
|
+
"ap-northeast-1",
|
|
333
|
+
"ca-central-1",
|
|
334
|
+
"eu-central-1",
|
|
335
|
+
"eu-west-1",
|
|
336
|
+
"eu-west-2",
|
|
337
|
+
"eu-south-1",
|
|
338
|
+
"eu-west-3",
|
|
339
|
+
"eu-south-2",
|
|
340
|
+
"eu-north-1",
|
|
341
|
+
"eu-central-2",
|
|
342
|
+
"me-south-1",
|
|
343
|
+
"me-central-1",
|
|
344
|
+
"sa-east-1",
|
|
345
|
+
"us-gov-east-1",
|
|
346
|
+
"us-gov-west-1",
|
|
347
|
+
],
|
|
348
|
+
"",
|
|
349
|
+
),
|
|
350
|
+
("Stream name", [], ""),
|
|
351
|
+
_interactive_message_format_input(),
|
|
352
|
+
("Message schema", [], ""),
|
|
353
|
+
]
|
|
354
|
+
)
|
|
351
355
|
|
|
352
356
|
await _infer_schema_kinesis(
|
|
353
357
|
vc,
|
|
@@ -359,12 +363,14 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
359
363
|
info.get("message_schema", ""),
|
|
360
364
|
)
|
|
361
365
|
case "pubsub":
|
|
362
|
-
info = await _multip_prompt(
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
366
|
+
info = await _multip_prompt(
|
|
367
|
+
[
|
|
368
|
+
("Project", [], ""),
|
|
369
|
+
("Subscription ID", [], ""),
|
|
370
|
+
_interactive_message_format_input(),
|
|
371
|
+
("Message schema", [], ""),
|
|
372
|
+
]
|
|
373
|
+
)
|
|
368
374
|
|
|
369
375
|
await _infer_schema_pubsub(
|
|
370
376
|
vc,
|
|
@@ -376,51 +382,53 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
376
382
|
info.get("message_schema", ""),
|
|
377
383
|
)
|
|
378
384
|
case "pubsublite":
|
|
379
|
-
info = await _multip_prompt(
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
385
|
+
info = await _multip_prompt(
|
|
386
|
+
[
|
|
387
|
+
("Project", [], ""),
|
|
388
|
+
("Subscription ID", [], ""),
|
|
389
|
+
# https://cloud.google.com/pubsub/lite/docs/locations
|
|
390
|
+
(
|
|
391
|
+
"Location",
|
|
392
|
+
[
|
|
393
|
+
"asia-east1",
|
|
394
|
+
"asia-east2",
|
|
395
|
+
"asia-northeast1",
|
|
396
|
+
"asia-northeast2",
|
|
397
|
+
"asia-northeast3",
|
|
398
|
+
"asia-south1",
|
|
399
|
+
"asia-southeast1",
|
|
400
|
+
"asia-southeast2",
|
|
401
|
+
"australia-southeast1",
|
|
402
|
+
"australia-southeast2",
|
|
403
|
+
"europe-central2",
|
|
404
|
+
"europe-north1",
|
|
405
|
+
"europe-west1",
|
|
406
|
+
"europe-west2",
|
|
407
|
+
"europe-west3",
|
|
408
|
+
"europe-west4",
|
|
409
|
+
"europe-west6",
|
|
410
|
+
"europe-west8",
|
|
411
|
+
"europe-west9",
|
|
412
|
+
"me-central1",
|
|
413
|
+
"northamerica-northeast1",
|
|
414
|
+
"me-west1",
|
|
415
|
+
"northamerica-northeast2",
|
|
416
|
+
"southamerica-east1",
|
|
417
|
+
"us-central1",
|
|
418
|
+
"us-east1",
|
|
419
|
+
"us-east4",
|
|
420
|
+
"us-east5",
|
|
421
|
+
"us-west1",
|
|
422
|
+
"us-west2",
|
|
423
|
+
"us-west3",
|
|
424
|
+
"us-west4",
|
|
425
|
+
],
|
|
426
|
+
"",
|
|
427
|
+
),
|
|
428
|
+
_interactive_message_format_input(),
|
|
429
|
+
("Message schema", [], ""),
|
|
430
|
+
]
|
|
431
|
+
)
|
|
424
432
|
await _infer_schema_pubsub_lite(
|
|
425
433
|
vc,
|
|
426
434
|
filename,
|
|
@@ -432,11 +440,13 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
432
440
|
info.get("message_schema", ""),
|
|
433
441
|
)
|
|
434
442
|
case "kafka":
|
|
435
|
-
info = await _multip_prompt(
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
443
|
+
info = await _multip_prompt(
|
|
444
|
+
[
|
|
445
|
+
("topic", [], ""),
|
|
446
|
+
_interactive_message_format_input(),
|
|
447
|
+
("Message schema", [], ""),
|
|
448
|
+
]
|
|
449
|
+
)
|
|
440
450
|
await _infer_schema_kafka(
|
|
441
451
|
vc,
|
|
442
452
|
filename,
|
|
@@ -446,12 +456,14 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
446
456
|
info.get("message_schema", ""),
|
|
447
457
|
)
|
|
448
458
|
case "s3":
|
|
449
|
-
info = await _multip_prompt(
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
459
|
+
info = await _multip_prompt(
|
|
460
|
+
[
|
|
461
|
+
("Bucket", [], ""),
|
|
462
|
+
("File pattern", [], "*.csv"),
|
|
463
|
+
("Prefix", [], ""),
|
|
464
|
+
("CSV delimiter", [",", "|"], ","),
|
|
465
|
+
]
|
|
466
|
+
)
|
|
455
467
|
|
|
456
468
|
print()
|
|
457
469
|
null_marker = await _get_null_marker()
|
|
@@ -467,11 +479,13 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
467
479
|
null_marker,
|
|
468
480
|
)
|
|
469
481
|
case "postgresql":
|
|
470
|
-
info = await _multip_prompt(
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
482
|
+
info = await _multip_prompt(
|
|
483
|
+
[
|
|
484
|
+
("Schema", [], ""),
|
|
485
|
+
("Database", [], ""),
|
|
486
|
+
("Table", [], ""),
|
|
487
|
+
]
|
|
488
|
+
)
|
|
475
489
|
await _infer_schema_postgresql(
|
|
476
490
|
vc,
|
|
477
491
|
filename,
|
|
@@ -481,11 +495,13 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
481
495
|
info.get("table", ""),
|
|
482
496
|
)
|
|
483
497
|
case "redshift":
|
|
484
|
-
info = await _multip_prompt(
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
498
|
+
info = await _multip_prompt(
|
|
499
|
+
[
|
|
500
|
+
("Schema", [], ""),
|
|
501
|
+
("Database", [], ""),
|
|
502
|
+
("Table", [], ""),
|
|
503
|
+
]
|
|
504
|
+
)
|
|
489
505
|
await _infer_schema_redshift(
|
|
490
506
|
vc,
|
|
491
507
|
filename,
|
|
@@ -495,13 +511,15 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
495
511
|
info.get("table", ""),
|
|
496
512
|
)
|
|
497
513
|
case "snowflake":
|
|
498
|
-
info = await _multip_prompt(
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
514
|
+
info = await _multip_prompt(
|
|
515
|
+
[
|
|
516
|
+
("Schema", [], ""),
|
|
517
|
+
("Database", [], ""),
|
|
518
|
+
("Table", [], ""),
|
|
519
|
+
("Warehouse", [], ""),
|
|
520
|
+
("Role", [], ""),
|
|
521
|
+
]
|
|
522
|
+
)
|
|
505
523
|
await _infer_schema_snowflake(
|
|
506
524
|
vc,
|
|
507
525
|
filename,
|
|
@@ -513,11 +531,13 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
513
531
|
info.get("warehouse", ""),
|
|
514
532
|
)
|
|
515
533
|
case "bigquery":
|
|
516
|
-
info = await _multip_prompt(
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
534
|
+
info = await _multip_prompt(
|
|
535
|
+
[
|
|
536
|
+
("Project", [], ""),
|
|
537
|
+
("Dataset", [], ""),
|
|
538
|
+
("Table", [], ""),
|
|
539
|
+
]
|
|
540
|
+
)
|
|
521
541
|
await _infer_schema_bigquery(
|
|
522
542
|
vc,
|
|
523
543
|
filename,
|
|
@@ -527,13 +547,15 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
|
|
|
527
547
|
info.get("table", ""),
|
|
528
548
|
)
|
|
529
549
|
case "gcs":
|
|
530
|
-
info = await _multip_prompt(
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
550
|
+
info = await _multip_prompt(
|
|
551
|
+
[
|
|
552
|
+
("Project", [], ""),
|
|
553
|
+
("Bucket", [], ""),
|
|
554
|
+
("Folder", [], ""),
|
|
555
|
+
("File pattern", ["*.csv"], "*.csv"),
|
|
556
|
+
("CSV delimiter", [",", "|"], ","),
|
|
557
|
+
]
|
|
558
|
+
)
|
|
537
559
|
|
|
538
560
|
print()
|
|
539
561
|
null_marker = await _get_null_marker()
|
|
@@ -134,11 +134,13 @@ def _list_executables() -> None:
|
|
|
134
134
|
executables = set()
|
|
135
135
|
for path in paths:
|
|
136
136
|
try:
|
|
137
|
-
executables.update(
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
137
|
+
executables.update(
|
|
138
|
+
{
|
|
139
|
+
filename.lstrip("validio-")
|
|
140
|
+
for filename in os.listdir(path)
|
|
141
|
+
if filename.startswith("validio-")
|
|
142
|
+
}
|
|
143
|
+
)
|
|
142
144
|
except FileNotFoundError:
|
|
143
145
|
continue
|
|
144
146
|
|
|
@@ -1,254 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from datetime import datetime, timedelta
|
|
3
|
-
from typing import Any
|
|
4
|
-
|
|
5
|
-
import typer
|
|
6
|
-
from validio_sdk import ValidioError
|
|
7
|
-
from validio_sdk.graphql_client import (
|
|
8
|
-
GetSegmentIncidentsSegmentIncidents,
|
|
9
|
-
GetSourceIncidentsSourceIncidentsSegmentLimitExceededNotification,
|
|
10
|
-
GetValidatorIncidentsValidatorIncidents,
|
|
11
|
-
GetValidatorSegmentIncidentsValidatorSegmentIncidents,
|
|
12
|
-
)
|
|
13
|
-
from validio_sdk.graphql_client.input_types import (
|
|
14
|
-
SegmentIncidentsInput,
|
|
15
|
-
SourceIncidentsInput,
|
|
16
|
-
TimeRangeInput,
|
|
17
|
-
ValidatorIncidentsInput,
|
|
18
|
-
ValidatorSegmentIncidentsInput,
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
from validio_cli import (
|
|
22
|
-
AsyncTyper,
|
|
23
|
-
ConfigDir,
|
|
24
|
-
Namespace,
|
|
25
|
-
OutputFormat,
|
|
26
|
-
OutputFormatOption,
|
|
27
|
-
OutputSettings,
|
|
28
|
-
_format_relative_time,
|
|
29
|
-
get_client_and_config,
|
|
30
|
-
output_json,
|
|
31
|
-
output_text,
|
|
32
|
-
)
|
|
33
|
-
from validio_cli.bin.entities import sources, validators
|
|
34
|
-
|
|
35
|
-
app = AsyncTyper(help="Incidents from validators")
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
@app.async_command(help="""List all incidents.
|
|
39
|
-
|
|
40
|
-
By default you will get incidents from the last hour. You can specify a time
|
|
41
|
-
range for when the incident occurred by specifying when the incident ended.
|
|
42
|
-
|
|
43
|
-
You can list incidents in different ways:
|
|
44
|
-
|
|
45
|
-
* Listing all incidents
|
|
46
|
-
|
|
47
|
-
* Listing all source incidents for a specific source with --source
|
|
48
|
-
|
|
49
|
-
* Listing all incidents for a specific validator with --validator
|
|
50
|
-
|
|
51
|
-
* Listing all incidents for a specific segment with --segment
|
|
52
|
-
|
|
53
|
-
* Listing all incidents for a specific validator and segment with --validator
|
|
54
|
-
and --segment together
|
|
55
|
-
""")
|
|
56
|
-
async def get(
|
|
57
|
-
config_dir: str = ConfigDir,
|
|
58
|
-
output_format: OutputFormat = OutputFormatOption,
|
|
59
|
-
namespace: str = Namespace(),
|
|
60
|
-
ended_before: datetime = typer.Option(
|
|
61
|
-
datetime.utcnow(),
|
|
62
|
-
help="The incident ended before this timestamp",
|
|
63
|
-
),
|
|
64
|
-
ended_after: datetime = typer.Option(
|
|
65
|
-
(datetime.utcnow() - timedelta(hours=1)),
|
|
66
|
-
help="The incident ended after this timestamp",
|
|
67
|
-
),
|
|
68
|
-
validator: str = typer.Option(None, help="Validator to fetch incidents for"),
|
|
69
|
-
segment: str = typer.Option(None, help="Segment to fetch incidents for"),
|
|
70
|
-
source: str = typer.Option(None, help="Source to fetch incidents for"),
|
|
71
|
-
) -> None:
|
|
72
|
-
vc, cfg = await get_client_and_config(config_dir)
|
|
73
|
-
|
|
74
|
-
if validator is not None:
|
|
75
|
-
validator_id = await validators.get_validator_id(vc, cfg, validator, namespace)
|
|
76
|
-
if validator_id is None:
|
|
77
|
-
return None
|
|
78
|
-
|
|
79
|
-
if source is not None:
|
|
80
|
-
source_id = await sources.get_source_id(vc, cfg, source, namespace)
|
|
81
|
-
if source_id is None:
|
|
82
|
-
return None
|
|
83
|
-
|
|
84
|
-
# TODO(UI-2006): These should all support namespace
|
|
85
|
-
|
|
86
|
-
incidents: (
|
|
87
|
-
None
|
|
88
|
-
| list[GetValidatorIncidentsValidatorIncidents]
|
|
89
|
-
| list[GetValidatorSegmentIncidentsValidatorSegmentIncidents]
|
|
90
|
-
| list[GetSegmentIncidentsSegmentIncidents]
|
|
91
|
-
) = None
|
|
92
|
-
|
|
93
|
-
if validator and segment:
|
|
94
|
-
incidents = await vc.get_validator_segment_incidents(
|
|
95
|
-
ValidatorSegmentIncidentsInput(
|
|
96
|
-
time_range=TimeRangeInput(
|
|
97
|
-
start=ended_after,
|
|
98
|
-
end=ended_before,
|
|
99
|
-
),
|
|
100
|
-
validator_id=validator_id,
|
|
101
|
-
segment_id=segment,
|
|
102
|
-
)
|
|
103
|
-
)
|
|
104
|
-
elif validator:
|
|
105
|
-
incidents = await vc.get_validator_incidents(
|
|
106
|
-
ValidatorIncidentsInput(
|
|
107
|
-
time_range=TimeRangeInput(
|
|
108
|
-
start=ended_after,
|
|
109
|
-
end=ended_before,
|
|
110
|
-
),
|
|
111
|
-
validator_id=validator_id,
|
|
112
|
-
)
|
|
113
|
-
)
|
|
114
|
-
elif segment:
|
|
115
|
-
incidents = await vc.get_segment_incidents(
|
|
116
|
-
SegmentIncidentsInput(
|
|
117
|
-
time_range=TimeRangeInput(
|
|
118
|
-
start=ended_after,
|
|
119
|
-
end=ended_before,
|
|
120
|
-
),
|
|
121
|
-
segment_id=segment,
|
|
122
|
-
)
|
|
123
|
-
)
|
|
124
|
-
elif source:
|
|
125
|
-
# Since source incidents looks completely different we hand them off to
|
|
126
|
-
# a separate function, we can't re-use any of the rendering.
|
|
127
|
-
return _output_text_source_incidents(
|
|
128
|
-
output_format,
|
|
129
|
-
await vc.get_source_incidents(
|
|
130
|
-
SourceIncidentsInput(
|
|
131
|
-
time_range=TimeRangeInput(
|
|
132
|
-
start=ended_after,
|
|
133
|
-
end=ended_before,
|
|
134
|
-
),
|
|
135
|
-
source_id=source_id,
|
|
136
|
-
)
|
|
137
|
-
),
|
|
138
|
-
)
|
|
139
|
-
else:
|
|
140
|
-
raise ValidioError(
|
|
141
|
-
"You need to specify one of --validator and/or --segment or --source"
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
if output_format == OutputFormat.JSON:
|
|
145
|
-
return output_json(incidents)
|
|
146
|
-
|
|
147
|
-
return output_text(
|
|
148
|
-
incidents,
|
|
149
|
-
fields={
|
|
150
|
-
"operator": OutputSettings(
|
|
151
|
-
attribute_name="metric",
|
|
152
|
-
reformat=calculate_operator,
|
|
153
|
-
),
|
|
154
|
-
"bound": OutputSettings(
|
|
155
|
-
attribute_name="metric",
|
|
156
|
-
reformat=calculate_bound,
|
|
157
|
-
),
|
|
158
|
-
"value": OutputSettings(
|
|
159
|
-
attribute_name="metric",
|
|
160
|
-
reformat=lambda x: x.value,
|
|
161
|
-
),
|
|
162
|
-
"age": OutputSettings(
|
|
163
|
-
attribute_name="metric",
|
|
164
|
-
reformat=lambda x: _format_relative_time(x.end_time),
|
|
165
|
-
),
|
|
166
|
-
},
|
|
167
|
-
)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def _output_text_source_incidents(
|
|
171
|
-
output_format: OutputFormat, incidents: list[Any]
|
|
172
|
-
) -> None:
|
|
173
|
-
if output_format == OutputFormat.JSON:
|
|
174
|
-
return output_json(incidents)
|
|
175
|
-
|
|
176
|
-
@dataclass
|
|
177
|
-
class SchemaChange:
|
|
178
|
-
field: str
|
|
179
|
-
change_type: str
|
|
180
|
-
old_value: str
|
|
181
|
-
new_value: str
|
|
182
|
-
|
|
183
|
-
change_type_to_key = {
|
|
184
|
-
"Nullability": "nullable",
|
|
185
|
-
"JtdType": "type",
|
|
186
|
-
"UnderlyingType": "underlying_type",
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
changes = []
|
|
190
|
-
for item in incidents:
|
|
191
|
-
# TODO: We only show schema changes for now
|
|
192
|
-
if item.typename__ != "SchemaChangeNotification":
|
|
193
|
-
continue
|
|
194
|
-
|
|
195
|
-
for field, field_changes in item.payload.items():
|
|
196
|
-
for ct in field_changes["change_types"]:
|
|
197
|
-
k = change_type_to_key.get(ct)
|
|
198
|
-
|
|
199
|
-
changes.append(
|
|
200
|
-
SchemaChange(
|
|
201
|
-
field=field,
|
|
202
|
-
change_type=ct,
|
|
203
|
-
old_value=field_changes["old"][k] if k else "-",
|
|
204
|
-
new_value=field_changes["new"][k] if k else "-",
|
|
205
|
-
)
|
|
206
|
-
)
|
|
207
|
-
|
|
208
|
-
return output_text(
|
|
209
|
-
changes,
|
|
210
|
-
fields={
|
|
211
|
-
"field": None,
|
|
212
|
-
"change_type": None,
|
|
213
|
-
"old_value": None,
|
|
214
|
-
"new_value": None,
|
|
215
|
-
},
|
|
216
|
-
)
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def calculate_operator(item: Any) -> str:
|
|
220
|
-
type_ = item.typename__[len("ValidatorMetricWith") :]
|
|
221
|
-
if type_ == "DynamicThreshold":
|
|
222
|
-
operator = item.decision_bounds_type
|
|
223
|
-
else:
|
|
224
|
-
operator = item.operator
|
|
225
|
-
|
|
226
|
-
return f"{type_}/{operator}"
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
def calculate_bound(item: Any) -> str:
|
|
230
|
-
type_ = item.typename__[len("ValidatorMetricWith") :]
|
|
231
|
-
if type_ == "DynamicThreshold":
|
|
232
|
-
bound = f"{item.lower_bound:.2f} - {item.upper_bound:.2f}"
|
|
233
|
-
elif type_ == "FixedThreshold":
|
|
234
|
-
bound = item.bound
|
|
235
|
-
else:
|
|
236
|
-
bound = "-"
|
|
237
|
-
|
|
238
|
-
return bound
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
def _output_text_source_incidents_segment_limit_exceeded(
|
|
242
|
-
incidents: list[GetSourceIncidentsSourceIncidentsSegmentLimitExceededNotification],
|
|
243
|
-
) -> None:
|
|
244
|
-
return output_text(
|
|
245
|
-
incidents,
|
|
246
|
-
fields={
|
|
247
|
-
"limit": None,
|
|
248
|
-
"age": OutputSettings(attribute_name="created_at"),
|
|
249
|
-
},
|
|
250
|
-
)
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
if __name__ == "__main__":
|
|
254
|
-
typer.run(app())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|