de-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_agent/__init__.py +1 -0
- de_agent/applyplan.py +270 -0
- de_agent/approval.py +110 -0
- de_agent/cli.py +650 -0
- de_agent/cliclass.py +1651 -0
- de_agent/clients.py +74 -0
- de_agent/collectors/__init__.py +31 -0
- de_agent/collectors/_airflow_core.py +343 -0
- de_agent/collectors/_pack_http_core.py +219 -0
- de_agent/collectors/airflow.py +125 -0
- de_agent/collectors/aws/__init__.py +172 -0
- de_agent/collectors/aws/_aws_core.py +823 -0
- de_agent/collectors/base.py +66 -0
- de_agent/collectors/dbt.py +198 -0
- de_agent/collectors/pack_base.py +225 -0
- de_agent/collectors/pack_cli.py +93 -0
- de_agent/collectors/pack_http.py +137 -0
- de_agent/collectors/pack_sql.py +217 -0
- de_agent/collectors/postgres.py +285 -0
- de_agent/collectors/redshift.py +815 -0
- de_agent/config.py +450 -0
- de_agent/configdoc.py +115 -0
- de_agent/data/sql_cases.yaml +1735 -0
- de_agent/diff.py +280 -0
- de_agent/extract.py +662 -0
- de_agent/find.py +218 -0
- de_agent/hubpath.py +32 -0
- de_agent/initcmd.py +49 -0
- de_agent/lint.py +297 -0
- de_agent/packs/__init__.py +5 -0
- de_agent/packs/argv.py +269 -0
- de_agent/packs/bigquery.yaml +67 -0
- de_agent/packs/check.py +228 -0
- de_agent/packs/cli.py +147 -0
- de_agent/packs/dagster.yaml +59 -0
- de_agent/packs/databricks-system.yaml +44 -0
- de_agent/packs/databricks.yaml +56 -0
- de_agent/packs/errors.py +6 -0
- de_agent/packs/expr.py +314 -0
- de_agent/packs/fivetran.yaml +30 -0
- de_agent/packs/gcp.yaml +53 -0
- de_agent/packs/kubernetes.yaml +64 -0
- de_agent/packs/loader.py +85 -0
- de_agent/packs/model.py +154 -0
- de_agent/packs/mssql.yaml +61 -0
- de_agent/packs/mysql.yaml +53 -0
- de_agent/packs/postgres-lite.yaml +65 -0
- de_agent/packs/scaffold.py +75 -0
- de_agent/packs/snowflake.yaml +91 -0
- de_agent/packs/template.py +111 -0
- de_agent/packs/view.py +64 -0
- de_agent/plans.py +621 -0
- de_agent/pyscan.py +214 -0
- de_agent/render.py +854 -0
- de_agent/runner.py +66 -0
- de_agent/scanners/__init__.py +1 -0
- de_agent/scanners/airflow.py +691 -0
- de_agent/scanners/catalog.py +205 -0
- de_agent/scanners/common.py +146 -0
- de_agent/scanners/dbt.py +99 -0
- de_agent/scanners/edges.py +358 -0
- de_agent/scanners/headers.py +117 -0
- de_agent/scanners/pycode.py +719 -0
- de_agent/scanners/pyexpr.py +308 -0
- de_agent/scanners/python.py +240 -0
- de_agent/scanners/run.py +288 -0
- de_agent/scanners/sql.py +131 -0
- de_agent/scanners/terraform.py +769 -0
- de_agent/secretscan.py +28 -0
- de_agent/snapshot.py +159 -0
- de_agent/sqlclass.py +333 -0
- de_agent/sqlexec.py +626 -0
- de_agent/stepexec.py +257 -0
- de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
- de_agent/templates/example/map_row.txt +1 -0
- de_agent/templates/example/projects/orders_etl.md +21 -0
- de_agent/templates/hub/README.md +16 -0
- de_agent/templates/hub/conventions.md +15 -0
- de_agent/templates/hub/de-agent.yaml +20 -0
- de_agent/templates/hub/domains/_template.md +10 -0
- de_agent/templates/hub/gitignore.tmpl +1 -0
- de_agent/templates/hub/integrations/_template.md +12 -0
- de_agent/templates/hub/map.md +17 -0
- de_agent/templates/hub/projects/_template.md +24 -0
- de_agent/templates/hub/status.md +23 -0
- de_agent-0.2.0.dist-info/METADATA +94 -0
- de_agent-0.2.0.dist-info/RECORD +91 -0
- de_agent-0.2.0.dist-info/WHEEL +4 -0
- de_agent-0.2.0.dist-info/entry_points.txt +2 -0
- de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
- de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/clients.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""The ONE list of database clients the guards know about. Stdlib only, Python 3.9 compatible (hooks import it).
|
|
2
|
+
|
|
3
|
+
`extract.py` (sql_guard, query_log) reads SQL out of these clients' command lines; `secret_guard.py` uses the same set to
|
|
4
|
+
spot inline passwords. Add a client here and both layers pick it up.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
# executable basename -> sqlglot dialect used to classify the SQL it runs
|
|
9
|
+
SQL_CLIENTS = {
|
|
10
|
+
"psql": "postgres",
|
|
11
|
+
"mysql": "mysql",
|
|
12
|
+
"mariadb": "mysql",
|
|
13
|
+
"snowsql": "snowflake",
|
|
14
|
+
"sqlcmd": "tsql",
|
|
15
|
+
"clickhouse-client": "clickhouse",
|
|
16
|
+
"clickhouse-local": "clickhouse",
|
|
17
|
+
"sqlite3": "sqlite",
|
|
18
|
+
"duckdb": "duckdb",
|
|
19
|
+
"trino": "trino",
|
|
20
|
+
"trino-cli": "trino",
|
|
21
|
+
"presto": "presto",
|
|
22
|
+
"presto-cli": "presto",
|
|
23
|
+
"spark-sql": "spark",
|
|
24
|
+
"beeline": "hive",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# clients with their own non-generic argument handling in extract.py (dialects come from the source name)
|
|
28
|
+
CLOUD_CLIENTS = {"bq": "bigquery", "aws": "postgres"}
|
|
29
|
+
|
|
30
|
+
# database tools whose SQL we do not read, but whose command lines can carry credentials
|
|
31
|
+
OTHER_DB_CLIENTS = frozenset({
|
|
32
|
+
"pg_dump", "pg_dumpall", "pg_restore", "mysqldump", "mysqladmin", "redshift-data", "pgcli", "mycli",
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
DB_CLIENTS = frozenset(SQL_CLIENTS) | {"bq"} | OTHER_DB_CLIENTS
|
|
36
|
+
|
|
37
|
+
# cloud / data CLIs whose own SQL (after a `sql` subcommand, or `-e`) `extract.py` reads; the value is the sqlglot dialect
|
|
38
|
+
CLI_SQL_CLIENTS = {"snow": "snowflake", "dbsqlcli": "databricks", "databricks": "databricks"}
|
|
39
|
+
|
|
40
|
+
# CLIs the CLI write guard (`scripts/cli_guard.py`, `cliclass.py`) classifies as read / write / unknown
|
|
41
|
+
GUARDED_CLIS = frozenset({
|
|
42
|
+
"aws", "gcloud", "gsutil", "bq", "databricks", "dbsqlcli", "kubectl", "helm", "terraform", "tofu", "pulumi", "az", "snow",
|
|
43
|
+
"dbt", "airflow", "dagster", "prefect", "kafka-topics", "kafka-configs",
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
# Python database drivers (import name prefixes): code that imports one is scanned for SQL (pyscan.py)
|
|
47
|
+
PY_DRIVERS = (
|
|
48
|
+
"psycopg", "psycopg2", "psycopg_pool", "sqlalchemy", "pymysql", "MySQLdb", "mysql", "mysql.connector",
|
|
49
|
+
"snowflake", "redshift_connector", "pyodbc", "pymssql", "duckdb", "sqlite3", "clickhouse_driver",
|
|
50
|
+
"clickhouse_connect", "asyncpg", "pg8000", "oracledb", "cx_Oracle", "pyhive", "trino", "databricks",
|
|
51
|
+
)
|
|
52
|
+
PY_DRIVER_DIALECT = {
|
|
53
|
+
"psycopg": "postgres", "psycopg2": "postgres", "psycopg_pool": "postgres", "asyncpg": "postgres",
|
|
54
|
+
"pg8000": "postgres", "sqlalchemy": "postgres", "pymysql": "mysql", "MySQLdb": "mysql", "mysql": "mysql",
|
|
55
|
+
"snowflake": "snowflake", "redshift_connector": "redshift", "pyodbc": "tsql", "pymssql": "tsql",
|
|
56
|
+
"duckdb": "duckdb", "sqlite3": "sqlite", "clickhouse_driver": "clickhouse", "clickhouse_connect": "clickhouse",
|
|
57
|
+
"pyhive": "hive", "trino": "trino", "oracledb": "oracle", "cx_Oracle": "oracle", "databricks": "spark",
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
_SOURCE_DIALECTS = {"redshift-data": "redshift", "bq": "bigquery", "athena": "athena"}
|
|
61
|
+
_SOURCE_DIALECTS.update(SQL_CLIENTS)
|
|
62
|
+
_SOURCE_DIALECTS.update(CLI_SQL_CLIENTS)
|
|
63
|
+
_SUFFIXES = ("-file", "-heredoc", "-herestring", "-stdin")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def dialect_for(source: str) -> str:
|
|
67
|
+
"""sqlglot dialect for an extract_sql source tag (`mysql`, `sqlite3-file`, `python:duckdb`, `redshift-data`, ...)."""
|
|
68
|
+
if source.startswith("python:"):
|
|
69
|
+
return source[len("python:"):] or "postgres"
|
|
70
|
+
for suf in _SUFFIXES:
|
|
71
|
+
if source.endswith(suf):
|
|
72
|
+
source = source[: -len(suf)]
|
|
73
|
+
break
|
|
74
|
+
return _SOURCE_DIALECTS.get(source, "postgres")
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Collector registry: {type: Collector}."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from de_agent.collectors.airflow import AirflowCollector
|
|
5
|
+
from de_agent.collectors.aws import AwsCollector
|
|
6
|
+
from de_agent.collectors.base import Collector, CollectorError, RunContext # noqa: F401
|
|
7
|
+
from de_agent.collectors.dbt import DbtCollector
|
|
8
|
+
from de_agent.collectors.pack_cli import CliPackCollector
|
|
9
|
+
from de_agent.collectors.pack_http import HttpPackCollector
|
|
10
|
+
from de_agent.collectors.pack_sql import SqlPackCollector
|
|
11
|
+
from de_agent.collectors.postgres import PostgresCollector
|
|
12
|
+
from de_agent.collectors.redshift import RedshiftCollector
|
|
13
|
+
|
|
14
|
+
REGISTRY: dict[str, Collector] = {
|
|
15
|
+
PostgresCollector.type: PostgresCollector(),
|
|
16
|
+
AirflowCollector.type: AirflowCollector(),
|
|
17
|
+
DbtCollector.type: DbtCollector(),
|
|
18
|
+
RedshiftCollector.type: RedshiftCollector(),
|
|
19
|
+
AwsCollector.type: AwsCollector(),
|
|
20
|
+
SqlPackCollector.type: SqlPackCollector(),
|
|
21
|
+
CliPackCollector.type: CliPackCollector(),
|
|
22
|
+
HttpPackCollector.type: HttpPackCollector(),
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def registered_types() -> list[str]:
|
|
27
|
+
return sorted(REGISTRY)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_collector(ctype: str) -> Collector:
|
|
31
|
+
return REGISTRY[ctype]
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""Airflow REST gathering, stdlib only (no de-agent imports).
|
|
2
|
+
|
|
3
|
+
This file is executed in two places: imported locally by collectors/airflow.py, and shipped as text over
|
|
4
|
+
`ssh <host> python3 -` for `exec: ssh:<host>` sources. Keep it Python 3.8+ and stdlib-only.
|
|
5
|
+
`get(path, params, accept)` returns (status, body); it is the only thing that touches the network, so it is
|
|
6
|
+
GET-only by construction and replaced by a replay function in tests.
|
|
7
|
+
"""
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
from datetime import datetime, timedelta, timezone
|
|
11
|
+
from urllib.parse import quote
|
|
12
|
+
|
|
13
|
+
LOG_TAIL_LINES = 40
|
|
14
|
+
STACK_LINES = 5
|
|
15
|
+
MAX_FAILURES = 50
|
|
16
|
+
MAX_LOG_FETCHES = 20
|
|
17
|
+
PAGE = 100
|
|
18
|
+
|
|
19
|
+
_REDACT = [
|
|
20
|
+
(re.compile(r"(\b[a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s]+(@)", re.I), r"\1***\2"),
|
|
21
|
+
(re.compile(r"(\b(?:password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*)[^\s'\",&]+", re.I), r"\1***"),
|
|
22
|
+
(re.compile(r"\b((?:PGPASSWORD|MYSQL_PWD)=)\S+"), r"\1***"),
|
|
23
|
+
(re.compile(r"(\bAWS_SECRET_ACCESS_KEY\s*[=:]\s*[\"']?)[A-Za-z0-9/+=]{20,}"), r"\1***"),
|
|
24
|
+
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "***"),
|
|
25
|
+
(re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]{8,}", re.I), r"\1***"),
|
|
26
|
+
(re.compile(r"(\bAuthorization:\s*(?:Basic|Bearer)\s+)\S+", re.I), r"\1***"),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def redact_text(text):
|
|
31
|
+
for rx, rep in _REDACT:
|
|
32
|
+
text = rx.sub(rep, text)
|
|
33
|
+
return text
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _parse_ts(v):
|
|
37
|
+
if not v or not isinstance(v, str):
|
|
38
|
+
return None
|
|
39
|
+
try:
|
|
40
|
+
return datetime.fromisoformat(v.replace("Z", "+00:00")).astimezone(timezone.utc)
|
|
41
|
+
except ValueError:
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _iso(dt):
|
|
46
|
+
return dt.strftime("%Y-%m-%dT%H:%M:%SZ") if dt else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class ApiError(Exception):
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _json(get, path, params=None):
|
|
54
|
+
status, body = get(path, params, "application/json")
|
|
55
|
+
if status != 200 or not isinstance(body, (dict, list)):
|
|
56
|
+
raise ApiError("GET %s -> HTTP %s" % (path, status))
|
|
57
|
+
return body
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _paged(get, path, key, params, cap):
|
|
61
|
+
out, offset = [], 0
|
|
62
|
+
while len(out) < cap:
|
|
63
|
+
body = _json(get, path, list(params) + [("limit", PAGE), ("offset", offset)])
|
|
64
|
+
rows = body.get(key) or []
|
|
65
|
+
out += rows
|
|
66
|
+
total = body.get("total_entries")
|
|
67
|
+
offset += PAGE
|
|
68
|
+
if not rows or (total is not None and offset >= total):
|
|
69
|
+
break
|
|
70
|
+
return out[:cap]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def detect_api(get):
|
|
74
|
+
net_error = None
|
|
75
|
+
for base in ("/api/v2", "/api/v1"):
|
|
76
|
+
try:
|
|
77
|
+
status, body = get(base + "/version", None, "application/json")
|
|
78
|
+
except ApiError as ex: # network failure: remember it, it is a better message than "no API found"
|
|
79
|
+
net_error = ex
|
|
80
|
+
continue
|
|
81
|
+
if status == 200 and isinstance(body, dict) and body.get("version"):
|
|
82
|
+
return base, str(body["version"])
|
|
83
|
+
if net_error is not None:
|
|
84
|
+
raise net_error
|
|
85
|
+
raise ApiError("no Airflow API found at /api/v2/version or /api/v1/version")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# ---- normalisers (Airflow 2.x /api/v1 and 3.x /api/v2 differ in field names) --------------------------
|
|
89
|
+
|
|
90
|
+
def _dag_row(d):
|
|
91
|
+
si = d.get("schedule_interval")
|
|
92
|
+
expr = si.get("value") if isinstance(si, dict) else (si if isinstance(si, str) else None)
|
|
93
|
+
expr = expr or d.get("timetable_summary")
|
|
94
|
+
nxt = d.get("next_dagrun_create_after") or d.get("next_dagrun_run_after") or d.get("next_dagrun")
|
|
95
|
+
return {
|
|
96
|
+
"dag_id": d.get("dag_id"),
|
|
97
|
+
"is_paused": bool(d.get("is_paused")),
|
|
98
|
+
"schedule": expr,
|
|
99
|
+
"schedule_desc": d.get("timetable_description"),
|
|
100
|
+
"tags": sorted(t.get("name") if isinstance(t, dict) else str(t) for t in d.get("tags") or []),
|
|
101
|
+
"owners": ",".join(d.get("owners") or []) if isinstance(d.get("owners"), list) else d.get("owners"),
|
|
102
|
+
"fileloc": d.get("fileloc"),
|
|
103
|
+
"next_dagrun": nxt,
|
|
104
|
+
"has_import_errors": bool(d.get("has_import_errors")),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _is_live(d):
|
|
109
|
+
if "is_stale" in d:
|
|
110
|
+
return not d["is_stale"]
|
|
111
|
+
return d.get("is_active", True) is not False
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _run_row(dag_id, r):
|
|
115
|
+
start, end = _parse_ts(r.get("start_date")), _parse_ts(r.get("end_date"))
|
|
116
|
+
return {
|
|
117
|
+
"dag_id": dag_id,
|
|
118
|
+
"run_id": r.get("dag_run_id"),
|
|
119
|
+
"state": r.get("state"),
|
|
120
|
+
"logical_date": r.get("logical_date") or r.get("execution_date"),
|
|
121
|
+
"start": _iso(start),
|
|
122
|
+
"end": _iso(end),
|
|
123
|
+
"duration_s": int((end - start).total_seconds()) if start and end else None,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _log_lines(body):
|
|
128
|
+
if isinstance(body, str):
|
|
129
|
+
return body.splitlines()
|
|
130
|
+
if isinstance(body, dict):
|
|
131
|
+
content = body.get("content")
|
|
132
|
+
if isinstance(content, str):
|
|
133
|
+
return content.splitlines()
|
|
134
|
+
lines = []
|
|
135
|
+
for item in content or []:
|
|
136
|
+
if isinstance(item, dict):
|
|
137
|
+
lines.append(str(item.get("event", item)))
|
|
138
|
+
elif isinstance(item, (list, tuple)) and item:
|
|
139
|
+
lines.append(str(item[-1]))
|
|
140
|
+
else:
|
|
141
|
+
lines.append(str(item))
|
|
142
|
+
return lines
|
|
143
|
+
return []
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _log_tail(get, api, ti):
|
|
147
|
+
path = "%s/dags/%s/dagRuns/%s/taskInstances/%s/logs/%s" % (
|
|
148
|
+
api, quote(ti["dag_id"], safe=""), quote(ti["dag_run_id"], safe=""),
|
|
149
|
+
quote(ti["task_id"], safe=""), int(ti.get("try_number") or 1))
|
|
150
|
+
status, body = get(path, [("full_content", "true")], "text/plain")
|
|
151
|
+
if status != 200:
|
|
152
|
+
raise ApiError("log HTTP %s" % status)
|
|
153
|
+
lines = [ln.rstrip() for ln in _log_lines(body) if ln.strip()]
|
|
154
|
+
return redact_text("\n".join(lines[-LOG_TAIL_LINES:]))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def schedule_interval_s(schedule, runs):
|
|
158
|
+
"""Best-effort interval in seconds: cron/preset heuristics, else the median gap of scheduled runs."""
|
|
159
|
+
s = (schedule or "").strip()
|
|
160
|
+
presets = {"@hourly": 3600, "@daily": 86400, "@weekly": 604800, "@monthly": 2592000, "@yearly": 31536000,
|
|
161
|
+
"@once": None, "@continuous": None}
|
|
162
|
+
if s in presets:
|
|
163
|
+
return presets[s]
|
|
164
|
+
parts = s.split()
|
|
165
|
+
if len(parts) == 5:
|
|
166
|
+
minute, hour, dom, mon, dow = parts
|
|
167
|
+
if hour == "*" and dom == mon == dow == "*" and not minute.startswith("*/"):
|
|
168
|
+
return 3600
|
|
169
|
+
if minute.startswith("*/") and minute[2:].isdigit():
|
|
170
|
+
return int(minute[2:]) * 60
|
|
171
|
+
if hour.isdigit() and dom == mon == dow == "*":
|
|
172
|
+
return 86400
|
|
173
|
+
if dow != "*" and dom == mon == "*":
|
|
174
|
+
return 604800
|
|
175
|
+
scheduled = sorted(t for t in (_parse_ts(r.get("logical_date")) for r in runs
|
|
176
|
+
if str(r.get("run_id", "")).startswith("scheduled__")) if t)
|
|
177
|
+
gaps = sorted((b - a).total_seconds() for a, b in zip(scheduled, scheduled[1:]))
|
|
178
|
+
return int(gaps[len(gaps) // 2]) if gaps else None
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def gather(cfg, get, now=None):
|
|
182
|
+
"""Return {"data": {...}, "errors": [...]}; a failing section is reported, the others still return."""
|
|
183
|
+
now = now or datetime.now(timezone.utc)
|
|
184
|
+
max_dags = int(cfg.get("max_dags", 500))
|
|
185
|
+
runs_per_dag = int(cfg.get("runs_per_dag", 5))
|
|
186
|
+
fetch_logs = bool(cfg.get("fetch_logs", True))
|
|
187
|
+
data, errors = {}, []
|
|
188
|
+
api, version = detect_api(get)
|
|
189
|
+
data["version"] = {"version": version, "api": api.rsplit("/", 1)[1]}
|
|
190
|
+
is_v2 = api.endswith("v2")
|
|
191
|
+
|
|
192
|
+
def section(name, fn):
|
|
193
|
+
try:
|
|
194
|
+
data[name] = fn()
|
|
195
|
+
except ApiError as ex:
|
|
196
|
+
errors.append("%s: %s" % (name, ex))
|
|
197
|
+
|
|
198
|
+
def dags():
|
|
199
|
+
raw = _paged(get, api + "/dags", "dags", [("order_by", "dag_id")], max_dags)
|
|
200
|
+
return [_dag_row(d) for d in raw if _is_live(d)]
|
|
201
|
+
|
|
202
|
+
section("dags", dags)
|
|
203
|
+
dag_ids = [d["dag_id"] for d in data.get("dags", [])]
|
|
204
|
+
|
|
205
|
+
def runs():
|
|
206
|
+
out = []
|
|
207
|
+
for dag_id in dag_ids:
|
|
208
|
+
body = _json(get, "%s/dags/%s/dagRuns" % (api, quote(dag_id, safe="")),
|
|
209
|
+
[("order_by", "-start_date"), ("limit", runs_per_dag)])
|
|
210
|
+
out += [_run_row(dag_id, r) for r in body.get("dag_runs") or []]
|
|
211
|
+
return out
|
|
212
|
+
|
|
213
|
+
if "dags" in data:
|
|
214
|
+
section("runs", runs)
|
|
215
|
+
|
|
216
|
+
def failures():
|
|
217
|
+
since = _iso(now - timedelta(hours=24))
|
|
218
|
+
tis = _paged(get, api + "/dags/~/dagRuns/~/taskInstances", "task_instances",
|
|
219
|
+
[("state", "failed"), ("state", "up_for_retry"), ("start_date_gte", since)], MAX_FAILURES)
|
|
220
|
+
tis.sort(key=lambda t: str(t.get("start_date")), reverse=True) # the endpoint has no order_by
|
|
221
|
+
out = []
|
|
222
|
+
for i, ti in enumerate(tis):
|
|
223
|
+
row = {"dag_id": ti.get("dag_id"), "task_id": ti.get("task_id"), "try": int(ti.get("try_number") or 1),
|
|
224
|
+
"run_id": ti.get("dag_run_id"), "state": ti.get("state"), "start": ti.get("start_date"),
|
|
225
|
+
"duration_s": int(ti["duration"]) if isinstance(ti.get("duration"), (int, float)) else None}
|
|
226
|
+
if fetch_logs and i < MAX_LOG_FETCHES:
|
|
227
|
+
try:
|
|
228
|
+
row["log_tail"] = _log_tail(get, api, ti)
|
|
229
|
+
except ApiError as ex:
|
|
230
|
+
row["log_tail"] = "(log unavailable: %s)" % ex
|
|
231
|
+
out.append(row)
|
|
232
|
+
return out
|
|
233
|
+
|
|
234
|
+
section("failures_24h", failures)
|
|
235
|
+
|
|
236
|
+
def import_errors():
|
|
237
|
+
rows = _paged(get, api + "/importErrors", "import_errors", [], 100)
|
|
238
|
+
return [{"filename": r.get("filename"),
|
|
239
|
+
"stack": "\n".join((r.get("stack_trace") or "").splitlines()[:STACK_LINES])} for r in rows]
|
|
240
|
+
|
|
241
|
+
section("import_errors", import_errors)
|
|
242
|
+
|
|
243
|
+
def datasets():
|
|
244
|
+
key = "assets" if is_v2 else "datasets"
|
|
245
|
+
rows = _paged(get, "%s/%s" % (api, key), key, [], 200)
|
|
246
|
+
out = []
|
|
247
|
+
for r in rows:
|
|
248
|
+
last = None
|
|
249
|
+
le = r.get("last_asset_event") or r.get("last_dataset_event")
|
|
250
|
+
if isinstance(le, dict):
|
|
251
|
+
last = le.get("timestamp")
|
|
252
|
+
if last is None:
|
|
253
|
+
ev = "assets/events" if is_v2 else "datasets/events"
|
|
254
|
+
idk = "asset_id" if is_v2 else "dataset_id"
|
|
255
|
+
evk = "asset_events" if is_v2 else "dataset_events"
|
|
256
|
+
try:
|
|
257
|
+
body = _json(get, "%s/%s" % (api, ev), [(idk, r.get("id")), ("order_by", "-timestamp"), ("limit", 1)])
|
|
258
|
+
evs = body.get(evk) or []
|
|
259
|
+
last = evs[0].get("timestamp") if evs else None
|
|
260
|
+
except ApiError:
|
|
261
|
+
last = None
|
|
262
|
+
prod = sorted("%s.%s" % (t.get("dag_id"), t.get("task_id")) for t in r.get("producing_tasks") or [])
|
|
263
|
+
cons = sorted(set(c.get("dag_id") for c in (r.get("consuming_dags") or []) if isinstance(c, dict)))
|
|
264
|
+
out.append({"uri": r.get("uri") or r.get("name"), "producing_tasks": prod, "consuming_dags": cons,
|
|
265
|
+
"last_event": _iso(_parse_ts(last))})
|
|
266
|
+
return out
|
|
267
|
+
|
|
268
|
+
section("datasets", datasets)
|
|
269
|
+
|
|
270
|
+
def pools():
|
|
271
|
+
body = _json(get, api + "/pools", [("limit", PAGE)])
|
|
272
|
+
return [{"name": p.get("name"), "slots": p.get("slots"), "occupied": p.get("occupied_slots"),
|
|
273
|
+
"queued": p.get("queued_slots")} for p in body.get("pools") or []]
|
|
274
|
+
|
|
275
|
+
section("pools", pools)
|
|
276
|
+
data["meta"] = {"collected_at": _iso(now), "runs_per_dag": runs_per_dag, "max_dags": max_dags,
|
|
277
|
+
"fetch_logs": fetch_logs}
|
|
278
|
+
return {"data": data, "errors": errors}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# ---- remote entry point (exec: ssh:<host>) ------------------------------------------------------------
|
|
282
|
+
|
|
283
|
+
def resolve_remote_secret(ref):
|
|
284
|
+
import os
|
|
285
|
+
if ref.startswith("env:"):
|
|
286
|
+
val = os.environ.get(ref[4:])
|
|
287
|
+
if val is None:
|
|
288
|
+
raise ApiError("environment variable %s is not set on the remote host" % ref[4:])
|
|
289
|
+
return val
|
|
290
|
+
if ref.startswith("file:"):
|
|
291
|
+
try:
|
|
292
|
+
with open(ref[5:]) as fh:
|
|
293
|
+
return fh.read().strip()
|
|
294
|
+
except OSError as ex:
|
|
295
|
+
raise ApiError("cannot read %s on the remote host: %s" % (ref[5:], ex.strerror)) from None
|
|
296
|
+
raise ApiError("on a remote host the secret ref must be env: or file:")
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def make_urllib_get(base_url, auth_header, timeout=20):
|
|
300
|
+
import urllib.error
|
|
301
|
+
import urllib.request
|
|
302
|
+
|
|
303
|
+
def get(path, params=None, accept="application/json"):
|
|
304
|
+
from urllib.parse import urlencode
|
|
305
|
+
url = base_url.rstrip("/") + path + ("?" + urlencode(params) if params else "")
|
|
306
|
+
req = urllib.request.Request(url, method="GET")
|
|
307
|
+
req.add_header("Accept", accept)
|
|
308
|
+
if auth_header:
|
|
309
|
+
req.add_header("Authorization", auth_header)
|
|
310
|
+
try:
|
|
311
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
312
|
+
raw, status, ctype = resp.read(), resp.status, resp.headers.get("Content-Type", "")
|
|
313
|
+
except urllib.error.HTTPError as ex:
|
|
314
|
+
raw, status, ctype = ex.read(), ex.code, ex.headers.get("Content-Type", "")
|
|
315
|
+
except (urllib.error.URLError, OSError) as ex:
|
|
316
|
+
raise ApiError("cannot reach %s: %s" % (base_url, getattr(ex, "reason", ex))) from None
|
|
317
|
+
text = raw.decode("utf-8", "replace")
|
|
318
|
+
if "json" in ctype:
|
|
319
|
+
try:
|
|
320
|
+
return status, json.loads(text)
|
|
321
|
+
except ValueError:
|
|
322
|
+
return status, text
|
|
323
|
+
return status, text
|
|
324
|
+
|
|
325
|
+
return get
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def auth_header_for(cfg, secret):
|
|
329
|
+
import base64
|
|
330
|
+
if cfg.get("auth") == "token":
|
|
331
|
+
return "Bearer " + secret
|
|
332
|
+
return "Basic " + base64.b64encode(("%s:%s" % (cfg.get("user", ""), secret)).encode()).decode()
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def remote_main(cfg):
|
|
336
|
+
"""Entry point used on the remote host: resolve the secret THERE, call the API, return the result dict."""
|
|
337
|
+
try:
|
|
338
|
+
ref = cfg.get("token") if cfg.get("auth") == "token" else cfg.get("password")
|
|
339
|
+
secret = resolve_remote_secret(ref) if ref else None
|
|
340
|
+
get = make_urllib_get(cfg["base_url"], auth_header_for(cfg, secret) if secret else None)
|
|
341
|
+
return gather(cfg, get)
|
|
342
|
+
except ApiError as ex:
|
|
343
|
+
return {"data": {}, "errors": [str(ex)]}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""HTTP fetching for `kind: http` packs, stdlib only (no de-agent imports).
|
|
2
|
+
|
|
3
|
+
Executed in two places: imported locally by collectors/pack_http.py, and shipped as text over `ssh <host> python3 -` for
|
|
4
|
+
`exec: ssh:<host>` sources (secrets are then resolved on the remote host from env:/file: refs). Keep it Python 3.8+.
|
|
5
|
+
|
|
6
|
+
`request()` is the ONLY function that touches the network, and it is read-only by construction: GET, or POST of a GraphQL
|
|
7
|
+
document that contains no mutation or subscription. Anything else raises ValueError before a socket is opened. Redirects are not
|
|
8
|
+
followed (an Authorization header must never travel to another host). Responses are capped at 5 MB.
|
|
9
|
+
"""
|
|
10
|
+
import base64
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import stat
|
|
15
|
+
import urllib.error
|
|
16
|
+
import urllib.request
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
20
|
+
MAX_GRAPHQL = 10000
|
|
21
|
+
|
|
22
|
+
_REDACT = [
|
|
23
|
+
(re.compile(r"(\b[a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s]+(@)", re.I), r"\1***\2"),
|
|
24
|
+
(re.compile(r"(\b(?:password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*)[^\s'\",&]+", re.I), r"\1***"),
|
|
25
|
+
(re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]{8,}", re.I), r"\1***"),
|
|
26
|
+
(re.compile(r"(\bAuthorization:\s*(?:Basic|Bearer)\s+)\S+", re.I), r"\1***"),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def redact_text(text):
|
|
31
|
+
for rx, rep in _REDACT:
|
|
32
|
+
text = rx.sub(rep, text)
|
|
33
|
+
return text
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
37
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
42
|
+
|
|
43
|
+
_WRITE_WORDS = re.compile(r"\b(mutation|subscription)\b")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _graphql_bare(q):
|
|
47
|
+
"""`q` with comments blanked and string / block-string contents emptied, scanned left to right so that a `#` inside a
|
|
48
|
+
string and a quote inside a comment are each read correctly. None when a string is not terminated."""
|
|
49
|
+
out = []
|
|
50
|
+
i, n = 0, len(q)
|
|
51
|
+
while i < n:
|
|
52
|
+
c = q[i]
|
|
53
|
+
if c == "#":
|
|
54
|
+
j = q.find("\n", i)
|
|
55
|
+
i = n if j == -1 else j
|
|
56
|
+
out.append(" ")
|
|
57
|
+
elif q.startswith('"""', i):
|
|
58
|
+
j = i + 3
|
|
59
|
+
while j < n and not q.startswith('"""', j):
|
|
60
|
+
j += 4 if q.startswith('\\"""', j) else 1
|
|
61
|
+
if j >= n:
|
|
62
|
+
return None
|
|
63
|
+
out.append('""')
|
|
64
|
+
i = j + 3
|
|
65
|
+
elif c == '"':
|
|
66
|
+
j = i + 1
|
|
67
|
+
while j < n and q[j] not in '"\n':
|
|
68
|
+
j += 2 if q[j] == "\\" else 1
|
|
69
|
+
if j >= n or q[j] != '"':
|
|
70
|
+
return None
|
|
71
|
+
out.append('""')
|
|
72
|
+
i = j + 1
|
|
73
|
+
else:
|
|
74
|
+
out.append(c)
|
|
75
|
+
i += 1
|
|
76
|
+
return "".join(out)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def graphql_refusal(query):
|
|
80
|
+
"""None when `query` is a read-only GraphQL document (starts with `query` or `{`; no mutation/subscription anywhere)."""
|
|
81
|
+
if not isinstance(query, str) or not query.strip():
|
|
82
|
+
return "graphql query is empty"
|
|
83
|
+
if len(query) > MAX_GRAPHQL:
|
|
84
|
+
return "graphql query is too long"
|
|
85
|
+
bare = _graphql_bare(query)
|
|
86
|
+
if bare is None:
|
|
87
|
+
return "graphql query has a string that is not terminated"
|
|
88
|
+
head = bare.lstrip()
|
|
89
|
+
if not (head.startswith("{") or re.match(r"query\b", head)):
|
|
90
|
+
return "a graphql document must start with `query` or `{` (never mutation or subscription)"
|
|
91
|
+
if _WRITE_WORDS.search(bare):
|
|
92
|
+
return "graphql documents may not contain mutation or subscription"
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def path_refusal(path):
|
|
97
|
+
"""None when `path` is a relative request path (`/v1/things?limit=5`) that stays on the source's own host."""
|
|
98
|
+
if not isinstance(path, str) or not path:
|
|
99
|
+
return "path is empty"
|
|
100
|
+
if not path.startswith("/") or path.startswith("//"):
|
|
101
|
+
return "path must start with a single '/' (it is relative to the source's base_url)"
|
|
102
|
+
if re.search(r"[\s\x00-\x1f\\]", path) or "://" in path:
|
|
103
|
+
return "path may not contain whitespace, backslashes or a scheme"
|
|
104
|
+
if ".." in path.split("?", 1)[0].split("/"):
|
|
105
|
+
return "path may not contain '..' segments"
|
|
106
|
+
return None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def build_auth_header(auth, user, secret, header_name):
|
|
110
|
+
"""(header name, value) or None."""
|
|
111
|
+
if auth == "none" or secret is None:
|
|
112
|
+
return None
|
|
113
|
+
if auth == "bearer":
|
|
114
|
+
return ("Authorization", "Bearer " + secret)
|
|
115
|
+
if auth == "basic":
|
|
116
|
+
raw = ("%s:%s" % (user or "", secret)).encode("utf-8")
|
|
117
|
+
return ("Authorization", "Basic " + base64.b64encode(raw).decode("ascii"))
|
|
118
|
+
if auth == "header":
|
|
119
|
+
return (header_name or "X-Api-Key", secret)
|
|
120
|
+
raise ValueError("unknown auth %r" % (auth,))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def request(base_url, method, path, headers, timeout, graphql_query=None):
|
|
124
|
+
"""One read-only request. Returns (status, parsed JSON body). Raises ValueError (refused) or IOError (failed)."""
|
|
125
|
+
why = path_refusal(path)
|
|
126
|
+
if why:
|
|
127
|
+
raise ValueError(why)
|
|
128
|
+
if method == "GET":
|
|
129
|
+
if graphql_query is not None:
|
|
130
|
+
raise ValueError("GET does not carry a graphql document")
|
|
131
|
+
data = None
|
|
132
|
+
elif method == "POST":
|
|
133
|
+
why = graphql_refusal(graphql_query)
|
|
134
|
+
if why:
|
|
135
|
+
raise ValueError(why)
|
|
136
|
+
data = json.dumps({"query": graphql_query}).encode("utf-8")
|
|
137
|
+
else:
|
|
138
|
+
raise ValueError("only GET and read-only GraphQL POST are allowed (got %s)" % (method,))
|
|
139
|
+
base = urlsplit(base_url)
|
|
140
|
+
if base.scheme not in ("http", "https") or not base.netloc:
|
|
141
|
+
raise ValueError("base_url must be an http(s) URL")
|
|
142
|
+
url = base_url.rstrip("/") + path
|
|
143
|
+
if urlsplit(url).netloc != base.netloc:
|
|
144
|
+
raise ValueError("request would leave the source's host")
|
|
145
|
+
req = urllib.request.Request(url, data=data, method=method)
|
|
146
|
+
req.add_header("Accept", "application/json")
|
|
147
|
+
if data is not None:
|
|
148
|
+
req.add_header("Content-Type", "application/json")
|
|
149
|
+
for k, v in (headers or {}).items():
|
|
150
|
+
req.add_header(k, v)
|
|
151
|
+
try:
|
|
152
|
+
resp = _OPENER.open(req, timeout=timeout)
|
|
153
|
+
except urllib.error.HTTPError as ex:
|
|
154
|
+
return ex.code, None
|
|
155
|
+
except (urllib.error.URLError, OSError, ValueError) as ex:
|
|
156
|
+
raise OSError(redact_text("request failed: %s" % (getattr(ex, "reason", None) or type(ex).__name__))) from None
|
|
157
|
+
try:
|
|
158
|
+
raw = resp.read(MAX_BYTES + 1)
|
|
159
|
+
status = getattr(resp, "status", 200)
|
|
160
|
+
finally:
|
|
161
|
+
resp.close()
|
|
162
|
+
if len(raw) > MAX_BYTES:
|
|
163
|
+
raise OSError("response larger than 5 MB")
|
|
164
|
+
try:
|
|
165
|
+
return status, json.loads(raw.decode("utf-8"))
|
|
166
|
+
except ValueError:
|
|
167
|
+
raise OSError("response was not JSON") from None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def fetch(base_url, reqs, headers, timeout):
|
|
171
|
+
"""reqs: [{"kind": "get"|"graphql", "path": str, "query": str|None}] -> [{"ok": True, "body": ...} | {"ok": False, "error": str}]."""
|
|
172
|
+
out = []
|
|
173
|
+
for r in reqs:
|
|
174
|
+
try:
|
|
175
|
+
if r.get("kind") == "get":
|
|
176
|
+
status, body = request(base_url, "GET", r["path"], headers, timeout)
|
|
177
|
+
elif r.get("kind") == "graphql":
|
|
178
|
+
status, body = request(base_url, "POST", r["path"], headers, timeout, r.get("query"))
|
|
179
|
+
else:
|
|
180
|
+
raise ValueError("only GET and read-only GraphQL POST are allowed")
|
|
181
|
+
except (OSError, ValueError) as ex:
|
|
182
|
+
out.append({"ok": False, "error": redact_text(str(ex))})
|
|
183
|
+
continue
|
|
184
|
+
if status is None or status >= 300 or body is None:
|
|
185
|
+
out.append({"ok": False, "error": "HTTP %s" % (status,)})
|
|
186
|
+
else:
|
|
187
|
+
out.append({"ok": True, "body": body})
|
|
188
|
+
return out
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _resolve_remote(ref):
|
|
192
|
+
if ref is None or ref == "none":
|
|
193
|
+
return None
|
|
194
|
+
if ref.startswith("env:"):
|
|
195
|
+
val = os.environ.get(ref[4:])
|
|
196
|
+
if val is None:
|
|
197
|
+
raise ValueError("environment variable %s is not set on the remote host" % ref[4:])
|
|
198
|
+
return val
|
|
199
|
+
if ref.startswith("file:"):
|
|
200
|
+
path = ref[5:]
|
|
201
|
+
try:
|
|
202
|
+
if stat.S_IMODE(os.stat(path).st_mode) & 0o077:
|
|
203
|
+
raise ValueError("secret file %s must be mode 0600 or stricter" % path)
|
|
204
|
+
with open(path) as f:
|
|
205
|
+
return f.read().strip()
|
|
206
|
+
except OSError as ex:
|
|
207
|
+
raise ValueError("cannot read secret file %s: %s" % (path, ex.strerror)) from None
|
|
208
|
+
raise ValueError("secret refs over ssh must be env: or file: (resolved on the remote host)")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def remote_main(cfg):
|
|
212
|
+
"""cfg: {base_url, auth, user, secret_ref, header_name, timeout, requests}. Secrets are resolved HERE, on the remote host."""
|
|
213
|
+
try:
|
|
214
|
+
secret = _resolve_remote(cfg.get("secret_ref"))
|
|
215
|
+
hdr = build_auth_header(cfg.get("auth", "none"), cfg.get("user"), secret, cfg.get("header_name"))
|
|
216
|
+
except ValueError as ex:
|
|
217
|
+
return {"error": redact_text(str(ex))}
|
|
218
|
+
headers = {hdr[0]: hdr[1]} if hdr else {}
|
|
219
|
+
return {"results": fetch(cfg["base_url"], cfg["requests"], headers, cfg.get("timeout", 20))}
|