de-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. de_agent/__init__.py +1 -0
  2. de_agent/applyplan.py +270 -0
  3. de_agent/approval.py +110 -0
  4. de_agent/cli.py +650 -0
  5. de_agent/cliclass.py +1651 -0
  6. de_agent/clients.py +74 -0
  7. de_agent/collectors/__init__.py +31 -0
  8. de_agent/collectors/_airflow_core.py +343 -0
  9. de_agent/collectors/_pack_http_core.py +219 -0
  10. de_agent/collectors/airflow.py +125 -0
  11. de_agent/collectors/aws/__init__.py +172 -0
  12. de_agent/collectors/aws/_aws_core.py +823 -0
  13. de_agent/collectors/base.py +66 -0
  14. de_agent/collectors/dbt.py +198 -0
  15. de_agent/collectors/pack_base.py +225 -0
  16. de_agent/collectors/pack_cli.py +93 -0
  17. de_agent/collectors/pack_http.py +137 -0
  18. de_agent/collectors/pack_sql.py +217 -0
  19. de_agent/collectors/postgres.py +285 -0
  20. de_agent/collectors/redshift.py +815 -0
  21. de_agent/config.py +450 -0
  22. de_agent/configdoc.py +115 -0
  23. de_agent/data/sql_cases.yaml +1735 -0
  24. de_agent/diff.py +280 -0
  25. de_agent/extract.py +662 -0
  26. de_agent/find.py +218 -0
  27. de_agent/hubpath.py +32 -0
  28. de_agent/initcmd.py +49 -0
  29. de_agent/lint.py +297 -0
  30. de_agent/packs/__init__.py +5 -0
  31. de_agent/packs/argv.py +269 -0
  32. de_agent/packs/bigquery.yaml +67 -0
  33. de_agent/packs/check.py +228 -0
  34. de_agent/packs/cli.py +147 -0
  35. de_agent/packs/dagster.yaml +59 -0
  36. de_agent/packs/databricks-system.yaml +44 -0
  37. de_agent/packs/databricks.yaml +56 -0
  38. de_agent/packs/errors.py +6 -0
  39. de_agent/packs/expr.py +314 -0
  40. de_agent/packs/fivetran.yaml +30 -0
  41. de_agent/packs/gcp.yaml +53 -0
  42. de_agent/packs/kubernetes.yaml +64 -0
  43. de_agent/packs/loader.py +85 -0
  44. de_agent/packs/model.py +154 -0
  45. de_agent/packs/mssql.yaml +61 -0
  46. de_agent/packs/mysql.yaml +53 -0
  47. de_agent/packs/postgres-lite.yaml +65 -0
  48. de_agent/packs/scaffold.py +75 -0
  49. de_agent/packs/snowflake.yaml +91 -0
  50. de_agent/packs/template.py +111 -0
  51. de_agent/packs/view.py +64 -0
  52. de_agent/plans.py +621 -0
  53. de_agent/pyscan.py +214 -0
  54. de_agent/render.py +854 -0
  55. de_agent/runner.py +66 -0
  56. de_agent/scanners/__init__.py +1 -0
  57. de_agent/scanners/airflow.py +691 -0
  58. de_agent/scanners/catalog.py +205 -0
  59. de_agent/scanners/common.py +146 -0
  60. de_agent/scanners/dbt.py +99 -0
  61. de_agent/scanners/edges.py +358 -0
  62. de_agent/scanners/headers.py +117 -0
  63. de_agent/scanners/pycode.py +719 -0
  64. de_agent/scanners/pyexpr.py +308 -0
  65. de_agent/scanners/python.py +240 -0
  66. de_agent/scanners/run.py +288 -0
  67. de_agent/scanners/sql.py +131 -0
  68. de_agent/scanners/terraform.py +769 -0
  69. de_agent/secretscan.py +28 -0
  70. de_agent/snapshot.py +159 -0
  71. de_agent/sqlclass.py +333 -0
  72. de_agent/sqlexec.py +626 -0
  73. de_agent/stepexec.py +257 -0
  74. de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
  75. de_agent/templates/example/map_row.txt +1 -0
  76. de_agent/templates/example/projects/orders_etl.md +21 -0
  77. de_agent/templates/hub/README.md +16 -0
  78. de_agent/templates/hub/conventions.md +15 -0
  79. de_agent/templates/hub/de-agent.yaml +20 -0
  80. de_agent/templates/hub/domains/_template.md +10 -0
  81. de_agent/templates/hub/gitignore.tmpl +1 -0
  82. de_agent/templates/hub/integrations/_template.md +12 -0
  83. de_agent/templates/hub/map.md +17 -0
  84. de_agent/templates/hub/projects/_template.md +24 -0
  85. de_agent/templates/hub/status.md +23 -0
  86. de_agent-0.2.0.dist-info/METADATA +94 -0
  87. de_agent-0.2.0.dist-info/RECORD +91 -0
  88. de_agent-0.2.0.dist-info/WHEEL +4 -0
  89. de_agent-0.2.0.dist-info/entry_points.txt +2 -0
  90. de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
  91. de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/clients.py ADDED
@@ -0,0 +1,74 @@
1
+ """The ONE list of database clients the guards know about. Stdlib only, Python 3.9 compatible (hooks import it).
2
+
3
+ `extract.py` (sql_guard, query_log) reads SQL out of these clients' command lines; `secret_guard.py` uses the same set to
4
+ spot inline passwords. Add a client here and both layers pick it up.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ # executable basename -> sqlglot dialect used to classify the SQL it runs
9
+ SQL_CLIENTS = {
10
+ "psql": "postgres",
11
+ "mysql": "mysql",
12
+ "mariadb": "mysql",
13
+ "snowsql": "snowflake",
14
+ "sqlcmd": "tsql",
15
+ "clickhouse-client": "clickhouse",
16
+ "clickhouse-local": "clickhouse",
17
+ "sqlite3": "sqlite",
18
+ "duckdb": "duckdb",
19
+ "trino": "trino",
20
+ "trino-cli": "trino",
21
+ "presto": "presto",
22
+ "presto-cli": "presto",
23
+ "spark-sql": "spark",
24
+ "beeline": "hive",
25
+ }
26
+
27
+ # clients with their own non-generic argument handling in extract.py (dialects come from the source name)
28
+ CLOUD_CLIENTS = {"bq": "bigquery", "aws": "postgres"}
29
+
30
+ # database tools whose SQL we do not read, but whose command lines can carry credentials
31
+ OTHER_DB_CLIENTS = frozenset({
32
+ "pg_dump", "pg_dumpall", "pg_restore", "mysqldump", "mysqladmin", "redshift-data", "pgcli", "mycli",
33
+ })
34
+
35
+ DB_CLIENTS = frozenset(SQL_CLIENTS) | {"bq"} | OTHER_DB_CLIENTS
36
+
37
+ # cloud / data CLIs whose own SQL (after a `sql` subcommand, or `-e`) `extract.py` reads; the value is the sqlglot dialect
38
+ CLI_SQL_CLIENTS = {"snow": "snowflake", "dbsqlcli": "databricks", "databricks": "databricks"}
39
+
40
+ # CLIs the CLI write guard (`scripts/cli_guard.py`, `cliclass.py`) classifies as read / write / unknown
41
+ GUARDED_CLIS = frozenset({
42
+ "aws", "gcloud", "gsutil", "bq", "databricks", "dbsqlcli", "kubectl", "helm", "terraform", "tofu", "pulumi", "az", "snow",
43
+ "dbt", "airflow", "dagster", "prefect", "kafka-topics", "kafka-configs",
44
+ })
45
+
46
+ # Python database drivers (import name prefixes): code that imports one is scanned for SQL (pyscan.py)
47
+ PY_DRIVERS = (
48
+ "psycopg", "psycopg2", "psycopg_pool", "sqlalchemy", "pymysql", "MySQLdb", "mysql", "mysql.connector",
49
+ "snowflake", "redshift_connector", "pyodbc", "pymssql", "duckdb", "sqlite3", "clickhouse_driver",
50
+ "clickhouse_connect", "asyncpg", "pg8000", "oracledb", "cx_Oracle", "pyhive", "trino", "databricks",
51
+ )
52
+ PY_DRIVER_DIALECT = {
53
+ "psycopg": "postgres", "psycopg2": "postgres", "psycopg_pool": "postgres", "asyncpg": "postgres",
54
+ "pg8000": "postgres", "sqlalchemy": "postgres", "pymysql": "mysql", "MySQLdb": "mysql", "mysql": "mysql",
55
+ "snowflake": "snowflake", "redshift_connector": "redshift", "pyodbc": "tsql", "pymssql": "tsql",
56
+ "duckdb": "duckdb", "sqlite3": "sqlite", "clickhouse_driver": "clickhouse", "clickhouse_connect": "clickhouse",
57
+ "pyhive": "hive", "trino": "trino", "oracledb": "oracle", "cx_Oracle": "oracle", "databricks": "spark",
58
+ }
59
+
60
+ _SOURCE_DIALECTS = {"redshift-data": "redshift", "bq": "bigquery", "athena": "athena"}
61
+ _SOURCE_DIALECTS.update(SQL_CLIENTS)
62
+ _SOURCE_DIALECTS.update(CLI_SQL_CLIENTS)
63
+ _SUFFIXES = ("-file", "-heredoc", "-herestring", "-stdin")
64
+
65
+
66
+ def dialect_for(source: str) -> str:
67
+ """sqlglot dialect for an extract_sql source tag (`mysql`, `sqlite3-file`, `python:duckdb`, `redshift-data`, ...)."""
68
+ if source.startswith("python:"):
69
+ return source[len("python:"):] or "postgres"
70
+ for suf in _SUFFIXES:
71
+ if source.endswith(suf):
72
+ source = source[: -len(suf)]
73
+ break
74
+ return _SOURCE_DIALECTS.get(source, "postgres")
@@ -0,0 +1,31 @@
1
+ """Collector registry: {type: Collector}."""
2
+ from __future__ import annotations
3
+
4
+ from de_agent.collectors.airflow import AirflowCollector
5
+ from de_agent.collectors.aws import AwsCollector
6
+ from de_agent.collectors.base import Collector, CollectorError, RunContext # noqa: F401
7
+ from de_agent.collectors.dbt import DbtCollector
8
+ from de_agent.collectors.pack_cli import CliPackCollector
9
+ from de_agent.collectors.pack_http import HttpPackCollector
10
+ from de_agent.collectors.pack_sql import SqlPackCollector
11
+ from de_agent.collectors.postgres import PostgresCollector
12
+ from de_agent.collectors.redshift import RedshiftCollector
13
+
14
+ REGISTRY: dict[str, Collector] = {
15
+ PostgresCollector.type: PostgresCollector(),
16
+ AirflowCollector.type: AirflowCollector(),
17
+ DbtCollector.type: DbtCollector(),
18
+ RedshiftCollector.type: RedshiftCollector(),
19
+ AwsCollector.type: AwsCollector(),
20
+ SqlPackCollector.type: SqlPackCollector(),
21
+ CliPackCollector.type: CliPackCollector(),
22
+ HttpPackCollector.type: HttpPackCollector(),
23
+ }
24
+
25
+
26
+ def registered_types() -> list[str]:
27
+ return sorted(REGISTRY)
28
+
29
+
30
+ def get_collector(ctype: str) -> Collector:
31
+ return REGISTRY[ctype]
@@ -0,0 +1,343 @@
1
+ """Airflow REST gathering, stdlib only (no de-agent imports).
2
+
3
+ This file is executed in two places: imported locally by collectors/airflow.py, and shipped as text over
4
+ `ssh <host> python3 -` for `exec: ssh:<host>` sources. Keep it Python 3.8+ and stdlib-only.
5
+ `get(path, params, accept)` returns (status, body); it is the only thing that touches the network, so it is
6
+ GET-only by construction and replaced by a replay function in tests.
7
+ """
8
+ import json
9
+ import re
10
+ from datetime import datetime, timedelta, timezone
11
+ from urllib.parse import quote
12
+
13
+ LOG_TAIL_LINES = 40
14
+ STACK_LINES = 5
15
+ MAX_FAILURES = 50
16
+ MAX_LOG_FETCHES = 20
17
+ PAGE = 100
18
+
19
+ _REDACT = [
20
+ (re.compile(r"(\b[a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s]+(@)", re.I), r"\1***\2"),
21
+ (re.compile(r"(\b(?:password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*)[^\s'\",&]+", re.I), r"\1***"),
22
+ (re.compile(r"\b((?:PGPASSWORD|MYSQL_PWD)=)\S+"), r"\1***"),
23
+ (re.compile(r"(\bAWS_SECRET_ACCESS_KEY\s*[=:]\s*[\"']?)[A-Za-z0-9/+=]{20,}"), r"\1***"),
24
+ (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "***"),
25
+ (re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]{8,}", re.I), r"\1***"),
26
+ (re.compile(r"(\bAuthorization:\s*(?:Basic|Bearer)\s+)\S+", re.I), r"\1***"),
27
+ ]
28
+
29
+
30
+ def redact_text(text):
31
+ for rx, rep in _REDACT:
32
+ text = rx.sub(rep, text)
33
+ return text
34
+
35
+
36
+ def _parse_ts(v):
37
+ if not v or not isinstance(v, str):
38
+ return None
39
+ try:
40
+ return datetime.fromisoformat(v.replace("Z", "+00:00")).astimezone(timezone.utc)
41
+ except ValueError:
42
+ return None
43
+
44
+
45
+ def _iso(dt):
46
+ return dt.strftime("%Y-%m-%dT%H:%M:%SZ") if dt else None
47
+
48
+
49
+ class ApiError(Exception):
50
+ pass
51
+
52
+
53
+ def _json(get, path, params=None):
54
+ status, body = get(path, params, "application/json")
55
+ if status != 200 or not isinstance(body, (dict, list)):
56
+ raise ApiError("GET %s -> HTTP %s" % (path, status))
57
+ return body
58
+
59
+
60
+ def _paged(get, path, key, params, cap):
61
+ out, offset = [], 0
62
+ while len(out) < cap:
63
+ body = _json(get, path, list(params) + [("limit", PAGE), ("offset", offset)])
64
+ rows = body.get(key) or []
65
+ out += rows
66
+ total = body.get("total_entries")
67
+ offset += PAGE
68
+ if not rows or (total is not None and offset >= total):
69
+ break
70
+ return out[:cap]
71
+
72
+
73
+ def detect_api(get):
74
+ net_error = None
75
+ for base in ("/api/v2", "/api/v1"):
76
+ try:
77
+ status, body = get(base + "/version", None, "application/json")
78
+ except ApiError as ex: # network failure: remember it, it is a better message than "no API found"
79
+ net_error = ex
80
+ continue
81
+ if status == 200 and isinstance(body, dict) and body.get("version"):
82
+ return base, str(body["version"])
83
+ if net_error is not None:
84
+ raise net_error
85
+ raise ApiError("no Airflow API found at /api/v2/version or /api/v1/version")
86
+
87
+
88
+ # ---- normalisers (Airflow 2.x /api/v1 and 3.x /api/v2 differ in field names) --------------------------
89
+
90
+ def _dag_row(d):
91
+ si = d.get("schedule_interval")
92
+ expr = si.get("value") if isinstance(si, dict) else (si if isinstance(si, str) else None)
93
+ expr = expr or d.get("timetable_summary")
94
+ nxt = d.get("next_dagrun_create_after") or d.get("next_dagrun_run_after") or d.get("next_dagrun")
95
+ return {
96
+ "dag_id": d.get("dag_id"),
97
+ "is_paused": bool(d.get("is_paused")),
98
+ "schedule": expr,
99
+ "schedule_desc": d.get("timetable_description"),
100
+ "tags": sorted(t.get("name") if isinstance(t, dict) else str(t) for t in d.get("tags") or []),
101
+ "owners": ",".join(d.get("owners") or []) if isinstance(d.get("owners"), list) else d.get("owners"),
102
+ "fileloc": d.get("fileloc"),
103
+ "next_dagrun": nxt,
104
+ "has_import_errors": bool(d.get("has_import_errors")),
105
+ }
106
+
107
+
108
+ def _is_live(d):
109
+ if "is_stale" in d:
110
+ return not d["is_stale"]
111
+ return d.get("is_active", True) is not False
112
+
113
+
114
+ def _run_row(dag_id, r):
115
+ start, end = _parse_ts(r.get("start_date")), _parse_ts(r.get("end_date"))
116
+ return {
117
+ "dag_id": dag_id,
118
+ "run_id": r.get("dag_run_id"),
119
+ "state": r.get("state"),
120
+ "logical_date": r.get("logical_date") or r.get("execution_date"),
121
+ "start": _iso(start),
122
+ "end": _iso(end),
123
+ "duration_s": int((end - start).total_seconds()) if start and end else None,
124
+ }
125
+
126
+
127
+ def _log_lines(body):
128
+ if isinstance(body, str):
129
+ return body.splitlines()
130
+ if isinstance(body, dict):
131
+ content = body.get("content")
132
+ if isinstance(content, str):
133
+ return content.splitlines()
134
+ lines = []
135
+ for item in content or []:
136
+ if isinstance(item, dict):
137
+ lines.append(str(item.get("event", item)))
138
+ elif isinstance(item, (list, tuple)) and item:
139
+ lines.append(str(item[-1]))
140
+ else:
141
+ lines.append(str(item))
142
+ return lines
143
+ return []
144
+
145
+
146
+ def _log_tail(get, api, ti):
147
+ path = "%s/dags/%s/dagRuns/%s/taskInstances/%s/logs/%s" % (
148
+ api, quote(ti["dag_id"], safe=""), quote(ti["dag_run_id"], safe=""),
149
+ quote(ti["task_id"], safe=""), int(ti.get("try_number") or 1))
150
+ status, body = get(path, [("full_content", "true")], "text/plain")
151
+ if status != 200:
152
+ raise ApiError("log HTTP %s" % status)
153
+ lines = [ln.rstrip() for ln in _log_lines(body) if ln.strip()]
154
+ return redact_text("\n".join(lines[-LOG_TAIL_LINES:]))
155
+
156
+
157
+ def schedule_interval_s(schedule, runs):
158
+ """Best-effort interval in seconds: cron/preset heuristics, else the median gap of scheduled runs."""
159
+ s = (schedule or "").strip()
160
+ presets = {"@hourly": 3600, "@daily": 86400, "@weekly": 604800, "@monthly": 2592000, "@yearly": 31536000,
161
+ "@once": None, "@continuous": None}
162
+ if s in presets:
163
+ return presets[s]
164
+ parts = s.split()
165
+ if len(parts) == 5:
166
+ minute, hour, dom, mon, dow = parts
167
+ if hour == "*" and dom == mon == dow == "*" and not minute.startswith("*/"):
168
+ return 3600
169
+ if minute.startswith("*/") and minute[2:].isdigit():
170
+ return int(minute[2:]) * 60
171
+ if hour.isdigit() and dom == mon == dow == "*":
172
+ return 86400
173
+ if dow != "*" and dom == mon == "*":
174
+ return 604800
175
+ scheduled = sorted(t for t in (_parse_ts(r.get("logical_date")) for r in runs
176
+ if str(r.get("run_id", "")).startswith("scheduled__")) if t)
177
+ gaps = sorted((b - a).total_seconds() for a, b in zip(scheduled, scheduled[1:]))
178
+ return int(gaps[len(gaps) // 2]) if gaps else None
179
+
180
+
181
+ def gather(cfg, get, now=None):
182
+ """Return {"data": {...}, "errors": [...]}; a failing section is reported, the others still return."""
183
+ now = now or datetime.now(timezone.utc)
184
+ max_dags = int(cfg.get("max_dags", 500))
185
+ runs_per_dag = int(cfg.get("runs_per_dag", 5))
186
+ fetch_logs = bool(cfg.get("fetch_logs", True))
187
+ data, errors = {}, []
188
+ api, version = detect_api(get)
189
+ data["version"] = {"version": version, "api": api.rsplit("/", 1)[1]}
190
+ is_v2 = api.endswith("v2")
191
+
192
+ def section(name, fn):
193
+ try:
194
+ data[name] = fn()
195
+ except ApiError as ex:
196
+ errors.append("%s: %s" % (name, ex))
197
+
198
+ def dags():
199
+ raw = _paged(get, api + "/dags", "dags", [("order_by", "dag_id")], max_dags)
200
+ return [_dag_row(d) for d in raw if _is_live(d)]
201
+
202
+ section("dags", dags)
203
+ dag_ids = [d["dag_id"] for d in data.get("dags", [])]
204
+
205
+ def runs():
206
+ out = []
207
+ for dag_id in dag_ids:
208
+ body = _json(get, "%s/dags/%s/dagRuns" % (api, quote(dag_id, safe="")),
209
+ [("order_by", "-start_date"), ("limit", runs_per_dag)])
210
+ out += [_run_row(dag_id, r) for r in body.get("dag_runs") or []]
211
+ return out
212
+
213
+ if "dags" in data:
214
+ section("runs", runs)
215
+
216
+ def failures():
217
+ since = _iso(now - timedelta(hours=24))
218
+ tis = _paged(get, api + "/dags/~/dagRuns/~/taskInstances", "task_instances",
219
+ [("state", "failed"), ("state", "up_for_retry"), ("start_date_gte", since)], MAX_FAILURES)
220
+ tis.sort(key=lambda t: str(t.get("start_date")), reverse=True) # the endpoint has no order_by
221
+ out = []
222
+ for i, ti in enumerate(tis):
223
+ row = {"dag_id": ti.get("dag_id"), "task_id": ti.get("task_id"), "try": int(ti.get("try_number") or 1),
224
+ "run_id": ti.get("dag_run_id"), "state": ti.get("state"), "start": ti.get("start_date"),
225
+ "duration_s": int(ti["duration"]) if isinstance(ti.get("duration"), (int, float)) else None}
226
+ if fetch_logs and i < MAX_LOG_FETCHES:
227
+ try:
228
+ row["log_tail"] = _log_tail(get, api, ti)
229
+ except ApiError as ex:
230
+ row["log_tail"] = "(log unavailable: %s)" % ex
231
+ out.append(row)
232
+ return out
233
+
234
+ section("failures_24h", failures)
235
+
236
+ def import_errors():
237
+ rows = _paged(get, api + "/importErrors", "import_errors", [], 100)
238
+ return [{"filename": r.get("filename"),
239
+ "stack": "\n".join((r.get("stack_trace") or "").splitlines()[:STACK_LINES])} for r in rows]
240
+
241
+ section("import_errors", import_errors)
242
+
243
+ def datasets():
244
+ key = "assets" if is_v2 else "datasets"
245
+ rows = _paged(get, "%s/%s" % (api, key), key, [], 200)
246
+ out = []
247
+ for r in rows:
248
+ last = None
249
+ le = r.get("last_asset_event") or r.get("last_dataset_event")
250
+ if isinstance(le, dict):
251
+ last = le.get("timestamp")
252
+ if last is None:
253
+ ev = "assets/events" if is_v2 else "datasets/events"
254
+ idk = "asset_id" if is_v2 else "dataset_id"
255
+ evk = "asset_events" if is_v2 else "dataset_events"
256
+ try:
257
+ body = _json(get, "%s/%s" % (api, ev), [(idk, r.get("id")), ("order_by", "-timestamp"), ("limit", 1)])
258
+ evs = body.get(evk) or []
259
+ last = evs[0].get("timestamp") if evs else None
260
+ except ApiError:
261
+ last = None
262
+ prod = sorted("%s.%s" % (t.get("dag_id"), t.get("task_id")) for t in r.get("producing_tasks") or [])
263
+ cons = sorted(set(c.get("dag_id") for c in (r.get("consuming_dags") or []) if isinstance(c, dict)))
264
+ out.append({"uri": r.get("uri") or r.get("name"), "producing_tasks": prod, "consuming_dags": cons,
265
+ "last_event": _iso(_parse_ts(last))})
266
+ return out
267
+
268
+ section("datasets", datasets)
269
+
270
+ def pools():
271
+ body = _json(get, api + "/pools", [("limit", PAGE)])
272
+ return [{"name": p.get("name"), "slots": p.get("slots"), "occupied": p.get("occupied_slots"),
273
+ "queued": p.get("queued_slots")} for p in body.get("pools") or []]
274
+
275
+ section("pools", pools)
276
+ data["meta"] = {"collected_at": _iso(now), "runs_per_dag": runs_per_dag, "max_dags": max_dags,
277
+ "fetch_logs": fetch_logs}
278
+ return {"data": data, "errors": errors}
279
+
280
+
281
+ # ---- remote entry point (exec: ssh:<host>) ------------------------------------------------------------
282
+
283
+ def resolve_remote_secret(ref):
284
+ import os
285
+ if ref.startswith("env:"):
286
+ val = os.environ.get(ref[4:])
287
+ if val is None:
288
+ raise ApiError("environment variable %s is not set on the remote host" % ref[4:])
289
+ return val
290
+ if ref.startswith("file:"):
291
+ try:
292
+ with open(ref[5:]) as fh:
293
+ return fh.read().strip()
294
+ except OSError as ex:
295
+ raise ApiError("cannot read %s on the remote host: %s" % (ref[5:], ex.strerror)) from None
296
+ raise ApiError("on a remote host the secret ref must be env: or file:")
297
+
298
+
299
+ def make_urllib_get(base_url, auth_header, timeout=20):
300
+ import urllib.error
301
+ import urllib.request
302
+
303
+ def get(path, params=None, accept="application/json"):
304
+ from urllib.parse import urlencode
305
+ url = base_url.rstrip("/") + path + ("?" + urlencode(params) if params else "")
306
+ req = urllib.request.Request(url, method="GET")
307
+ req.add_header("Accept", accept)
308
+ if auth_header:
309
+ req.add_header("Authorization", auth_header)
310
+ try:
311
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
312
+ raw, status, ctype = resp.read(), resp.status, resp.headers.get("Content-Type", "")
313
+ except urllib.error.HTTPError as ex:
314
+ raw, status, ctype = ex.read(), ex.code, ex.headers.get("Content-Type", "")
315
+ except (urllib.error.URLError, OSError) as ex:
316
+ raise ApiError("cannot reach %s: %s" % (base_url, getattr(ex, "reason", ex))) from None
317
+ text = raw.decode("utf-8", "replace")
318
+ if "json" in ctype:
319
+ try:
320
+ return status, json.loads(text)
321
+ except ValueError:
322
+ return status, text
323
+ return status, text
324
+
325
+ return get
326
+
327
+
328
+ def auth_header_for(cfg, secret):
329
+ import base64
330
+ if cfg.get("auth") == "token":
331
+ return "Bearer " + secret
332
+ return "Basic " + base64.b64encode(("%s:%s" % (cfg.get("user", ""), secret)).encode()).decode()
333
+
334
+
335
+ def remote_main(cfg):
336
+ """Entry point used on the remote host: resolve the secret THERE, call the API, return the result dict."""
337
+ try:
338
+ ref = cfg.get("token") if cfg.get("auth") == "token" else cfg.get("password")
339
+ secret = resolve_remote_secret(ref) if ref else None
340
+ get = make_urllib_get(cfg["base_url"], auth_header_for(cfg, secret) if secret else None)
341
+ return gather(cfg, get)
342
+ except ApiError as ex:
343
+ return {"data": {}, "errors": [str(ex)]}
@@ -0,0 +1,219 @@
1
+ """HTTP fetching for `kind: http` packs, stdlib only (no de-agent imports).
2
+
3
+ Executed in two places: imported locally by collectors/pack_http.py, and shipped as text over `ssh <host> python3 -` for
4
+ `exec: ssh:<host>` sources (secrets are then resolved on the remote host from env:/file: refs). Keep it Python 3.8+.
5
+
6
+ `request()` is the ONLY function that touches the network, and it is read-only by construction: GET, or POST of a GraphQL
7
+ document that contains no mutation or subscription. Anything else raises ValueError before a socket is opened. Redirects are not
8
+ followed (an Authorization header must never travel to another host). Responses are capped at 5 MB.
9
+ """
10
+ import base64
11
+ import json
12
+ import os
13
+ import re
14
+ import stat
15
+ import urllib.error
16
+ import urllib.request
17
+ from urllib.parse import urlsplit
18
+
19
+ MAX_BYTES = 5 * 1024 * 1024
20
+ MAX_GRAPHQL = 10000
21
+
22
+ _REDACT = [
23
+ (re.compile(r"(\b[a-z][a-z0-9+.-]*://[^:/\s@]+:)[^@\s]+(@)", re.I), r"\1***\2"),
24
+ (re.compile(r"(\b(?:password|passwd|pwd|secret|token|api[_-]?key)\s*[=:]\s*)[^\s'\",&]+", re.I), r"\1***"),
25
+ (re.compile(r"(\bBearer\s+)[A-Za-z0-9._~+/=-]{8,}", re.I), r"\1***"),
26
+ (re.compile(r"(\bAuthorization:\s*(?:Basic|Bearer)\s+)\S+", re.I), r"\1***"),
27
+ ]
28
+
29
+
30
+ def redact_text(text):
31
+ for rx, rep in _REDACT:
32
+ text = rx.sub(rep, text)
33
+ return text
34
+
35
+
36
+ class _NoRedirect(urllib.request.HTTPRedirectHandler):
37
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
38
+ return None
39
+
40
+
41
+ _OPENER = urllib.request.build_opener(_NoRedirect)
42
+
43
+ _WRITE_WORDS = re.compile(r"\b(mutation|subscription)\b")
44
+
45
+
46
+ def _graphql_bare(q):
47
+ """`q` with comments blanked and string / block-string contents emptied, scanned left to right so that a `#` inside a
48
+ string and a quote inside a comment are each read correctly. None when a string is not terminated."""
49
+ out = []
50
+ i, n = 0, len(q)
51
+ while i < n:
52
+ c = q[i]
53
+ if c == "#":
54
+ j = q.find("\n", i)
55
+ i = n if j == -1 else j
56
+ out.append(" ")
57
+ elif q.startswith('"""', i):
58
+ j = i + 3
59
+ while j < n and not q.startswith('"""', j):
60
+ j += 4 if q.startswith('\\"""', j) else 1
61
+ if j >= n:
62
+ return None
63
+ out.append('""')
64
+ i = j + 3
65
+ elif c == '"':
66
+ j = i + 1
67
+ while j < n and q[j] not in '"\n':
68
+ j += 2 if q[j] == "\\" else 1
69
+ if j >= n or q[j] != '"':
70
+ return None
71
+ out.append('""')
72
+ i = j + 1
73
+ else:
74
+ out.append(c)
75
+ i += 1
76
+ return "".join(out)
77
+
78
+
79
+ def graphql_refusal(query):
80
+ """None when `query` is a read-only GraphQL document (starts with `query` or `{`; no mutation/subscription anywhere)."""
81
+ if not isinstance(query, str) or not query.strip():
82
+ return "graphql query is empty"
83
+ if len(query) > MAX_GRAPHQL:
84
+ return "graphql query is too long"
85
+ bare = _graphql_bare(query)
86
+ if bare is None:
87
+ return "graphql query has a string that is not terminated"
88
+ head = bare.lstrip()
89
+ if not (head.startswith("{") or re.match(r"query\b", head)):
90
+ return "a graphql document must start with `query` or `{` (never mutation or subscription)"
91
+ if _WRITE_WORDS.search(bare):
92
+ return "graphql documents may not contain mutation or subscription"
93
+ return None
94
+
95
+
96
+ def path_refusal(path):
97
+ """None when `path` is a relative request path (`/v1/things?limit=5`) that stays on the source's own host."""
98
+ if not isinstance(path, str) or not path:
99
+ return "path is empty"
100
+ if not path.startswith("/") or path.startswith("//"):
101
+ return "path must start with a single '/' (it is relative to the source's base_url)"
102
+ if re.search(r"[\s\x00-\x1f\\]", path) or "://" in path:
103
+ return "path may not contain whitespace, backslashes or a scheme"
104
+ if ".." in path.split("?", 1)[0].split("/"):
105
+ return "path may not contain '..' segments"
106
+ return None
107
+
108
+
109
+ def build_auth_header(auth, user, secret, header_name):
110
+ """(header name, value) or None."""
111
+ if auth == "none" or secret is None:
112
+ return None
113
+ if auth == "bearer":
114
+ return ("Authorization", "Bearer " + secret)
115
+ if auth == "basic":
116
+ raw = ("%s:%s" % (user or "", secret)).encode("utf-8")
117
+ return ("Authorization", "Basic " + base64.b64encode(raw).decode("ascii"))
118
+ if auth == "header":
119
+ return (header_name or "X-Api-Key", secret)
120
+ raise ValueError("unknown auth %r" % (auth,))
121
+
122
+
123
+ def request(base_url, method, path, headers, timeout, graphql_query=None):
124
+ """One read-only request. Returns (status, parsed JSON body). Raises ValueError (refused) or IOError (failed)."""
125
+ why = path_refusal(path)
126
+ if why:
127
+ raise ValueError(why)
128
+ if method == "GET":
129
+ if graphql_query is not None:
130
+ raise ValueError("GET does not carry a graphql document")
131
+ data = None
132
+ elif method == "POST":
133
+ why = graphql_refusal(graphql_query)
134
+ if why:
135
+ raise ValueError(why)
136
+ data = json.dumps({"query": graphql_query}).encode("utf-8")
137
+ else:
138
+ raise ValueError("only GET and read-only GraphQL POST are allowed (got %s)" % (method,))
139
+ base = urlsplit(base_url)
140
+ if base.scheme not in ("http", "https") or not base.netloc:
141
+ raise ValueError("base_url must be an http(s) URL")
142
+ url = base_url.rstrip("/") + path
143
+ if urlsplit(url).netloc != base.netloc:
144
+ raise ValueError("request would leave the source's host")
145
+ req = urllib.request.Request(url, data=data, method=method)
146
+ req.add_header("Accept", "application/json")
147
+ if data is not None:
148
+ req.add_header("Content-Type", "application/json")
149
+ for k, v in (headers or {}).items():
150
+ req.add_header(k, v)
151
+ try:
152
+ resp = _OPENER.open(req, timeout=timeout)
153
+ except urllib.error.HTTPError as ex:
154
+ return ex.code, None
155
+ except (urllib.error.URLError, OSError, ValueError) as ex:
156
+ raise OSError(redact_text("request failed: %s" % (getattr(ex, "reason", None) or type(ex).__name__))) from None
157
+ try:
158
+ raw = resp.read(MAX_BYTES + 1)
159
+ status = getattr(resp, "status", 200)
160
+ finally:
161
+ resp.close()
162
+ if len(raw) > MAX_BYTES:
163
+ raise OSError("response larger than 5 MB")
164
+ try:
165
+ return status, json.loads(raw.decode("utf-8"))
166
+ except ValueError:
167
+ raise OSError("response was not JSON") from None
168
+
169
+
170
+ def fetch(base_url, reqs, headers, timeout):
171
+ """reqs: [{"kind": "get"|"graphql", "path": str, "query": str|None}] -> [{"ok": True, "body": ...} | {"ok": False, "error": str}]."""
172
+ out = []
173
+ for r in reqs:
174
+ try:
175
+ if r.get("kind") == "get":
176
+ status, body = request(base_url, "GET", r["path"], headers, timeout)
177
+ elif r.get("kind") == "graphql":
178
+ status, body = request(base_url, "POST", r["path"], headers, timeout, r.get("query"))
179
+ else:
180
+ raise ValueError("only GET and read-only GraphQL POST are allowed")
181
+ except (OSError, ValueError) as ex:
182
+ out.append({"ok": False, "error": redact_text(str(ex))})
183
+ continue
184
+ if status is None or status >= 300 or body is None:
185
+ out.append({"ok": False, "error": "HTTP %s" % (status,)})
186
+ else:
187
+ out.append({"ok": True, "body": body})
188
+ return out
189
+
190
+
191
+ def _resolve_remote(ref):
192
+ if ref is None or ref == "none":
193
+ return None
194
+ if ref.startswith("env:"):
195
+ val = os.environ.get(ref[4:])
196
+ if val is None:
197
+ raise ValueError("environment variable %s is not set on the remote host" % ref[4:])
198
+ return val
199
+ if ref.startswith("file:"):
200
+ path = ref[5:]
201
+ try:
202
+ if stat.S_IMODE(os.stat(path).st_mode) & 0o077:
203
+ raise ValueError("secret file %s must be mode 0600 or stricter" % path)
204
+ with open(path) as f:
205
+ return f.read().strip()
206
+ except OSError as ex:
207
+ raise ValueError("cannot read secret file %s: %s" % (path, ex.strerror)) from None
208
+ raise ValueError("secret refs over ssh must be env: or file: (resolved on the remote host)")
209
+
210
+
211
+ def remote_main(cfg):
212
+ """cfg: {base_url, auth, user, secret_ref, header_name, timeout, requests}. Secrets are resolved HERE, on the remote host."""
213
+ try:
214
+ secret = _resolve_remote(cfg.get("secret_ref"))
215
+ hdr = build_auth_header(cfg.get("auth", "none"), cfg.get("user"), secret, cfg.get("header_name"))
216
+ except ValueError as ex:
217
+ return {"error": redact_text(str(ex))}
218
+ headers = {hdr[0]: hdr[1]} if hdr else {}
219
+ return {"results": fetch(cfg["base_url"], cfg["requests"], headers, cfg.get("timeout", 20))}