de-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_agent/__init__.py +1 -0
- de_agent/applyplan.py +270 -0
- de_agent/approval.py +110 -0
- de_agent/cli.py +650 -0
- de_agent/cliclass.py +1651 -0
- de_agent/clients.py +74 -0
- de_agent/collectors/__init__.py +31 -0
- de_agent/collectors/_airflow_core.py +343 -0
- de_agent/collectors/_pack_http_core.py +219 -0
- de_agent/collectors/airflow.py +125 -0
- de_agent/collectors/aws/__init__.py +172 -0
- de_agent/collectors/aws/_aws_core.py +823 -0
- de_agent/collectors/base.py +66 -0
- de_agent/collectors/dbt.py +198 -0
- de_agent/collectors/pack_base.py +225 -0
- de_agent/collectors/pack_cli.py +93 -0
- de_agent/collectors/pack_http.py +137 -0
- de_agent/collectors/pack_sql.py +217 -0
- de_agent/collectors/postgres.py +285 -0
- de_agent/collectors/redshift.py +815 -0
- de_agent/config.py +450 -0
- de_agent/configdoc.py +115 -0
- de_agent/data/sql_cases.yaml +1735 -0
- de_agent/diff.py +280 -0
- de_agent/extract.py +662 -0
- de_agent/find.py +218 -0
- de_agent/hubpath.py +32 -0
- de_agent/initcmd.py +49 -0
- de_agent/lint.py +297 -0
- de_agent/packs/__init__.py +5 -0
- de_agent/packs/argv.py +269 -0
- de_agent/packs/bigquery.yaml +67 -0
- de_agent/packs/check.py +228 -0
- de_agent/packs/cli.py +147 -0
- de_agent/packs/dagster.yaml +59 -0
- de_agent/packs/databricks-system.yaml +44 -0
- de_agent/packs/databricks.yaml +56 -0
- de_agent/packs/errors.py +6 -0
- de_agent/packs/expr.py +314 -0
- de_agent/packs/fivetran.yaml +30 -0
- de_agent/packs/gcp.yaml +53 -0
- de_agent/packs/kubernetes.yaml +64 -0
- de_agent/packs/loader.py +85 -0
- de_agent/packs/model.py +154 -0
- de_agent/packs/mssql.yaml +61 -0
- de_agent/packs/mysql.yaml +53 -0
- de_agent/packs/postgres-lite.yaml +65 -0
- de_agent/packs/scaffold.py +75 -0
- de_agent/packs/snowflake.yaml +91 -0
- de_agent/packs/template.py +111 -0
- de_agent/packs/view.py +64 -0
- de_agent/plans.py +621 -0
- de_agent/pyscan.py +214 -0
- de_agent/render.py +854 -0
- de_agent/runner.py +66 -0
- de_agent/scanners/__init__.py +1 -0
- de_agent/scanners/airflow.py +691 -0
- de_agent/scanners/catalog.py +205 -0
- de_agent/scanners/common.py +146 -0
- de_agent/scanners/dbt.py +99 -0
- de_agent/scanners/edges.py +358 -0
- de_agent/scanners/headers.py +117 -0
- de_agent/scanners/pycode.py +719 -0
- de_agent/scanners/pyexpr.py +308 -0
- de_agent/scanners/python.py +240 -0
- de_agent/scanners/run.py +288 -0
- de_agent/scanners/sql.py +131 -0
- de_agent/scanners/terraform.py +769 -0
- de_agent/secretscan.py +28 -0
- de_agent/snapshot.py +159 -0
- de_agent/sqlclass.py +333 -0
- de_agent/sqlexec.py +626 -0
- de_agent/stepexec.py +257 -0
- de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
- de_agent/templates/example/map_row.txt +1 -0
- de_agent/templates/example/projects/orders_etl.md +21 -0
- de_agent/templates/hub/README.md +16 -0
- de_agent/templates/hub/conventions.md +15 -0
- de_agent/templates/hub/de-agent.yaml +20 -0
- de_agent/templates/hub/domains/_template.md +10 -0
- de_agent/templates/hub/gitignore.tmpl +1 -0
- de_agent/templates/hub/integrations/_template.md +12 -0
- de_agent/templates/hub/map.md +17 -0
- de_agent/templates/hub/projects/_template.md +24 -0
- de_agent/templates/hub/status.md +23 -0
- de_agent-0.2.0.dist-info/METADATA +94 -0
- de_agent-0.2.0.dist-info/RECORD +91 -0
- de_agent-0.2.0.dist-info/WHEEL +4 -0
- de_agent-0.2.0.dist-info/entry_points.txt +2 -0
- de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
- de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/cliclass.py
ADDED
|
@@ -0,0 +1,1651 @@
|
|
|
1
|
+
"""Classify cloud / data CLI command lines as read, write or unknown (v0.2 section 3). Stdlib only, Python 3.9 compatible.
|
|
2
|
+
|
|
3
|
+
The hook `scripts/cli_guard.py` and the pack runner (`type: cli`) share this module, so one table decides what a "read-only
|
|
4
|
+
command" means. `classify_command` takes a shell command line, follows wrappers (`env`, `sudo`, `bash -c`, `xargs`,
|
|
5
|
+
`find -exec`, `timeout`, `nice`, `uv run`, `poetry run`, `docker exec`, `docker compose exec`, `kubectl exec --`, `ssh`, ...)
|
|
6
|
+
and returns one `Verdict` per guarded CLI invocation found. Kinds:
|
|
7
|
+
|
|
8
|
+
read the command only reads
|
|
9
|
+
write a known mutating command (denied outside `dea apply`)
|
|
10
|
+
unknown a verb of a known CLI that is not on its read list (denied, fail closed)
|
|
11
|
+
sql carries SQL (`aws redshift-data`, `bq query`, `snow sql`, `psql` inside `kubectl exec`): `sql_guard` classifies it
|
|
12
|
+
allow matched an entry of `guard.cli_allow` (exact word prefix)
|
|
13
|
+
|
|
14
|
+
It is a pattern matcher over command text, not a sandbox: variables used as the program name, `eval` of built strings, base64 and
|
|
15
|
+
scripts it cannot read are out of scope (docs/SAFETY.md).
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import ast
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import shlex
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
|
|
25
|
+
from de_agent.clients import DB_CLIENTS, GUARDED_CLIS
|
|
26
|
+
|
|
27
|
+
MAX_DEPTH = 4
|
|
28
|
+
MAX_SCRIPT = 200_000
|
|
29
|
+
PLACEHOLDER = re.compile(r"__DEA_HEREDOC_(\d+)__")
|
|
30
|
+
UNPARSEABLE = "de-agent: cannot parse this command line to check the CLI calls in it; split it into simple commands"
|
|
31
|
+
PIPED_SHELL = ("de-agent: shell code piped on stdin cannot be checked for CLI writes; write the commands out, or put them "
|
|
32
|
+
"in a plan as a shell step")
|
|
33
|
+
UNKNOWN_MSG = "not a recognised read-only command; put it in a plan as a shell step"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Verdict:
|
|
38
|
+
kind: str # read | write | unknown | sql | allow | unparseable
|
|
39
|
+
cli: str
|
|
40
|
+
verb: str
|
|
41
|
+
reason: str = ""
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def denied(self) -> bool:
|
|
45
|
+
return self.kind in ("write", "unknown", "unparseable")
|
|
46
|
+
|
|
47
|
+
def message(self) -> str:
|
|
48
|
+
label = f"{self.cli} {self.verb}".strip()
|
|
49
|
+
if self.kind == "write":
|
|
50
|
+
return (f"de-agent: `{label}` is a write; CLI writes run only as an approved plan shell step "
|
|
51
|
+
f"(put it in a plan; `dea apply` runs it after human approval){': ' + self.reason if self.reason else ''}")
|
|
52
|
+
if self.kind == "unknown":
|
|
53
|
+
return f"de-agent: `{label}`: {UNKNOWN_MSG}{' (' + self.reason + ')' if self.reason else ''}"
|
|
54
|
+
return self.reason or UNPARSEABLE
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _r(cli: str, verb: str = "") -> Verdict:
|
|
58
|
+
return Verdict("read", cli, verb)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _w(cli: str, verb: str, why: str = "") -> Verdict:
|
|
62
|
+
return Verdict("write", cli, verb, why)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _u(cli: str, verb: str, why: str = "") -> Verdict:
|
|
66
|
+
return Verdict("unknown", cli, verb, why)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _s(cli: str, verb: str) -> Verdict:
|
|
70
|
+
return Verdict("sql", cli, verb)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------- argument helpers
|
|
74
|
+
|
|
75
|
+
def scan(args: list[str], valued: frozenset[str] | set[str] = frozenset()) -> tuple[list[str], list[str]]:
|
|
76
|
+
"""(positionals, flags). A flag listed in `valued` (as `--name` or `-n`) consumes the next word unless written `--name=value`."""
|
|
77
|
+
pos: list[str] = []
|
|
78
|
+
flags: list[str] = []
|
|
79
|
+
i = 0
|
|
80
|
+
while i < len(args):
|
|
81
|
+
a = args[i]
|
|
82
|
+
if a == "--":
|
|
83
|
+
pos.extend(args[i + 1:])
|
|
84
|
+
break
|
|
85
|
+
if a.startswith("--") or (a.startswith("-") and len(a) > 1 and not a[1:].isdigit()):
|
|
86
|
+
flags.append(a)
|
|
87
|
+
if "=" not in a and a in valued and i + 1 < len(args):
|
|
88
|
+
i += 1
|
|
89
|
+
else:
|
|
90
|
+
pos.append(a)
|
|
91
|
+
i += 1
|
|
92
|
+
return pos, flags
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _has_help(args: list[str]) -> bool:
|
|
96
|
+
return "--help" in args
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _prefix(t: str, prefixes: tuple[str, ...]) -> bool:
|
|
100
|
+
return t.startswith(prefixes)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _base(w: str) -> str:
|
|
104
|
+
return os.path.basename(w)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _path_verb(pos: list[str], reads: set[str], writes: set[str], rp: tuple[str, ...], wp: tuple[str, ...],
|
|
108
|
+
start: int = 1, limit: int = 8) -> tuple[str, str] | None:
|
|
109
|
+
"""First token of the command path (after the service group) that is a known read or write verb."""
|
|
110
|
+
for i in range(start, min(len(pos), limit)):
|
|
111
|
+
t = pos[i]
|
|
112
|
+
if t in writes or (wp and t.startswith(wp)):
|
|
113
|
+
return "write", t
|
|
114
|
+
if t in reads or (rp and t.startswith(rp)):
|
|
115
|
+
return "read", t
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _label(parts: list[str], n: int = 4) -> str:
|
|
120
|
+
return " ".join(parts[:n])
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# ---------------------------------------------------------------- aws
|
|
124
|
+
|
|
125
|
+
_AWS_VALUED = {"--profile", "--region", "--output", "--endpoint-url", "--query", "--ca-bundle", "--cli-read-timeout",
|
|
126
|
+
"--cli-connect-timeout", "--color", "--cli-binary-format", "--cli-input-json", "--cli-input-yaml",
|
|
127
|
+
"--cli-auth-type"}
|
|
128
|
+
_S3_VALUED = _AWS_VALUED | {"--exclude", "--include", "--acl", "--storage-class", "--sse", "--sse-kms-key-id", "--sse-c",
|
|
129
|
+
"--sse-c-key", "--sse-c-copy-source", "--sse-c-copy-source-key", "--content-type", "--cache-control",
|
|
130
|
+
"--expires", "--metadata", "--metadata-directive", "--grants", "--website-redirect",
|
|
131
|
+
"--page-size", "--request-payer", "--source-region", "--expected-size", "--content-disposition",
|
|
132
|
+
"--content-encoding", "--content-language", "--copy-props", "--checksum-algorithm",
|
|
133
|
+
"--checksum-mode", "--bucket", "--key"}
|
|
134
|
+
_AWS_READ_PREFIX = ("describe-", "list-", "get-", "head-", "batch-get-", "lookup-")
|
|
135
|
+
_AWS_WRITE_PREFIX = ("start-", "stop-", "create-", "delete-", "update-", "put-", "modify-", "run-", "terminate-", "reboot-")
|
|
136
|
+
_AWS_EXTRA_READ = {("logs", "tail"), ("logs", "filter-log-events"), ("dynamodb", "query"), ("dynamodb", "scan"),
|
|
137
|
+
("s3", "presign"), ("cloudformation", "validate-template"), ("sts", "decode-authorization-message")}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _aws(args: list[str]) -> Verdict:
|
|
141
|
+
pos, flags = scan(args, _S3_VALUED)
|
|
142
|
+
if not pos:
|
|
143
|
+
return _r("aws")
|
|
144
|
+
svc = pos[0]
|
|
145
|
+
if svc in ("help", "version"):
|
|
146
|
+
return _r("aws", svc)
|
|
147
|
+
if len(pos) == 1 or _has_help(args) or pos[1] == "help":
|
|
148
|
+
return _r("aws", svc)
|
|
149
|
+
op = pos[1]
|
|
150
|
+
if svc == "configure":
|
|
151
|
+
return _r("aws", f"configure {op}") if op in ("list", "get", "list-profiles") else _u("aws", f"configure {op}")
|
|
152
|
+
if svc == "redshift-data" and op in ("execute-statement", "batch-execute-statement"):
|
|
153
|
+
return _s("aws", f"{svc} {op}")
|
|
154
|
+
if svc == "athena" and op == "start-query-execution":
|
|
155
|
+
return _s("aws", f"{svc} {op}")
|
|
156
|
+
if svc == "s3":
|
|
157
|
+
return _aws_s3(op, pos)
|
|
158
|
+
label = f"{svc} {op}"
|
|
159
|
+
if op == "wait" or op.startswith(_AWS_READ_PREFIX) or (svc, op) in _AWS_EXTRA_READ:
|
|
160
|
+
return _r("aws", label)
|
|
161
|
+
if op.startswith(_AWS_WRITE_PREFIX) or op == "invoke":
|
|
162
|
+
return _w("aws", label)
|
|
163
|
+
return _u("aws", label)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _aws_s3(op: str, pos: list[str]) -> Verdict:
|
|
167
|
+
label = f"s3 {op}"
|
|
168
|
+
if op in ("ls", "presign"):
|
|
169
|
+
return _r("aws", label)
|
|
170
|
+
if op in ("cp", "sync"):
|
|
171
|
+
paths = pos[2:]
|
|
172
|
+
if len(paths) < 2:
|
|
173
|
+
return _u("aws", label, "needs a source and a destination")
|
|
174
|
+
src, dst = paths[0], paths[1]
|
|
175
|
+
if dst.startswith("s3://"):
|
|
176
|
+
return _w("aws", label, "destination is s3://")
|
|
177
|
+
if src.startswith("s3://") or src == "-":
|
|
178
|
+
return _r("aws", label) if src.startswith("s3://") else _u("aws", label, "local to local")
|
|
179
|
+
return _u("aws", label, "neither side is s3://")
|
|
180
|
+
if op in ("mv", "rm", "mb", "rb", "website"):
|
|
181
|
+
return _w("aws", label)
|
|
182
|
+
return _u("aws", label)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------- gcloud / gsutil / bq
|
|
186
|
+
|
|
187
|
+
_GCLOUD_VALUED = {"--project", "--account", "--billing-project", "--configuration", "--format", "--filter", "--flags-file",
|
|
188
|
+
"--impersonate-service-account", "--verbosity", "--zone", "--region", "--location", "--limit", "--sort-by",
|
|
189
|
+
"--page-size"}
|
|
190
|
+
_GCLOUD_READS = {"list", "describe", "read", "tail", "ls", "cat", "du", "hash", "show"}
|
|
191
|
+
_GCLOUD_WRITES = {"create", "delete", "update", "deploy", "run", "submit", "import", "start", "stop", "restart", "resume",
|
|
192
|
+
"suspend", "pause", "cancel", "enable", "disable", "reset", "resize", "trigger", "publish", "execute",
|
|
193
|
+
"copy", "move", "rename", "attach", "detach", "apply", "replace", "restore", "undelete", "upgrade", "promote",
|
|
194
|
+
"rollback", "patch", "cp", "mv", "rm", "rsync", "ssh", "scp", "login", "revoke", "add", "remove", "call", "pull", "invoke",
|
|
195
|
+
"export", "clone", "ack", "bind"}
|
|
196
|
+
_GCLOUD_RP = ("list-", "describe-", "get-", "show-")
|
|
197
|
+
_GCLOUD_WP = ("set-", "add-", "remove-", "delete-", "create-", "update-", "enable-", "disable-")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
_SCHEME = re.compile(r"^(?!file:)[A-Za-z][A-Za-z0-9+.-]*://")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _gs(p: str) -> bool:
|
|
204
|
+
return p.startswith("gs://")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _remote(p: str) -> bool:
|
|
208
|
+
return bool(_SCHEME.match(p))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _gcloud(args: list[str]) -> Verdict:
|
|
212
|
+
pos, flags = scan(args, _GCLOUD_VALUED)
|
|
213
|
+
while pos and pos[0] in ("alpha", "beta", "preview"):
|
|
214
|
+
pos = pos[1:]
|
|
215
|
+
if not pos or _has_help(args):
|
|
216
|
+
return _r("gcloud")
|
|
217
|
+
if pos[0] in ("version", "info", "help", "topic", "feedback") or (pos[0] == "config" and len(pos) > 1 and pos[1] in (
|
|
218
|
+
"list", "get", "get-value", "configurations")):
|
|
219
|
+
return _r("gcloud", _label(pos, 2))
|
|
220
|
+
if pos[0] == "auth" and len(pos) > 1 and pos[1] == "list":
|
|
221
|
+
return _r("gcloud", "auth list")
|
|
222
|
+
if pos[0] == "storage" and len(pos) > 1:
|
|
223
|
+
sub = pos[1]
|
|
224
|
+
if sub in ("ls", "cat", "du", "hash"):
|
|
225
|
+
return _r("gcloud", f"storage {sub}")
|
|
226
|
+
if sub in ("cp", "rsync"):
|
|
227
|
+
return _w("gcloud", f"storage {sub}", "destination is not local") if len(pos) > 2 and _remote(pos[-1]) else (
|
|
228
|
+
_r("gcloud", f"storage {sub}") if len(pos) > 3 and _gs(pos[2]) else _u("gcloud", f"storage {sub}", "neither side is gs://"))
|
|
229
|
+
if sub in ("mv", "rm"):
|
|
230
|
+
return _w("gcloud", f"storage {sub}")
|
|
231
|
+
hit = _path_verb(pos, _GCLOUD_READS, _GCLOUD_WRITES, _GCLOUD_RP, _GCLOUD_WP)
|
|
232
|
+
label = _label(pos)
|
|
233
|
+
if hit is None:
|
|
234
|
+
return _u("gcloud", label)
|
|
235
|
+
return _r("gcloud", label) if hit[0] == "read" else _w("gcloud", label)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_GSUTIL_VALUED = {"-o", "-h", "-u", "-i"}
|
|
239
|
+
_GSUTIL_SUBVERB_GROUPS = {"acl", "iam", "notification", "label", "lifecycle", "cors", "defacl", "versioning", "web", "retention",
|
|
240
|
+
"logging", "requesterpays", "kms", "hmac", "autoclass", "bucketpolicyonly", "ubla", "pap", "rpo", "ls"}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _gsutil(args: list[str]) -> Verdict:
|
|
244
|
+
pos, flags = scan(args, _GSUTIL_VALUED)
|
|
245
|
+
if not pos or _has_help(args):
|
|
246
|
+
return _r("gsutil")
|
|
247
|
+
v = pos[0]
|
|
248
|
+
if v in ("ls", "cat", "du", "stat", "hash", "version", "help", "lsattr"):
|
|
249
|
+
return _r("gsutil", v)
|
|
250
|
+
if v in ("cp", "rsync"):
|
|
251
|
+
if len(pos) < 3:
|
|
252
|
+
return _u("gsutil", v, "needs a source and a destination")
|
|
253
|
+
return _w("gsutil", v, "destination is not local") if _remote(pos[-1]) else (
|
|
254
|
+
_r("gsutil", v) if _gs(pos[1]) else _u("gsutil", v, "neither side is gs://"))
|
|
255
|
+
if v in ("mv", "rm", "mb", "rb", "setmeta", "rewrite", "compose", "update", "signurl"):
|
|
256
|
+
return _w("gsutil", v)
|
|
257
|
+
if v in _GSUTIL_SUBVERB_GROUPS:
|
|
258
|
+
if len(pos) > 1 and pos[1] in ("get", "list"):
|
|
259
|
+
return _r("gsutil", f"{v} {pos[1]}")
|
|
260
|
+
return _w("gsutil", v + (f" {pos[1]}" if len(pos) > 1 else ""))
|
|
261
|
+
return _u("gsutil", v)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
_BQ_VALUED = {"--project_id", "--dataset_id", "--location", "--format", "--api", "--api_version", "--job_id", "--max_rows",
|
|
265
|
+
"--bigquery_discovery_api_key", "--discovery_file", "--bq_discovery_api_key", "--service_account",
|
|
266
|
+
"--service_account_credential_file", "--credential_file", "--proxy_address", "--destination_table", "--schedule",
|
|
267
|
+
"--target_dataset", "--destination_schema", "--destination_kms_key", "--flagfile"}
|
|
268
|
+
_BQ_WRITE_FLAGS = {"--destination_table", "--append_table", "--replace", "--schedule", "--target_dataset", "--destination_schema",
|
|
269
|
+
"--destination_kms_key"}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _bq(args: list[str]) -> Verdict:
|
|
273
|
+
pos, flags = scan(args, _BQ_VALUED)
|
|
274
|
+
if not pos or _has_help(args):
|
|
275
|
+
return _r("bq")
|
|
276
|
+
v = pos[0]
|
|
277
|
+
names = {f.split("=", 1)[0] for f in flags}
|
|
278
|
+
if "--flagfile" in names:
|
|
279
|
+
return _w("bq", v, "--flagfile can carry any flag")
|
|
280
|
+
if v == "query":
|
|
281
|
+
return _w("bq", v, "writes to a table or schedules the query") if names & _BQ_WRITE_FLAGS else _s("bq", v)
|
|
282
|
+
if v in ("ls", "show", "head", "version", "help", "wait", "info"):
|
|
283
|
+
return _r("bq", v)
|
|
284
|
+
if v in ("mk", "rm", "cp", "load", "insert", "update", "extract", "cancel", "partition", "undelete", "truncate", "add-iam-policy-binding",
|
|
285
|
+
"remove-iam-policy-binding", "set-iam-policy"):
|
|
286
|
+
return _w("bq", v)
|
|
287
|
+
return _u("bq", v)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# ---------------------------------------------------------------- databricks
|
|
291
|
+
|
|
292
|
+
_DBX_VALUED = {"-p", "--profile", "-o", "--output", "--log-level", "--log-file", "--log-format", "-t", "--target",
|
|
293
|
+
"--progress-format"}
|
|
294
|
+
_DBX_WRITE_SUB = {"create", "delete", "update", "run-now", "submit", "reset", "cancel", "start", "stop", "restart", "terminate",
|
|
295
|
+
"edit", "resize", "pin", "unpin", "import", "put", "set", "patch", "deploy", "destroy", "run", "trigger",
|
|
296
|
+
"permanent-delete", "repair", "install", "uninstall", "set-permissions", "update-permissions", "execute",
|
|
297
|
+
"mkdirs", "mkdir", "move", "rename", "migrate", "publish", "unpublish", "enable", "disable", "grant", "revoke"}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _dbx_dbfs(p: str) -> bool:
|
|
301
|
+
return p.startswith(("dbfs:", "/Volumes", "/Workspace"))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _databricks(args: list[str]) -> Verdict:
|
|
305
|
+
pos, flags = scan(args, _DBX_VALUED)
|
|
306
|
+
if not pos or _has_help(args):
|
|
307
|
+
return _r("databricks")
|
|
308
|
+
grp = pos[0]
|
|
309
|
+
if grp in ("version", "help", "completion"):
|
|
310
|
+
return _r("databricks", grp)
|
|
311
|
+
if grp == "sql":
|
|
312
|
+
return _s("databricks", "sql")
|
|
313
|
+
if len(pos) < 2:
|
|
314
|
+
return _r("databricks", grp)
|
|
315
|
+
sub = pos[1]
|
|
316
|
+
label = f"{grp} {sub}"
|
|
317
|
+
if grp == "fs":
|
|
318
|
+
if sub in ("ls", "cat"):
|
|
319
|
+
return _r("databricks", label)
|
|
320
|
+
if sub == "cp":
|
|
321
|
+
return _w("databricks", label, "destination is dbfs") if len(pos) > 3 and _dbx_dbfs(pos[-1]) else _r("databricks", label)
|
|
322
|
+
return _w("databricks", label)
|
|
323
|
+
if grp == "api":
|
|
324
|
+
return _r("databricks", label) if sub in ("get", "head") else _w("databricks", label)
|
|
325
|
+
if grp == "auth":
|
|
326
|
+
return _r("databricks", label) if sub in ("profiles", "describe", "env") else _u("databricks", label)
|
|
327
|
+
if grp == "bundle":
|
|
328
|
+
if sub in ("validate", "summary", "schema", "generate", "init", "open"):
|
|
329
|
+
return _r("databricks", label) if sub in ("validate", "summary", "schema") else _u("databricks", label)
|
|
330
|
+
return _w("databricks", label) if sub in ("deploy", "run", "destroy") else _u("databricks", label)
|
|
331
|
+
if sub.startswith(("list", "get", "describe", "export", "show", "search")) or sub in ("me", "ls", "events", "export-dir", "wait"):
|
|
332
|
+
return _r("databricks", label)
|
|
333
|
+
if sub in _DBX_WRITE_SUB or sub.startswith(("cancel", "create-", "delete-", "update-", "set-")):
|
|
334
|
+
return _w("databricks", label)
|
|
335
|
+
return _u("databricks", label)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _dbsqlcli(args: list[str]) -> Verdict:
|
|
339
|
+
return _s("dbsqlcli", "")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
# ---------------------------------------------------------------- kubectl / helm
|
|
343
|
+
|
|
344
|
+
_KUBECTL_VALUED = {"-n", "--namespace", "--context", "--cluster", "--user", "--kubeconfig", "-s", "--server", "--token", "--as",
|
|
345
|
+
"--as-group", "--request-timeout", "--cache-dir", "--certificate-authority", "--client-certificate",
|
|
346
|
+
"--client-key", "-v", "--v", "--tls-server-name", "--username", "--password", "--profile", "--profile-output",
|
|
347
|
+
"--as-uid"}
|
|
348
|
+
_KUBECTL_EXEC_VALUED = _KUBECTL_VALUED | {"-c", "--container", "-f", "--filename", "--pod-running-timeout"}
|
|
349
|
+
_KUBECTL_READ = {"get", "describe", "logs", "top", "explain", "version", "api-resources", "api-versions", "cluster-info", "diff",
|
|
350
|
+
"events", "wait", "completion", "options", "help"}
|
|
351
|
+
_KUBECTL_WRITE = {"apply", "create", "delete", "edit", "patch", "replace", "scale", "cordon", "uncordon", "drain", "taint", "label",
|
|
352
|
+
"annotate", "set", "port-forward", "expose", "run", "autoscale", "certificate", "debug", "attach", "proxy",
|
|
353
|
+
"kustomize"}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _kubectl(args: list[str]) -> Verdict:
|
|
357
|
+
pos, flags = scan(args, _KUBECTL_VALUED)
|
|
358
|
+
if not pos or _has_help(args):
|
|
359
|
+
return _r("kubectl")
|
|
360
|
+
v = pos[0]
|
|
361
|
+
if v in _KUBECTL_READ:
|
|
362
|
+
return _r("kubectl", v)
|
|
363
|
+
if v == "config":
|
|
364
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
365
|
+
return _r("kubectl", f"config {sub}") if sub in ("view", "get-contexts", "current-context", "get-clusters", "get-users") or not sub else _u(
|
|
366
|
+
"kubectl", f"config {sub}")
|
|
367
|
+
if v == "auth":
|
|
368
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
369
|
+
return _r("kubectl", f"auth {sub}") if sub in ("can-i", "whoami") else _u("kubectl", f"auth {sub}")
|
|
370
|
+
if v == "rollout":
|
|
371
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
372
|
+
if sub in ("status", "history"):
|
|
373
|
+
return _r("kubectl", f"rollout {sub}")
|
|
374
|
+
return _w("kubectl", f"rollout {sub}") if sub in ("restart", "undo", "pause", "resume") else _u("kubectl", f"rollout {sub}")
|
|
375
|
+
if v == "cp":
|
|
376
|
+
dests = pos[1:]
|
|
377
|
+
if len(dests) >= 2 and ":" in dests[-1] and not re.match(r"^[A-Za-z]:[\\/]", dests[-1]):
|
|
378
|
+
return _w("kubectl", "cp", "copies into a pod")
|
|
379
|
+
return _r("kubectl", "cp") if len(dests) >= 2 else _u("kubectl", "cp")
|
|
380
|
+
if v in _KUBECTL_WRITE:
|
|
381
|
+
return _w("kubectl", v)
|
|
382
|
+
if v == "exec":
|
|
383
|
+
return _u("kubectl", "exec", "unreachable: handled by the wrapper logic")
|
|
384
|
+
return _u("kubectl", v)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
_HELM_VALUED = {"-n", "--namespace", "--kube-context", "--kubeconfig", "--burst-limit", "--kube-apiserver", "--kube-token",
|
|
388
|
+
"--kube-as-user", "--kube-as-group", "--kube-ca-file", "--registry-config", "--repository-cache",
|
|
389
|
+
"--repository-config"}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _helm(args: list[str]) -> Verdict:
|
|
393
|
+
pos, flags = scan(args, _HELM_VALUED)
|
|
394
|
+
if not pos or _has_help(args):
|
|
395
|
+
return _r("helm")
|
|
396
|
+
v = pos[0]
|
|
397
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
398
|
+
if v in ("list", "ls", "status", "get", "history", "show", "inspect", "template", "version", "lint", "search", "env", "help",
|
|
399
|
+
"completion", "verify"):
|
|
400
|
+
return _r("helm", v)
|
|
401
|
+
if v == "repo":
|
|
402
|
+
return _r("helm", "repo list") if sub in ("list", "") else _u("helm", f"repo {sub}")
|
|
403
|
+
if v in ("install", "upgrade", "uninstall", "delete", "rollback"):
|
|
404
|
+
return _w("helm", v)
|
|
405
|
+
return _u("helm", v)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# ---------------------------------------------------------------- terraform / pulumi
|
|
409
|
+
|
|
410
|
+
_TF_VALUED: set[str] = set()
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _terraform(args: list[str], name: str = "terraform") -> Verdict:
|
|
414
|
+
pos = [a for a in args if not a.startswith("-chdir=")]
|
|
415
|
+
pos, flags = scan(pos)
|
|
416
|
+
if not pos:
|
|
417
|
+
return _r(name)
|
|
418
|
+
if any(f in ("-help", "-h", "--help", "-version", "--version") for f in flags) and len(pos) <= 1:
|
|
419
|
+
return _r(name, pos[0])
|
|
420
|
+
v = pos[0]
|
|
421
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
422
|
+
if v in ("plan", "show", "validate", "output", "providers", "version", "init", "graph", "get", "help", "metadata", "console"):
|
|
423
|
+
if v == "console":
|
|
424
|
+
return _u(name, v, "interactive")
|
|
425
|
+
if v == "providers" and sub in ("lock", "mirror"):
|
|
426
|
+
return _u(name, f"providers {sub}")
|
|
427
|
+
return _r(name, v)
|
|
428
|
+
if v == "fmt":
|
|
429
|
+
return _r(name, "fmt -check") if any(f in ("-check", "--check") for f in flags) else _w(name, "fmt", "rewrites files")
|
|
430
|
+
if v == "state":
|
|
431
|
+
if sub in ("list", "show", "pull", ""):
|
|
432
|
+
return _r(name, f"state {sub}".strip())
|
|
433
|
+
return _w(name, f"state {sub}") if sub in ("rm", "mv", "push", "replace-provider") else _u(name, f"state {sub}")
|
|
434
|
+
if v == "workspace":
|
|
435
|
+
return _r(name, f"workspace {sub}") if sub in ("list", "show") else _w(name, f"workspace {sub}".strip())
|
|
436
|
+
if v in ("apply", "destroy", "import", "taint", "untaint", "force-unlock", "refresh", "login", "logout", "test"):
|
|
437
|
+
return _w(name, v)
|
|
438
|
+
return _u(name, v)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
_PULUMI_VALUED = {"--cwd", "-C", "-s", "--stack", "--color", "--tracing", "--profiling", "-v", "--config-file"}
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _pulumi(args: list[str]) -> Verdict:
|
|
445
|
+
pos, flags = scan(args, _PULUMI_VALUED)
|
|
446
|
+
if not pos or _has_help(args):
|
|
447
|
+
return _r("pulumi")
|
|
448
|
+
v = pos[0]
|
|
449
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
450
|
+
if v in ("preview", "whoami", "version", "about", "logs", "help", "completion", "gen-completion"):
|
|
451
|
+
return _r("pulumi", v)
|
|
452
|
+
if v == "stack":
|
|
453
|
+
if sub in ("", "ls", "output", "history", "graph", "export", "tag"):
|
|
454
|
+
if sub == "tag" and len(pos) > 2 and pos[2] != "ls" and pos[2] != "get":
|
|
455
|
+
return _w("pulumi", f"stack tag {pos[2]}")
|
|
456
|
+
return _r("pulumi", f"stack {sub}".strip())
|
|
457
|
+
return _w("pulumi", f"stack {sub}")
|
|
458
|
+
if v == "config":
|
|
459
|
+
if sub in ("", "get"):
|
|
460
|
+
return _r("pulumi", f"config {sub}".strip())
|
|
461
|
+
return _w("pulumi", f"config {sub}")
|
|
462
|
+
if v in ("up", "destroy", "refresh", "import", "cancel", "login", "logout", "new", "state", "watch", "convert", "policy"):
|
|
463
|
+
return _w("pulumi", v + (f" {sub}" if sub else ""))
|
|
464
|
+
return _u("pulumi", v)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# ---------------------------------------------------------------- az / snow
|
|
468
|
+
|
|
469
|
+
_AZ_VALUED = {"--subscription", "-o", "--output", "--query", "-g", "--resource-group", "--name", "-n", "--location", "-l"}
|
|
470
|
+
_AZ_READS = {"list", "show", "query", "exists", "wait", "list-locations", "download", "download-batch"}
|
|
471
|
+
_AZ_WRITES = {"create", "delete", "update", "set", "start", "stop", "restart", "run", "invoke", "deploy", "add", "remove",
|
|
472
|
+
"import", "scale", "swap", "purge", "restore", "assign", "cancel", "upload", "upload-batch", "login",
|
|
473
|
+
"logout", "move", "regenerate-key", "rotate", "sync", "copy", "attach", "detach", "enable", "disable"}
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _az(args: list[str]) -> Verdict:
|
|
477
|
+
pos, flags = scan(args, _AZ_VALUED)
|
|
478
|
+
if not pos or _has_help(args):
|
|
479
|
+
return _r("az")
|
|
480
|
+
if pos[0] in ("version", "help", "find", "feedback", "interactive") and len(pos) <= 2:
|
|
481
|
+
return _r("az", pos[0])
|
|
482
|
+
if pos[0] == "rest":
|
|
483
|
+
m = None
|
|
484
|
+
for k, f in enumerate(args):
|
|
485
|
+
if f in ("--method", "-m") and k + 1 < len(args):
|
|
486
|
+
m = args[k + 1].lower()
|
|
487
|
+
elif f.startswith("--method="):
|
|
488
|
+
m = f.split("=", 1)[1].lower()
|
|
489
|
+
return _r("az", "rest") if m in (None, "get", "head") else _w("az", f"rest --method {m}")
|
|
490
|
+
hit = _path_verb(pos, _AZ_READS, _AZ_WRITES, ("list-", "show-", "get-", "check-", "download"), ("upload", "create-", "delete-",
|
|
491
|
+
"update-", "set-", "add-", "remove-"))
|
|
492
|
+
label = _label(pos)
|
|
493
|
+
if hit is None:
|
|
494
|
+
return _u("az", label)
|
|
495
|
+
return _r("az", label) if hit[0] == "read" else _w("az", label)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
_SNOW_VALUED = {"-c", "--connection", "--format", "--config-file", "--database", "--schema", "--role", "--warehouse"}
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _snow(args: list[str]) -> Verdict:
|
|
502
|
+
pos, flags = scan(args, _SNOW_VALUED)
|
|
503
|
+
if not pos or _has_help(args):
|
|
504
|
+
return _r("snow")
|
|
505
|
+
v = pos[0]
|
|
506
|
+
if v == "sql":
|
|
507
|
+
return _s("snow", "sql")
|
|
508
|
+
if v == "object" and len(pos) > 1 and pos[1] in ("list", "describe"):
|
|
509
|
+
return _r("snow", f"object {pos[1]}")
|
|
510
|
+
if v in ("help", "--version"):
|
|
511
|
+
return _r("snow", v)
|
|
512
|
+
return _w("snow", _label(pos, 2), "snow: only `sql` and `object list|describe` are read")
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
# ---------------------------------------------------------------- dbt / airflow / dagster / prefect / kafka
|
|
516
|
+
|
|
517
|
+
_DBT_VALUED = {"--log-level", "--log-path", "--profiles-dir", "--project-dir", "--target", "-t", "--profile", "--vars",
|
|
518
|
+
"--threads", "--log-format", "--state", "--defer-state"}
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _dbt(args: list[str]) -> Verdict:
|
|
522
|
+
pos, flags = scan(args, _DBT_VALUED)
|
|
523
|
+
if not pos or _has_help(args):
|
|
524
|
+
return _r("dbt")
|
|
525
|
+
v = pos[0]
|
|
526
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
527
|
+
if v in ("parse", "compile", "ls", "list", "debug", "deps", "test", "show", "version", "help"):
|
|
528
|
+
return _r("dbt", v)
|
|
529
|
+
if v == "docs":
|
|
530
|
+
return _r("dbt", f"docs {sub}".strip()) if sub in ("generate", "serve", "") else _u("dbt", f"docs {sub}")
|
|
531
|
+
if v == "source":
|
|
532
|
+
return _r("dbt", "source freshness") if sub in ("freshness", "") else _u("dbt", f"source {sub}")
|
|
533
|
+
if v in ("run", "build", "seed", "snapshot", "run-operation", "retry", "clone"):
|
|
534
|
+
return _w("dbt", v)
|
|
535
|
+
return _u("dbt", v)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
_AF_READ = {
|
|
539
|
+
"dags": {"list", "show", "state", "list-runs", "list-jobs", "report", "details", "next-execution", "list-import-errors", "status"},
|
|
540
|
+
"tasks": {"list", "state", "states-for-dag-run", "failed-deps", "render"},
|
|
541
|
+
"config": {"get-value", "list"},
|
|
542
|
+
"connections": {"list", "get"},
|
|
543
|
+
"variables": {"list", "get"},
|
|
544
|
+
"pools": {"list", "get"},
|
|
545
|
+
"jobs": {"check"},
|
|
546
|
+
"providers": {"list", "get", "behaviours", "links", "widgets", "hooks", "triggers", "auth"},
|
|
547
|
+
"plugins": {""},
|
|
548
|
+
"assets": {"list", "details"},
|
|
549
|
+
"roles": {"list"},
|
|
550
|
+
}
|
|
551
|
+
_AF_WRITE = {
|
|
552
|
+
"dags": {"trigger", "pause", "unpause", "delete", "backfill", "reserialize", "test"},
|
|
553
|
+
"tasks": {"clear", "run", "test"},
|
|
554
|
+
"connections": {"add", "delete", "import", "export", "create-default-connections", "test"},
|
|
555
|
+
"variables": {"set", "delete", "import", "export"},
|
|
556
|
+
"pools": {"set", "delete", "import", "export"},
|
|
557
|
+
"roles": {"create", "delete", "add-perms", "del-perms", "export", "import"},
|
|
558
|
+
}
|
|
559
|
+
_AF_VALUED = {"-o", "--output"}
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _airflow(args: list[str]) -> Verdict:
|
|
563
|
+
pos, flags = scan(args, _AF_VALUED)
|
|
564
|
+
if not pos or _has_help(args):
|
|
565
|
+
return _r("airflow")
|
|
566
|
+
g = pos[0]
|
|
567
|
+
sub = pos[1] if len(pos) > 1 else ""
|
|
568
|
+
if g == "info" and any(f.split("=", 1)[0] == "--file-io" for f in flags):
|
|
569
|
+
return _w("airflow", "info --file-io", "uploads the environment info")
|
|
570
|
+
if g in ("version", "info", "cheat-sheet", "help"):
|
|
571
|
+
return _r("airflow", g)
|
|
572
|
+
if g in ("db", "users", "kerberos", "rotate-fernet-key", "scheduler", "webserver", "standalone", "triggerer", "celery",
|
|
573
|
+
"kubernetes", "api-server", "dag-processor", "internal-api", "sync-perm", "task", "migrate", "upgradecheck"):
|
|
574
|
+
return _w("airflow", _label(pos, 2), "")
|
|
575
|
+
label = f"{g} {sub}".strip()
|
|
576
|
+
if sub in _AF_WRITE.get(g, ()):
|
|
577
|
+
return _w("airflow", label)
|
|
578
|
+
if sub in _AF_READ.get(g, ()):
|
|
579
|
+
return _r("airflow", label)
|
|
580
|
+
if g == "config" and not sub:
|
|
581
|
+
return _r("airflow", g)
|
|
582
|
+
return _u("airflow", label)
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
_DAGSTER_READ = {"list", "info", "print", "preview", "logs", "version", "show", "list-versions", "export", "validate"}
|
|
586
|
+
_DAGSTER_WRITE = {"execute", "launch", "backfill", "materialize", "wipe", "delete", "start", "stop", "restart", "migrate", "reindex",
|
|
587
|
+
"cursor", "scaffold_config", "import", "run", "up", "reload", "set", "trigger"}
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _dagster(args: list[str]) -> Verdict:
|
|
591
|
+
pos, flags = scan(args)
|
|
592
|
+
if not pos or _has_help(args):
|
|
593
|
+
return _r("dagster")
|
|
594
|
+
if pos[0] in ("version", "help"):
|
|
595
|
+
return _r("dagster", pos[0])
|
|
596
|
+
hit = _path_verb(pos, _DAGSTER_READ, _DAGSTER_WRITE, ("list-",), ("delete-", "set-"), start=1, limit=4)
|
|
597
|
+
label = _label(pos, 3)
|
|
598
|
+
if hit is None:
|
|
599
|
+
return _u("dagster", label)
|
|
600
|
+
return _r("dagster", label) if hit[0] == "read" else _w("dagster", label)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
_PREFECT_READ = {"ls", "list", "inspect", "view", "get", "logs", "preview", "version", "status", "validate", "show"}
|
|
604
|
+
_PREFECT_WRITE = {"run", "execute", "cancel", "delete", "create", "apply", "build", "pause", "resume", "set", "unset", "set-schedule",
|
|
605
|
+
"clear-schedule", "register", "start", "stop", "reset", "retry", "restart", "use", "rename", "login", "logout",
|
|
606
|
+
"migrate", "upgrade", "downgrade", "revision", "push", "pull", "deploy", "trigger", "update", "provision-infra",
|
|
607
|
+
"enable", "disable", "sync"}
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _prefect(args: list[str]) -> Verdict:
|
|
611
|
+
pos, flags = scan(args)
|
|
612
|
+
if not pos or _has_help(args):
|
|
613
|
+
return _r("prefect")
|
|
614
|
+
if pos[0] in ("version", "help"):
|
|
615
|
+
return _r("prefect", pos[0])
|
|
616
|
+
hit = _path_verb(pos, _PREFECT_READ, _PREFECT_WRITE, (), (), start=1, limit=5)
|
|
617
|
+
label = _label(pos, 3)
|
|
618
|
+
if hit is None:
|
|
619
|
+
return _u("prefect", label)
|
|
620
|
+
return _r("prefect", label) if hit[0] == "read" else _w("prefect", label)
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _kafka(name: str, args: list[str]) -> Verdict:
|
|
624
|
+
flags = {a.split("=", 1)[0] for a in args if a.startswith("--")}
|
|
625
|
+
if flags & {"--create", "--delete", "--alter", "--delete-config", "--reassignment-json-file", "--execute", "--add-config"}:
|
|
626
|
+
verb = next(f for f in ("--create", "--delete", "--alter", "--delete-config", "--execute", "--add-config",
|
|
627
|
+
"--reassignment-json-file") if f in flags)
|
|
628
|
+
return _w(name, verb)
|
|
629
|
+
if flags & {"--list", "--describe", "--help", "--version"}:
|
|
630
|
+
return _r(name, next(iter(sorted(flags & {"--list", "--describe", "--help", "--version"}))))
|
|
631
|
+
return _u(name, "", "no --list or --describe")
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
# ---------------------------------------------------------------- program table
|
|
635
|
+
|
|
636
|
+
def _guarded(name: str, args: list[str]) -> Verdict | None:
|
|
637
|
+
n = name[:-3] if name.endswith(".sh") else name
|
|
638
|
+
if n == "aws":
|
|
639
|
+
return _aws(args)
|
|
640
|
+
if n == "gcloud":
|
|
641
|
+
return _gcloud(args)
|
|
642
|
+
if n == "gsutil":
|
|
643
|
+
return _gsutil(args)
|
|
644
|
+
if n == "bq":
|
|
645
|
+
return _bq(args)
|
|
646
|
+
if n == "databricks":
|
|
647
|
+
return _databricks(args)
|
|
648
|
+
if n == "dbsqlcli":
|
|
649
|
+
return _dbsqlcli(args)
|
|
650
|
+
if n == "kubectl":
|
|
651
|
+
return _kubectl(args)
|
|
652
|
+
if n == "helm":
|
|
653
|
+
return _helm(args)
|
|
654
|
+
if n in ("terraform", "tofu"):
|
|
655
|
+
return _terraform(args, n)
|
|
656
|
+
if n == "pulumi":
|
|
657
|
+
return _pulumi(args)
|
|
658
|
+
if n == "az":
|
|
659
|
+
return _az(args)
|
|
660
|
+
if n == "snow":
|
|
661
|
+
return _snow(args)
|
|
662
|
+
if n == "dbt":
|
|
663
|
+
return _dbt(args)
|
|
664
|
+
if n == "airflow":
|
|
665
|
+
return _airflow(args)
|
|
666
|
+
if n == "dagster":
|
|
667
|
+
return _dagster(args)
|
|
668
|
+
if n == "prefect":
|
|
669
|
+
return _prefect(args)
|
|
670
|
+
if n in ("kafka-topics", "kafka-configs"):
|
|
671
|
+
return _kafka(n, args)
|
|
672
|
+
return None
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def classify_cli(name: str, args: list[str]) -> Verdict | None:
|
|
676
|
+
"""Verdict for one guarded CLI's arguments, no wrappers followed (None: not a guarded CLI). Pack argv checks build on this."""
|
|
677
|
+
return _guarded(name, args)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# ---------------------------------------------------------------- wrappers
|
|
681
|
+
|
|
682
|
+
_ASSIGN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
|
|
683
|
+
_CONTROL = {"do", "then", "else", "elif", "if", "while", "until", "!", "{", "}", "function"}
|
|
684
|
+
_SHELLS = {"bash", "sh", "zsh", "dash", "ksh", "ash"}
|
|
685
|
+
_SIMPLE_PREFIX = {"nohup", "exec", "builtin", "setsid", "unbuffer", "chronic", "caffeinate", "ionice", "chrt", "taskset", "stdbuf",
|
|
686
|
+
"nice", "time", "command", "sudo", "doas", "env", "timeout", "watch", "flock", "su", "xargs", "find", "parallel",
|
|
687
|
+
"pkexec", "runuser"}
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _drop_options(args: list[str], valued: set[str]) -> list[str]:
|
|
691
|
+
"""Words after leading options (an option in `valued` consumes its value; a lone `--` ends options)."""
|
|
692
|
+
i = 0
|
|
693
|
+
while i < len(args):
|
|
694
|
+
a = args[i]
|
|
695
|
+
if a == "--":
|
|
696
|
+
return args[i + 1:]
|
|
697
|
+
if a.startswith("-") and len(a) > 1:
|
|
698
|
+
if a in valued and "=" not in a:
|
|
699
|
+
i += 2
|
|
700
|
+
continue
|
|
701
|
+
i += 1
|
|
702
|
+
continue
|
|
703
|
+
break
|
|
704
|
+
return args[i:]
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _unwrap_env(args: list[str]) -> list[str] | tuple[str, str] | None:
|
|
708
|
+
i = 0
|
|
709
|
+
while i < len(args):
|
|
710
|
+
a = args[i]
|
|
711
|
+
if a == "--":
|
|
712
|
+
return args[i + 1:]
|
|
713
|
+
if a in ("-u", "--unset", "-C", "--chdir", "-P"):
|
|
714
|
+
i += 2
|
|
715
|
+
continue
|
|
716
|
+
if a in ("-S", "--split-string") and i + 1 < len(args):
|
|
717
|
+
return ("string", args[i + 1] + " " + " ".join(shlex.quote(x) for x in args[i + 2:]))
|
|
718
|
+
if a.startswith("--split-string="):
|
|
719
|
+
return ("string", a.split("=", 1)[1] + " " + " ".join(shlex.quote(x) for x in args[i + 1:]))
|
|
720
|
+
if a.startswith("-S") and len(a) > 2:
|
|
721
|
+
return ("string", a[2:] + " " + " ".join(shlex.quote(x) for x in args[i + 1:]))
|
|
722
|
+
if a.startswith("-") or _ASSIGN.match(a):
|
|
723
|
+
i += 1
|
|
724
|
+
continue
|
|
725
|
+
break
|
|
726
|
+
return args[i:]
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _unwrap_sudo(args: list[str]) -> list[str]:
|
|
730
|
+
valued = {"-u", "-g", "-h", "-p", "-C", "-D", "-R", "-r", "-t", "-U", "-T", "--user", "--group", "--host", "--prompt",
|
|
731
|
+
"--chdir", "--role", "--type", "--other-user"}
|
|
732
|
+
rest = _drop_options(args, valued)
|
|
733
|
+
while rest and _ASSIGN.match(rest[0]):
|
|
734
|
+
rest = rest[1:]
|
|
735
|
+
return rest
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _unwrap_timeout(args: list[str]) -> list[str]:
|
|
739
|
+
rest = _drop_options(args, {"-s", "--signal", "-k", "--kill-after"})
|
|
740
|
+
return rest[1:] if rest else [] # the duration
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _unwrap_nice(args: list[str]) -> list[str]:
|
|
744
|
+
return _drop_options(args, {"-n", "--adjustment"})
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def _unwrap_stdbuf(args: list[str]) -> list[str]:
|
|
748
|
+
return _drop_options(args, {"-i", "-o", "-e", "--input", "--output", "--error"})
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def _unwrap_watch(args: list[str]) -> list[str]:
|
|
752
|
+
return _drop_options(args, {"-n", "--interval", "-w", "--width"})
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _unwrap_flock(args: list[str]) -> list[str]:
|
|
756
|
+
if "-c" in args or "--command" in args:
|
|
757
|
+
k = args.index("-c") if "-c" in args else args.index("--command")
|
|
758
|
+
return [] if k + 1 >= len(args) else ["bash", "-c", args[k + 1]]
|
|
759
|
+
rest = _drop_options(args, {"-w", "--timeout", "-E", "--conflict-exit-code"})
|
|
760
|
+
return rest[1:] if rest else []
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
_XARGS_LONG_VALUED = {"--arg-file", "--delimiter", "--max-args", "--max-chars", "--max-procs", "--process-slot-var"}
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _unwrap_xargs(args: list[str]) -> list[str]:
|
|
767
|
+
i = 0
|
|
768
|
+
while i < len(args):
|
|
769
|
+
a = args[i]
|
|
770
|
+
if a == "--":
|
|
771
|
+
return args[i + 1:]
|
|
772
|
+
if a.startswith("--"):
|
|
773
|
+
i += 2 if (a in _XARGS_LONG_VALUED) else 1
|
|
774
|
+
continue
|
|
775
|
+
if a.startswith("-") and len(a) > 1:
|
|
776
|
+
consumed_next = False
|
|
777
|
+
for j, ch in enumerate(a[1:], 1):
|
|
778
|
+
if ch in "aEdnPsIL": # takes a value: fused (`-I{}`, `-n1`) or the next word
|
|
779
|
+
consumed_next = j + 1 == len(a)
|
|
780
|
+
break
|
|
781
|
+
if ch in "li": # optional fused value only
|
|
782
|
+
break
|
|
783
|
+
i += 2 if consumed_next else 1
|
|
784
|
+
continue
|
|
785
|
+
break
|
|
786
|
+
return args[i:]
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
_PARALLEL_VALUED = {"-j", "--jobs", "-P", "-n", "--max-args", "-N", "-L", "-a", "--arg-file", "--arg-file-sep", "--arg-sep", "-S",
|
|
790
|
+
"--sshlogin", "--sshloginfile", "--slf", "--joblog", "--results", "--res", "--colsep", "-C", "--halt",
|
|
791
|
+
"--retries", "--timeout", "--delay", "--header", "--tagstring", "--workdir", "--wd", "--tmpdir", "--tmp",
|
|
792
|
+
"-I", "--rpl"}
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _unwrap_parallel(args: list[str]) -> list[str]:
|
|
796
|
+
rest = _drop_options(args, _PARALLEL_VALUED)
|
|
797
|
+
out = []
|
|
798
|
+
for a in rest:
|
|
799
|
+
if a in (":::", "::::", ":::+"):
|
|
800
|
+
break
|
|
801
|
+
out.append(a)
|
|
802
|
+
return out
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _unwrap_find(args: list[str]) -> list[list[str]]:
|
|
806
|
+
"""Commands of every -exec / -execdir / -ok / -okdir in a find argument list."""
|
|
807
|
+
cmds: list[list[str]] = []
|
|
808
|
+
i = 0
|
|
809
|
+
while i < len(args):
|
|
810
|
+
if args[i] in ("-exec", "-execdir", "-ok", "-okdir"):
|
|
811
|
+
j = i + 1
|
|
812
|
+
cur: list[str] = []
|
|
813
|
+
while j < len(args) and args[j] not in (";", "+", "\\;"):
|
|
814
|
+
cur.append(args[j])
|
|
815
|
+
j += 1
|
|
816
|
+
if cur:
|
|
817
|
+
cmds.append(cur)
|
|
818
|
+
i = j
|
|
819
|
+
i += 1
|
|
820
|
+
return cmds
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
_UV_VALUED = {
|
|
824
|
+
"--with", "--with-requirements", "--with-editable", "--python", "-p", "--project", "--directory", "--env-file", "--extra",
|
|
825
|
+
"--group", "--no-group", "--only-group", "--package", "--index", "--index-url", "-i", "--default-index", "--extra-index-url",
|
|
826
|
+
"-f", "--find-links", "--index-strategy", "--keyring-provider", "--resolution", "--prerelease", "--exclude-newer",
|
|
827
|
+
"--exclude-newer-package", "--refresh-package", "--reinstall-package", "--upgrade-package", "-P", "--no-binary-package",
|
|
828
|
+
"--no-build-package", "--link-mode", "--config-setting", "-C", "--cache-dir", "--color", "--config-file", "--constraint", "-c",
|
|
829
|
+
"--override", "--build-constraint", "--python-platform", "--fork-strategy", "--python-preference", "--python-fetch",
|
|
830
|
+
"--allow-insecure-host", "--from", "--script", "-s", "--no-extra", "--torch-backend", "--lfs"}
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _after_run(a: list[str], valued: set[str] | frozenset[str]) -> list[str] | None:
|
|
834
|
+
"""Words after the first `run` word, past the runner's own options (global ones before it are skipped by position)."""
|
|
835
|
+
for k, w in enumerate(a):
|
|
836
|
+
if w == "run":
|
|
837
|
+
return _drop_options(a[k + 1:], valued)
|
|
838
|
+
return None
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _hatch(a: list[str]) -> list[str] | None:
|
|
842
|
+
r = _after_run(a, set())
|
|
843
|
+
if r:
|
|
844
|
+
m = re.match(r"^[A-Za-z0-9_.-]+:(.+)$", r[0])
|
|
845
|
+
if m:
|
|
846
|
+
r = [m.group(1), *r[1:]]
|
|
847
|
+
return r
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
_RUNNER_PREFIX = {
|
|
851
|
+
"uv": lambda a: _after_run(a, _UV_VALUED),
|
|
852
|
+
"uvx": lambda a: _drop_options(a, _UV_VALUED),
|
|
853
|
+
"poetry": lambda a: _after_run(a, set()),
|
|
854
|
+
"pipenv": lambda a: _after_run(a, set()),
|
|
855
|
+
"pdm": lambda a: _after_run(a, {"--venv"}),
|
|
856
|
+
"hatch": _hatch,
|
|
857
|
+
"rye": lambda a: _after_run(a, set()),
|
|
858
|
+
"pipx": lambda a: _drop_options(a[1:], {"--spec", "--python", "--pip-args"}) if a and a[0] == "run" else None,
|
|
859
|
+
"conda": lambda a: _drop_options(a[1:], {"-n", "--name", "-p", "--prefix", "--cwd"}) if a and a[0] == "run" else None,
|
|
860
|
+
"mamba": lambda a: _drop_options(a[1:], {"-n", "--name", "-p", "--prefix", "--cwd"}) if a and a[0] == "run" else None,
|
|
861
|
+
"micromamba": lambda a: _drop_options(a[1:], {"-n", "--name", "-p", "--prefix", "--cwd"}) if a and a[0] == "run" else None,
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
_DOCKER_GLOBAL_VALUED = {"-H", "--host", "-c", "--context", "-l", "--log-level", "--config", "--tlscacert", "--tlscert", "--tlskey"}
|
|
866
|
+
_COMPOSE_GLOBAL_VALUED = {"-f", "--file", "-p", "--project-name", "--profile", "--env-file", "--project-directory", "--parallel",
|
|
867
|
+
"--ansi", "-H", "--host", "-c", "--context", "--progress"}
|
|
868
|
+
_DOCKER_EXEC_VALUED = {"-e", "--env", "--env-file", "-u", "--user", "-w", "--workdir", "--detach-keys", "--index"}
|
|
869
|
+
_DOCKER_RUN_VALUED = {
|
|
870
|
+
"-a", "--attach", "--add-host", "--blkio-weight", "--cap-add", "--cap-drop", "--cgroup-parent", "--cidfile", "--cpu-period",
|
|
871
|
+
"--cpu-quota", "--cpu-shares", "--cpus", "-c", "--cpuset-cpus", "--cpuset-mems", "--detach-keys", "--device", "--dns",
|
|
872
|
+
"--dns-option", "--dns-search", "-e", "--env", "--env-file", "--expose", "--gpus", "--group-add", "--health-cmd",
|
|
873
|
+
"--health-interval", "--health-retries", "--health-start-period", "--health-timeout", "-h", "--hostname", "--ip", "--ip6",
|
|
874
|
+
"--ipc", "--isolation", "-l", "--label", "--label-file", "--link", "--log-driver", "--log-opt", "--mac-address", "-m",
|
|
875
|
+
"--memory", "--memory-reservation", "--memory-swap", "--mount", "--name", "--network", "--net", "--network-alias",
|
|
876
|
+
"--oom-score-adj", "--pid", "--pids-limit", "--platform", "-p", "--publish", "--pull", "--restart", "--runtime",
|
|
877
|
+
"--security-opt", "--shm-size", "--stop-signal", "--stop-timeout", "--storage-opt", "--sysctl", "--tmpfs", "-u", "--user",
|
|
878
|
+
"--userns", "--uts", "-v", "--volume", "--volume-driver", "--volumes-from", "-w", "--workdir", "--entrypoint"}
|
|
879
|
+
_COMPOSE_RUN_VALUED = {"-e", "--env", "-v", "--volume", "-p", "--publish", "-u", "--user", "-w", "--workdir", "--name",
|
|
880
|
+
"--entrypoint", "-l", "--label", "--cap-add", "--cap-drop", "--use-aliases", "--pull"}
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
_IMAGE_CLI = {"aws-cli": "aws", "terraform": "terraform", "opentofu": "tofu", "tofu": "tofu", "kubectl": "kubectl", "helm": "helm",
|
|
884
|
+
"airflow": "airflow", "pulumi": "pulumi", "google-cloud-cli": "gcloud", "cloud-sdk": "gcloud", "azure-cli": "az",
|
|
885
|
+
"databricks-cli": "databricks", "snowflake-cli": "snow", "dbt": "dbt", "dagster": "dagster", "prefect": "prefect"}
|
|
886
|
+
NOARGS = "\0noargs"
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def _image_cli(image: str) -> str | None:
|
|
890
|
+
"""The guarded CLI a container image's entrypoint is (registry, tag and digest stripped), else None."""
|
|
891
|
+
n = image.split("@", 1)[0].rsplit("/", 1)[-1].split(":", 1)[0].lower()
|
|
892
|
+
if n.startswith("dbt-"):
|
|
893
|
+
return "dbt"
|
|
894
|
+
if n.endswith("-kubectl"):
|
|
895
|
+
return "kubectl"
|
|
896
|
+
return _IMAGE_CLI.get(n)
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _entry_cmd(args: list[str], valued: set[str], positionals_before_cmd: int, image: bool = False) -> list[str] | None:
|
|
900
|
+
"""Words of the inner command after options and `positionals_before_cmd` positionals (container, image or service).
|
|
901
|
+
With `image`, the first positional is an image (or compose service): one whose entrypoint is a guarded CLI runs that CLI."""
|
|
902
|
+
i = 0
|
|
903
|
+
seen = 0
|
|
904
|
+
first: str | None = None
|
|
905
|
+
entry: str | None = None
|
|
906
|
+
while i < len(args):
|
|
907
|
+
a = args[i]
|
|
908
|
+
if a == "--":
|
|
909
|
+
if seen >= positionals_before_cmd:
|
|
910
|
+
i += 1
|
|
911
|
+
break
|
|
912
|
+
i += 1
|
|
913
|
+
continue
|
|
914
|
+
if a.startswith("-") and len(a) > 1:
|
|
915
|
+
name = a.split("=", 1)[0]
|
|
916
|
+
if name in ("--entrypoint",):
|
|
917
|
+
entry = a.split("=", 1)[1] if "=" in a else (args[i + 1] if i + 1 < len(args) else None)
|
|
918
|
+
if "=" not in a and name in valued:
|
|
919
|
+
i += 2
|
|
920
|
+
else:
|
|
921
|
+
i += 1
|
|
922
|
+
continue
|
|
923
|
+
if first is None:
|
|
924
|
+
first = a
|
|
925
|
+
seen += 1
|
|
926
|
+
i += 1
|
|
927
|
+
if seen == positionals_before_cmd:
|
|
928
|
+
break
|
|
929
|
+
inner = args[i:]
|
|
930
|
+
if entry:
|
|
931
|
+
inner = [entry, *inner]
|
|
932
|
+
elif image and first:
|
|
933
|
+
cli = _image_cli(first)
|
|
934
|
+
if cli and not (inner and _base(inner[0]) == cli):
|
|
935
|
+
inner = [cli, *inner] if inner else [NOARGS, cli]
|
|
936
|
+
return inner or None
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def _container_inner(name: str, args: list[str]) -> list[str] | None:
|
|
940
|
+
"""For docker / podman / docker compose: the inner command of `exec` / `run`, or None for other verbs."""
|
|
941
|
+
if name in ("docker-compose",):
|
|
942
|
+
args = ["compose", *args]
|
|
943
|
+
if args and args[0] == "compose":
|
|
944
|
+
rest = _drop_options(args[1:], _COMPOSE_GLOBAL_VALUED)
|
|
945
|
+
if not rest or rest[0] not in ("exec", "run"):
|
|
946
|
+
return None
|
|
947
|
+
valued = _DOCKER_EXEC_VALUED if rest[0] == "exec" else _COMPOSE_RUN_VALUED
|
|
948
|
+
return _entry_cmd(rest[1:], valued, 1, rest[0] == "run")
|
|
949
|
+
rest = _drop_options(args, _DOCKER_GLOBAL_VALUED)
|
|
950
|
+
if rest and rest[0] == "container":
|
|
951
|
+
rest = rest[1:]
|
|
952
|
+
if not rest or rest[0] not in ("exec", "run", "create"):
|
|
953
|
+
return None
|
|
954
|
+
valued = _DOCKER_EXEC_VALUED if rest[0] == "exec" else _DOCKER_RUN_VALUED
|
|
955
|
+
return _entry_cmd(rest[1:], valued, 1, rest[0] != "exec")
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def _kubectl_exec_inner(args: list[str]) -> list[str] | None:
|
|
959
|
+
pos_args = args
|
|
960
|
+
if "--" in pos_args:
|
|
961
|
+
return pos_args[pos_args.index("--") + 1:]
|
|
962
|
+
pos, _ = scan(pos_args, _KUBECTL_EXEC_VALUED)
|
|
963
|
+
return pos[1:] or None # deprecated form: kubectl exec POD CMD
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
_SSH_VALUED = set("bcDEeFIiJLlmOopQRSWw")
|
|
967
|
+
_SSH_CMD_OPT = re.compile(r"\s*(?:proxycommand|localcommand|remotecommand)\s*[=\s]\s*(.*)$", re.I | re.S)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _ssh_parse(args: list[str]) -> tuple[list[str] | None, list[str]]:
|
|
971
|
+
"""(remote command words or None, commands carried by -o ProxyCommand / LocalCommand / RemoteCommand)."""
|
|
972
|
+
opt_cmds: list[str] = []
|
|
973
|
+
state = {"i": 0}
|
|
974
|
+
|
|
975
|
+
def option(a: str) -> None:
|
|
976
|
+
i = state["i"]
|
|
977
|
+
for k, ch in enumerate(a[1:], 1):
|
|
978
|
+
if ch in _SSH_VALUED:
|
|
979
|
+
val = a[k + 1:] if k + 1 < len(a) else (args[i + 1] if i + 1 < len(args) else "")
|
|
980
|
+
state["i"] += 0 if k + 1 < len(a) else 1
|
|
981
|
+
m = _SSH_CMD_OPT.match(val) if ch == "o" else None
|
|
982
|
+
if m:
|
|
983
|
+
opt_cmds.append(m.group(1))
|
|
984
|
+
return
|
|
985
|
+
|
|
986
|
+
host_seen = False
|
|
987
|
+
while state["i"] < len(args):
|
|
988
|
+
a = args[state["i"]]
|
|
989
|
+
if a == "--":
|
|
990
|
+
state["i"] += 1
|
|
991
|
+
if host_seen:
|
|
992
|
+
break
|
|
993
|
+
host_seen = True # the next word is the host
|
|
994
|
+
state["i"] += 1
|
|
995
|
+
continue
|
|
996
|
+
if a.startswith("-") and len(a) > 1:
|
|
997
|
+
option(a)
|
|
998
|
+
state["i"] += 1
|
|
999
|
+
continue
|
|
1000
|
+
if host_seen:
|
|
1001
|
+
break
|
|
1002
|
+
host_seen = True
|
|
1003
|
+
state["i"] += 1
|
|
1004
|
+
rest = args[state["i"]:] if host_seen else []
|
|
1005
|
+
return (rest or None), opt_cmds
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _shell_files(args: list[str]) -> list[str]:
|
|
1009
|
+
"""Non-option words of a shell invocation (the script first), past valued options such as `-o pipefail`."""
|
|
1010
|
+
out: list[str] = []
|
|
1011
|
+
i = 0
|
|
1012
|
+
while i < len(args):
|
|
1013
|
+
a = args[i]
|
|
1014
|
+
if a in ("-o", "-O", "+o", "+O", "--rcfile", "--init-file"):
|
|
1015
|
+
i += 2
|
|
1016
|
+
continue
|
|
1017
|
+
if a == "--":
|
|
1018
|
+
out.extend(args[i + 1:])
|
|
1019
|
+
break
|
|
1020
|
+
if a.startswith(("-", "+")) or a == "<":
|
|
1021
|
+
i += 1
|
|
1022
|
+
continue
|
|
1023
|
+
out.append(a)
|
|
1024
|
+
i += 1
|
|
1025
|
+
return out
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _shell_c(args: list[str]) -> str | None:
|
|
1029
|
+
"""The string after -c in `bash -lc 'cmd'` style options."""
|
|
1030
|
+
i = 0
|
|
1031
|
+
while i < len(args):
|
|
1032
|
+
a = args[i]
|
|
1033
|
+
if a in ("-o", "-O", "+o", "+O", "--rcfile", "--init-file"):
|
|
1034
|
+
i += 2
|
|
1035
|
+
continue
|
|
1036
|
+
if re.fullmatch(r"-[A-Za-z]*c[A-Za-z]*", a):
|
|
1037
|
+
return args[i + 1] if i + 1 < len(args) else ""
|
|
1038
|
+
if a.startswith("-") or a.startswith("+"):
|
|
1039
|
+
i += 1
|
|
1040
|
+
continue
|
|
1041
|
+
return None
|
|
1042
|
+
return None
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
# ---------------------------------------------------------------- small wrapper helpers
|
|
1046
|
+
|
|
1047
|
+
_REDIR_WORDS = {">", ">>", ">&", "&>", "&>>", "<", "<&", ">|", "<>", "<<<", "<<"}
|
|
1048
|
+
_WRITE_REDIR = {">", ">>", ">|", "&>", "&>>", "<>"}
|
|
1049
|
+
WRITE_RUN = ("de-agent: this command line writes a script and then runs it; write the file in one command and run it in a later "
|
|
1050
|
+
"one, or put the commands in a plan as a shell step")
|
|
1051
|
+
_TMUX_SHELL = {"new", "new-session", "neww", "new-window", "split-window", "splitw", "run", "run-shell", "respawn-pane",
|
|
1052
|
+
"respawnp", "respawn-window", "respawnw", "display-popup", "popup", "pipe-pane", "pipep", "if-shell", "if",
|
|
1053
|
+
"send-keys", "send"}
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _drop_clusters(args: list[str], letters: str, long_valued: set[str] | frozenset[str] = frozenset()) -> list[str]:
|
|
1057
|
+
"""Words after leading options; in a short cluster the first letter in `letters` takes a value (fused, or the next word)."""
|
|
1058
|
+
i = 0
|
|
1059
|
+
while i < len(args):
|
|
1060
|
+
a = args[i]
|
|
1061
|
+
if a == "--":
|
|
1062
|
+
return args[i + 1:]
|
|
1063
|
+
if a.startswith("--"):
|
|
1064
|
+
i += 2 if (a in long_valued) else 1
|
|
1065
|
+
continue
|
|
1066
|
+
if a.startswith("-") and len(a) > 1:
|
|
1067
|
+
k = next((j for j, ch in enumerate(a[1:], 1) if ch in letters), None)
|
|
1068
|
+
i += 2 if (k is not None and k + 1 == len(a)) else 1
|
|
1069
|
+
continue
|
|
1070
|
+
break
|
|
1071
|
+
return args[i:]
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def _su_command(args: list[str]) -> str | None:
|
|
1075
|
+
"""The command of `su -c X`, `su -lc X`, `su --command=X`, `runuser -c X`."""
|
|
1076
|
+
for k, a in enumerate(args):
|
|
1077
|
+
if a.startswith("--command="):
|
|
1078
|
+
return a.split("=", 1)[1]
|
|
1079
|
+
if a == "--command":
|
|
1080
|
+
return args[k + 1] if k + 1 < len(args) else ""
|
|
1081
|
+
if a.startswith("-") and not a.startswith("--") and "c" in a[1:]:
|
|
1082
|
+
j = a.index("c", 1)
|
|
1083
|
+
return a[j + 1:] if j + 1 < len(a) else (args[k + 1] if k + 1 < len(args) else "")
|
|
1084
|
+
return None
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def _script_cmd(args: list[str]) -> list[str] | str:
|
|
1088
|
+
"""`script`: util-linux `-c CMD` (a string), else the macOS form `script [opts] file cmd...` (a word list)."""
|
|
1089
|
+
for k, a in enumerate(args):
|
|
1090
|
+
if a.startswith("--command="):
|
|
1091
|
+
return a.split("=", 1)[1]
|
|
1092
|
+
if a == "--command":
|
|
1093
|
+
return args[k + 1] if k + 1 < len(args) else ""
|
|
1094
|
+
if a.startswith("-") and not a.startswith("--") and a.endswith("c") and k + 1 < len(args):
|
|
1095
|
+
return args[k + 1]
|
|
1096
|
+
return _drop_options(args, {"-t", "-T", "-I", "-O", "-B", "-E", "-m", "-F", "-o"})[1:]
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def _cron_lines(text: str) -> list[str]:
|
|
1100
|
+
"""Command part of each crontab line (the five time fields or `@word` dropped)."""
|
|
1101
|
+
out = []
|
|
1102
|
+
for line in text.splitlines():
|
|
1103
|
+
line = line.strip()
|
|
1104
|
+
if not line or line.startswith("#") or _ASSIGN.match(line):
|
|
1105
|
+
continue
|
|
1106
|
+
parts = line.split(None, 1 if line.startswith("@") else 5)
|
|
1107
|
+
out.append(parts[-1] if len(parts) > (1 if line.startswith("@") else 5) else "")
|
|
1108
|
+
return out
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
# ---------------------------------------------------------------- the walker
|
|
1112
|
+
|
|
1113
|
+
def _subst(word: str) -> list[str]:
|
|
1114
|
+
"""Command texts inside `$( ... )`, backticks and process substitutions of one shell word."""
|
|
1115
|
+
out: list[str] = []
|
|
1116
|
+
i, n = 0, len(word)
|
|
1117
|
+
while i < n:
|
|
1118
|
+
if word.startswith("$(", i) or word.startswith("<(", i) or word.startswith(">(", i):
|
|
1119
|
+
depth, j = 1, i + 2
|
|
1120
|
+
while j < n and depth:
|
|
1121
|
+
depth += (word[j] == "(") - (word[j] == ")")
|
|
1122
|
+
j += 1
|
|
1123
|
+
out.append(word[i + 2:j - 1])
|
|
1124
|
+
i = j
|
|
1125
|
+
continue
|
|
1126
|
+
if word[i] == "`":
|
|
1127
|
+
j = word.find("`", i + 1)
|
|
1128
|
+
j = n if j == -1 else j
|
|
1129
|
+
out.append(word[i + 1:j])
|
|
1130
|
+
i = j + 1
|
|
1131
|
+
continue
|
|
1132
|
+
i += 1
|
|
1133
|
+
return out
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def _sql_client(exe: str) -> bool:
|
|
1137
|
+
return exe in DB_CLIENTS or exe in ("dea", "dbsqlcli")
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
class Walker:
|
|
1141
|
+
def __init__(self, cwd: str | None, allow: list[list[str]], bodies: list[str] | None = None) -> None:
|
|
1142
|
+
self.cwd = cwd
|
|
1143
|
+
self.allow = allow
|
|
1144
|
+
self.bodies = bodies or []
|
|
1145
|
+
self.out: list[Verdict] = []
|
|
1146
|
+
self.written: set[str] = set()
|
|
1147
|
+
|
|
1148
|
+
# ---- command text
|
|
1149
|
+
def command(self, text: str, depth: int) -> None:
|
|
1150
|
+
from de_agent.extract import _preprocess, _split_simple, _tokenize
|
|
1151
|
+
|
|
1152
|
+
if depth > MAX_DEPTH:
|
|
1153
|
+
if GUARD_NAME_RE.search(text):
|
|
1154
|
+
self.out.append(Verdict("unparseable", "shell", "", "de-agent: command is nested too deeply to check"))
|
|
1155
|
+
return
|
|
1156
|
+
if not text.strip():
|
|
1157
|
+
return
|
|
1158
|
+
try:
|
|
1159
|
+
pre, bodies = _preprocess(text)
|
|
1160
|
+
tokens = _tokenize(pre)
|
|
1161
|
+
except ValueError:
|
|
1162
|
+
if GUARD_NAME_RE.search(text): # a command we cannot tokenise that mentions a guarded CLI: fail closed
|
|
1163
|
+
self.out.append(Verdict("unparseable", "shell", ""))
|
|
1164
|
+
return
|
|
1165
|
+
for inner in _subst(text): # `$( ... )`, backticks and process substitutions run their own commands
|
|
1166
|
+
self.command(inner, depth + 1)
|
|
1167
|
+
saved = self.bodies
|
|
1168
|
+
self.bodies = bodies
|
|
1169
|
+
try:
|
|
1170
|
+
for piped, words in _split_simple(tokens):
|
|
1171
|
+
self.words(words, depth, piped)
|
|
1172
|
+
finally:
|
|
1173
|
+
self.bodies = saved
|
|
1174
|
+
|
|
1175
|
+
# ---- one simple command
|
|
1176
|
+
def words(self, words: list[str], depth: int, piped: bool = False) -> None:
|
|
1177
|
+
args = list(words)
|
|
1178
|
+
self.note_writes(args)
|
|
1179
|
+
i = 0
|
|
1180
|
+
while i < len(args):
|
|
1181
|
+
a = args[i]
|
|
1182
|
+
if a == "function": # `function f { ... }`: the body is parsed as it stands
|
|
1183
|
+
i += 2
|
|
1184
|
+
elif a == "coproc":
|
|
1185
|
+
i += 2 if i + 2 < len(args) and args[i + 2] == "{" else 1
|
|
1186
|
+
elif a in _REDIR_WORDS: # leading redirect: `>/dev/null aws s3 rm ...`
|
|
1187
|
+
i += 2
|
|
1188
|
+
elif a.isdigit() and i + 1 < len(args) and args[i + 1] in _REDIR_WORDS:
|
|
1189
|
+
i += 1
|
|
1190
|
+
elif _ASSIGN.match(a) or a in _CONTROL:
|
|
1191
|
+
i += 1
|
|
1192
|
+
else:
|
|
1193
|
+
break
|
|
1194
|
+
args = args[i:]
|
|
1195
|
+
if not args:
|
|
1196
|
+
return
|
|
1197
|
+
if DOLLAR_PROG_RE.match(args[0]):
|
|
1198
|
+
self.out.append(Verdict("unparseable", "shell", "", "de-agent: a program name built from a variable after a guarded CLI "
|
|
1199
|
+
"name (`aws${IFS}...`) cannot be checked; write the command out"))
|
|
1200
|
+
return
|
|
1201
|
+
# `find ... -exec` continuation segments (the `;` terminator split the command)
|
|
1202
|
+
if args[0] in ("-exec", "-execdir", "-ok", "-okdir"):
|
|
1203
|
+
for cmd in _unwrap_find(args):
|
|
1204
|
+
self.words(cmd, depth + 1)
|
|
1205
|
+
return
|
|
1206
|
+
name = _base(args[0])
|
|
1207
|
+
rest = args[1:]
|
|
1208
|
+
self.program(name, rest, args, depth, piped)
|
|
1209
|
+
|
|
1210
|
+
def inner(self, cmd: list[str] | None, depth: int) -> None:
|
|
1211
|
+
if cmd:
|
|
1212
|
+
self.words(cmd, depth + 1)
|
|
1213
|
+
|
|
1214
|
+
# ---- files written by the command line itself, and scripts run directly
|
|
1215
|
+
def norm(self, path: str) -> str:
|
|
1216
|
+
return os.path.normpath(os.path.join(self.cwd or os.getcwd(), os.path.expanduser(path)))
|
|
1217
|
+
|
|
1218
|
+
def note_writes(self, args: list[str]) -> None:
|
|
1219
|
+
for k, a in enumerate(args):
|
|
1220
|
+
if a in _WRITE_REDIR and k + 1 < len(args):
|
|
1221
|
+
self.written.add(self.norm(args[k + 1]))
|
|
1222
|
+
elif a.startswith("of=") and len(a) > 3:
|
|
1223
|
+
self.written.add(self.norm(a[3:]))
|
|
1224
|
+
elif _base(a) == "tee":
|
|
1225
|
+
self.written.update(self.norm(w) for w in args[k + 1:] if not w.startswith("-") and w not in _REDIR_WORDS)
|
|
1226
|
+
elif _base(a) in ("cp", "mv", "install", "ln"):
|
|
1227
|
+
tail = [w for w in args[k + 1:] if not w.startswith("-") and w not in _REDIR_WORDS]
|
|
1228
|
+
if len(tail) >= 2:
|
|
1229
|
+
self.written.add(self.norm(tail[-1]))
|
|
1230
|
+
|
|
1231
|
+
def direct(self, path: str, depth: int) -> None:
|
|
1232
|
+
"""`./s.sh`, `bin/wrap`: a shell script under the working directory is scanned like `bash s.sh`."""
|
|
1233
|
+
p = self.norm(path)
|
|
1234
|
+
if p in self.written:
|
|
1235
|
+
self.out.append(Verdict("unparseable", "shell", "", WRITE_RUN))
|
|
1236
|
+
return
|
|
1237
|
+
try:
|
|
1238
|
+
base = os.path.realpath(self.cwd or os.getcwd())
|
|
1239
|
+
real = os.path.realpath(p)
|
|
1240
|
+
if os.path.commonpath([base, real]) != base or not os.path.isfile(real):
|
|
1241
|
+
return
|
|
1242
|
+
with open(real, "rb") as f:
|
|
1243
|
+
head = f.read(256)
|
|
1244
|
+
except (OSError, ValueError):
|
|
1245
|
+
return
|
|
1246
|
+
if b"\0" in head:
|
|
1247
|
+
return
|
|
1248
|
+
first = head.split(b"\n", 1)[0].decode("utf-8", "replace")
|
|
1249
|
+
if first.startswith("#!"):
|
|
1250
|
+
if re.search(r"(?:^|[/\s])(?:ba|z|da|k|a)?sh(?:\s|$)", first):
|
|
1251
|
+
self.script(path, depth)
|
|
1252
|
+
elif "python" in first:
|
|
1253
|
+
self.python([path], depth)
|
|
1254
|
+
elif real.endswith(".sh"):
|
|
1255
|
+
self.script(path, depth)
|
|
1256
|
+
|
|
1257
|
+
def split_stdin(self, words: list[str]) -> tuple[list[str], list[str]]:
|
|
1258
|
+
"""(words without here-doc / here-string / other redirects, the stdin texts they carried)."""
|
|
1259
|
+
from de_agent.extract import _strip_redirects
|
|
1260
|
+
|
|
1261
|
+
clean: list[str] = []
|
|
1262
|
+
texts: list[str] = []
|
|
1263
|
+
k = 0
|
|
1264
|
+
while k < len(words):
|
|
1265
|
+
w = words[k]
|
|
1266
|
+
if w == "<<<" and k + 1 < len(words):
|
|
1267
|
+
texts.append(words[k + 1])
|
|
1268
|
+
k += 2
|
|
1269
|
+
elif w == "<<" and k + 1 < len(words) and PLACEHOLDER.fullmatch(words[k + 1]):
|
|
1270
|
+
idx = int(PLACEHOLDER.fullmatch(words[k + 1]).group(1)) # type: ignore[union-attr]
|
|
1271
|
+
if idx < len(self.bodies):
|
|
1272
|
+
texts.append(self.bodies[idx])
|
|
1273
|
+
k += 2
|
|
1274
|
+
else:
|
|
1275
|
+
clean.append(w)
|
|
1276
|
+
k += 1
|
|
1277
|
+
return _strip_redirects(clean), texts
|
|
1278
|
+
|
|
1279
|
+
def tmux(self, rest: list[str], depth: int) -> None:
|
|
1280
|
+
i = 0
|
|
1281
|
+
while i < len(rest) and rest[i].startswith("-"):
|
|
1282
|
+
i += 2 if rest[i] in ("-L", "-S", "-f", "-c", "-T") else 1
|
|
1283
|
+
if i >= len(rest) or rest[i] not in _TMUX_SHELL:
|
|
1284
|
+
return
|
|
1285
|
+
sub = rest[i]
|
|
1286
|
+
valued = {"-c", "-n", "-s", "-t", "-x", "-y", "-e", "-F", "-l", "-f", "-E", "-S", "-h", "-w", "-N", "-p"}
|
|
1287
|
+
if sub in ("display-popup", "popup"):
|
|
1288
|
+
valued |= {"-d", "-b", "-B", "-T"}
|
|
1289
|
+
cmd = _drop_options(rest[i + 1:], valued)
|
|
1290
|
+
if sub in ("if-shell", "if"):
|
|
1291
|
+
for c in cmd:
|
|
1292
|
+
self.command(c, depth + 1)
|
|
1293
|
+
elif cmd:
|
|
1294
|
+
self.command(" ".join(cmd), depth + 1)
|
|
1295
|
+
|
|
1296
|
+
def deferred(self, name: str, rest: list[str], depth: int, piped: bool) -> None:
|
|
1297
|
+
"""at / batch / crontab: code run later, read here from a here-string, here-doc, `-f file` or file argument."""
|
|
1298
|
+
clean, texts = self.split_stdin(rest)
|
|
1299
|
+
pos = _drop_options(clean, {"-f", "-u", "-q", "-m"} if name != "crontab" else {"-u"})
|
|
1300
|
+
found = bool(texts)
|
|
1301
|
+
if name == "crontab":
|
|
1302
|
+
if pos and pos[0] != "-":
|
|
1303
|
+
found = True
|
|
1304
|
+
try:
|
|
1305
|
+
texts = [open(os.path.expanduser(pos[0]), encoding="utf-8", errors="replace").read()]
|
|
1306
|
+
except OSError:
|
|
1307
|
+
texts = []
|
|
1308
|
+
for t in texts:
|
|
1309
|
+
for c in _cron_lines(t):
|
|
1310
|
+
self.command(c, depth + 1)
|
|
1311
|
+
reads_stdin = bool(pos) and pos[0] == "-"
|
|
1312
|
+
else:
|
|
1313
|
+
for k, a in enumerate(clean):
|
|
1314
|
+
if a == "-f" and k + 1 < len(clean):
|
|
1315
|
+
found = True
|
|
1316
|
+
self.script(clean[k + 1], depth)
|
|
1317
|
+
for t in texts:
|
|
1318
|
+
self.command(t, depth + 1)
|
|
1319
|
+
reads_stdin = not found
|
|
1320
|
+
if piped and reads_stdin and not texts and GUARD_NAME_RE.search(self.full_text):
|
|
1321
|
+
self.out.append(Verdict("unparseable", "shell", "", PIPED_SHELL))
|
|
1322
|
+
|
|
1323
|
+
def program(self, name: str, rest: list[str], args: list[str], depth: int, piped: bool) -> None:
|
|
1324
|
+
if name in ("env",):
|
|
1325
|
+
r = _unwrap_env(rest)
|
|
1326
|
+
if isinstance(r, tuple):
|
|
1327
|
+
self.command(r[1], depth + 1)
|
|
1328
|
+
else:
|
|
1329
|
+
self.inner(r, depth)
|
|
1330
|
+
return
|
|
1331
|
+
if name in ("sudo", "doas", "pkexec"):
|
|
1332
|
+
self.inner(_unwrap_sudo(rest), depth)
|
|
1333
|
+
return
|
|
1334
|
+
if "/" in args[0] and not args[0].startswith(("-", "$")):
|
|
1335
|
+
self.direct(args[0], depth)
|
|
1336
|
+
if name in ("nohup", "exec", "builtin", "setsid", "unbuffer", "chronic", "time"):
|
|
1337
|
+
self.inner(_drop_options(rest, {"-f", "--format", "-o", "--output", "-a"} if name in ("time", "exec") else set()), depth)
|
|
1338
|
+
return
|
|
1339
|
+
if name == "command":
|
|
1340
|
+
if any(a in ("-v", "-V") for a in rest[:2]):
|
|
1341
|
+
return
|
|
1342
|
+
self.inner(_drop_options(rest, set()), depth)
|
|
1343
|
+
return
|
|
1344
|
+
if name == "caffeinate":
|
|
1345
|
+
self.inner(_drop_options(rest, {"-t", "-w"}), depth)
|
|
1346
|
+
return
|
|
1347
|
+
if name in ("ionice",):
|
|
1348
|
+
self.inner(_drop_options(rest, {"-c", "-n", "-p", "-P", "-u"}), depth)
|
|
1349
|
+
return
|
|
1350
|
+
if name == "chrt":
|
|
1351
|
+
self.inner(_drop_options(rest, set())[1:], depth)
|
|
1352
|
+
return
|
|
1353
|
+
if name == "taskset":
|
|
1354
|
+
r = _drop_options(rest, {"-c", "--cpu-list", "-p"})
|
|
1355
|
+
cpus = any(f.startswith("--cpu-list") or (f.startswith("-") and not f.startswith("--") and "c" in f[1:])
|
|
1356
|
+
for f in rest[:len(rest) - len(r)])
|
|
1357
|
+
self.inner(r if cpus else r[1:], depth) # the mask positional is absent when -c/--cpu-list gives the list
|
|
1358
|
+
return
|
|
1359
|
+
if name == "nice":
|
|
1360
|
+
self.inner(_unwrap_nice(rest), depth)
|
|
1361
|
+
return
|
|
1362
|
+
if name == "stdbuf":
|
|
1363
|
+
self.inner(_unwrap_stdbuf(rest), depth)
|
|
1364
|
+
return
|
|
1365
|
+
if name == "timeout":
|
|
1366
|
+
self.inner(_unwrap_timeout(rest), depth)
|
|
1367
|
+
return
|
|
1368
|
+
if name == "watch":
|
|
1369
|
+
r = _unwrap_watch(rest)
|
|
1370
|
+
if len(r) == 1 and " " in r[0]:
|
|
1371
|
+
self.command(r[0], depth + 1)
|
|
1372
|
+
else:
|
|
1373
|
+
self.inner(r, depth)
|
|
1374
|
+
return
|
|
1375
|
+
if name == "flock":
|
|
1376
|
+
self.inner(_unwrap_flock(rest), depth)
|
|
1377
|
+
return
|
|
1378
|
+
if name == "xargs":
|
|
1379
|
+
self.inner(_unwrap_xargs(rest), depth)
|
|
1380
|
+
return
|
|
1381
|
+
if name == "parallel":
|
|
1382
|
+
r = _unwrap_parallel(rest)
|
|
1383
|
+
if r and " " in r[0].strip():
|
|
1384
|
+
self.command(" ".join(r), depth + 1)
|
|
1385
|
+
elif r:
|
|
1386
|
+
self.inner(r, depth)
|
|
1387
|
+
else:
|
|
1388
|
+
for k, w in enumerate(rest): # `parallel ::: 'cmd1' 'cmd2'`: the arguments are the commands
|
|
1389
|
+
if w in (":::", ":::+"):
|
|
1390
|
+
for c in rest[k + 1:]:
|
|
1391
|
+
self.command(c, depth + 1)
|
|
1392
|
+
break
|
|
1393
|
+
return
|
|
1394
|
+
if name == "find":
|
|
1395
|
+
for cmd in _unwrap_find(rest):
|
|
1396
|
+
self.words(cmd, depth + 1)
|
|
1397
|
+
return
|
|
1398
|
+
if name in ("su", "runuser"):
|
|
1399
|
+
c = _su_command(rest)
|
|
1400
|
+
if c is not None:
|
|
1401
|
+
self.command(c, depth + 1)
|
|
1402
|
+
elif name == "runuser" and any(a in ("-u", "--user") for a in rest):
|
|
1403
|
+
self.inner(_drop_options(rest, {"-u", "--user", "-g", "--group", "-G", "--supp-group", "-s", "--shell"}), depth)
|
|
1404
|
+
return
|
|
1405
|
+
if name in ("strace", "ltrace"):
|
|
1406
|
+
self.inner(_drop_options(rest, set("-" + c for c in "oepusaEIOPSXbCW") | {"--output", "--trace", "--signal", "--env"}), depth)
|
|
1407
|
+
return
|
|
1408
|
+
if name == "chroot":
|
|
1409
|
+
self.inner(_drop_options(rest, {"--userspec", "--groups"})[1:], depth)
|
|
1410
|
+
return
|
|
1411
|
+
if name == "nsenter":
|
|
1412
|
+
self.inner(_drop_options(rest, {"-t", "--target", "-S", "--setuid", "-G", "--setgid"}), depth)
|
|
1413
|
+
return
|
|
1414
|
+
if name == "unshare":
|
|
1415
|
+
self.inner(_drop_options(rest, {"--propagation", "--setgroups", "-R", "--root", "-w", "--wd", "--forward-signals"}), depth)
|
|
1416
|
+
return
|
|
1417
|
+
if name == "busybox":
|
|
1418
|
+
self.inner(rest, depth)
|
|
1419
|
+
return
|
|
1420
|
+
if name == "systemd-run":
|
|
1421
|
+
self.inner(_drop_options(rest, {
|
|
1422
|
+
"-p", "--property", "-u", "--unit", "--slice", "-E", "--setenv", "--uid", "--gid", "--description", "--service-type",
|
|
1423
|
+
"--working-directory", "-H", "--host", "-M", "--machine", "--nice", "--timer-property", "--on-active", "--on-boot",
|
|
1424
|
+
"--on-startup", "--on-unit-active", "--on-unit-inactive", "--on-calendar", "--on-clock-change",
|
|
1425
|
+
"--on-timezone-change"}), depth)
|
|
1426
|
+
return
|
|
1427
|
+
if name == "script":
|
|
1428
|
+
c = _script_cmd(rest)
|
|
1429
|
+
if isinstance(c, str):
|
|
1430
|
+
self.command(c, depth + 1)
|
|
1431
|
+
else:
|
|
1432
|
+
self.inner(c, depth)
|
|
1433
|
+
return
|
|
1434
|
+
if name == "tmux":
|
|
1435
|
+
self.tmux(rest, depth)
|
|
1436
|
+
return
|
|
1437
|
+
if name == "screen":
|
|
1438
|
+
self.inner(_drop_clusters(rest, "SsctTphe"), depth)
|
|
1439
|
+
return
|
|
1440
|
+
if name == "trap":
|
|
1441
|
+
r = _drop_options(rest, set())
|
|
1442
|
+
if len(r) >= 2 and r[0] not in ("-", ""):
|
|
1443
|
+
self.command(r[0], depth + 1)
|
|
1444
|
+
return
|
|
1445
|
+
if name in ("at", "batch", "crontab"):
|
|
1446
|
+
self.deferred(name, rest, depth, piped)
|
|
1447
|
+
return
|
|
1448
|
+
if name in _RUNNER_PREFIX:
|
|
1449
|
+
r = _RUNNER_PREFIX[name](rest)
|
|
1450
|
+
if r is not None:
|
|
1451
|
+
self.inner(r, depth)
|
|
1452
|
+
return
|
|
1453
|
+
if name == "eval":
|
|
1454
|
+
self.command(" ".join(rest), depth + 1)
|
|
1455
|
+
return
|
|
1456
|
+
if name in _SHELLS:
|
|
1457
|
+
self.shell(name, rest, depth, piped)
|
|
1458
|
+
return
|
|
1459
|
+
if name in ("source", "."):
|
|
1460
|
+
if rest:
|
|
1461
|
+
self.script(rest[0], depth)
|
|
1462
|
+
return
|
|
1463
|
+
if name == "ssh":
|
|
1464
|
+
r, opt_cmds = _ssh_parse(rest)
|
|
1465
|
+
for c in opt_cmds:
|
|
1466
|
+
self.command(c, depth + 1)
|
|
1467
|
+
if r:
|
|
1468
|
+
clean, texts = self.split_stdin(r)
|
|
1469
|
+
self.command(" ".join(clean), depth + 1) # the remote shell re-parses the joined arguments
|
|
1470
|
+
if clean and _base(clean[0]) in _SHELLS:
|
|
1471
|
+
for t in texts:
|
|
1472
|
+
self.command(t, depth + 1)
|
|
1473
|
+
if piped and not texts and GUARD_NAME_RE.search(self.full_text):
|
|
1474
|
+
self.out.append(Verdict("unparseable", "shell", "", PIPED_SHELL))
|
|
1475
|
+
return
|
|
1476
|
+
if name in ("docker", "podman", "docker-compose", "nerdctl"):
|
|
1477
|
+
inner = _container_inner(name, rest)
|
|
1478
|
+
if inner and inner[0] == NOARGS:
|
|
1479
|
+
self.out.append(_u(inner[1], "", "a container image whose entrypoint is this CLI, run with no command"))
|
|
1480
|
+
else:
|
|
1481
|
+
self.inner(inner, depth)
|
|
1482
|
+
return
|
|
1483
|
+
if name == "kubectl":
|
|
1484
|
+
pos, _ = scan(rest, _KUBECTL_VALUED)
|
|
1485
|
+
if pos and pos[0] == "exec":
|
|
1486
|
+
k = rest.index("exec")
|
|
1487
|
+
inner = _kubectl_exec_inner(rest[k + 1:])
|
|
1488
|
+
self.kubectl_exec(inner, depth)
|
|
1489
|
+
return
|
|
1490
|
+
if name.startswith("python") and re.fullmatch(r"python[0-9.]*", name):
|
|
1491
|
+
self.python(rest, depth)
|
|
1492
|
+
return
|
|
1493
|
+
from de_agent.extract import _strip_redirects
|
|
1494
|
+
|
|
1495
|
+
rest = _strip_redirects(rest) # `2>&1`, `> file`: shell syntax, not arguments of the CLI
|
|
1496
|
+
v = _guarded(name, rest)
|
|
1497
|
+
if v is None:
|
|
1498
|
+
return
|
|
1499
|
+
self.out.append(self.apply_allow(name, [args[0], *rest], v))
|
|
1500
|
+
|
|
1501
|
+
def apply_allow(self, name: str, args: list[str], v: Verdict) -> Verdict:
|
|
1502
|
+
if v.kind in ("write", "unknown"):
|
|
1503
|
+
shaped = [_base(args[0]), *args[1:]]
|
|
1504
|
+
for p in self.allow:
|
|
1505
|
+
if len(shaped) >= len(p) and [_base(p[0]), *p[1:]] == shaped[:len(p)]:
|
|
1506
|
+
return Verdict("allow", v.cli, v.verb, "matched guard.cli_allow")
|
|
1507
|
+
return v
|
|
1508
|
+
|
|
1509
|
+
def kubectl_exec(self, inner: list[str] | None, depth: int) -> None:
|
|
1510
|
+
if not inner:
|
|
1511
|
+
self.out.append(_u("kubectl", "exec", "no command after `--`"))
|
|
1512
|
+
return
|
|
1513
|
+
before = len(self.out)
|
|
1514
|
+
self.words(inner, depth + 1)
|
|
1515
|
+
exe = _base(next((a for a in inner if not _ASSIGN.match(a)), ""))
|
|
1516
|
+
if len(self.out) == before and exe in _SHELLS and any(
|
|
1517
|
+
re.search(r"(?<![\w-])" + re.escape(c) + r"(?![\w-])", " ".join(inner)) for c in DB_CLIENTS):
|
|
1518
|
+
exe = "psql" # `bash -c "psql ..."`: sql_guard reads the client inside
|
|
1519
|
+
if len(self.out) == before and not _sql_client(exe):
|
|
1520
|
+
self.out.append(_u("kubectl", "exec", f"runs `{exe or '?'}` inside a pod"))
|
|
1521
|
+
elif len(self.out) == before:
|
|
1522
|
+
self.out.append(_s("kubectl", "exec"))
|
|
1523
|
+
|
|
1524
|
+
def shell(self, name: str, rest: list[str], depth: int, piped: bool) -> None:
|
|
1525
|
+
c = _shell_c(rest)
|
|
1526
|
+
if c is not None:
|
|
1527
|
+
self.command(c, depth + 1)
|
|
1528
|
+
return
|
|
1529
|
+
for k, w in enumerate(rest):
|
|
1530
|
+
if w == "<<" and k + 1 < len(rest) and PLACEHOLDER.fullmatch(rest[k + 1]):
|
|
1531
|
+
idx = int(PLACEHOLDER.fullmatch(rest[k + 1]).group(1)) # type: ignore[union-attr]
|
|
1532
|
+
if idx < len(self.bodies):
|
|
1533
|
+
self.command(self.bodies[idx], depth + 1)
|
|
1534
|
+
return
|
|
1535
|
+
if w == "<<<" and k + 1 < len(rest):
|
|
1536
|
+
self.command(rest[k + 1], depth + 1)
|
|
1537
|
+
return
|
|
1538
|
+
files = _shell_files(rest)
|
|
1539
|
+
if files:
|
|
1540
|
+
self.script(files[0], depth)
|
|
1541
|
+
return
|
|
1542
|
+
if piped and GUARD_NAME_RE.search(self.full_text):
|
|
1543
|
+
self.out.append(Verdict("unparseable", "shell", "", PIPED_SHELL))
|
|
1544
|
+
|
|
1545
|
+
full_text = ""
|
|
1546
|
+
|
|
1547
|
+
def script(self, path: str, depth: int) -> None:
|
|
1548
|
+
if self.norm(path) in self.written:
|
|
1549
|
+
self.out.append(Verdict("unparseable", "shell", "", WRITE_RUN))
|
|
1550
|
+
return
|
|
1551
|
+
if depth >= 3:
|
|
1552
|
+
return
|
|
1553
|
+
p = os.path.expanduser(path)
|
|
1554
|
+
if not os.path.isabs(p):
|
|
1555
|
+
p = os.path.join(self.cwd or os.getcwd(), p)
|
|
1556
|
+
try:
|
|
1557
|
+
if os.path.getsize(p) > MAX_SCRIPT:
|
|
1558
|
+
self.out.append(Verdict("unparseable", "shell", "", f"de-agent: script {path} is too large to check"))
|
|
1559
|
+
return
|
|
1560
|
+
text = open(p, encoding="utf-8", errors="replace").read()
|
|
1561
|
+
except OSError:
|
|
1562
|
+
return # a missing script: the shell itself fails, nothing runs
|
|
1563
|
+
self.command(text, depth + 1)
|
|
1564
|
+
|
|
1565
|
+
def python(self, rest: list[str], depth: int) -> None:
|
|
1566
|
+
"""Strings and argv lists in `python -c` / heredoc code that run a guarded CLI (subprocess, os.system)."""
|
|
1567
|
+
from de_agent.extract import _python_code
|
|
1568
|
+
|
|
1569
|
+
for k, a in enumerate(rest):
|
|
1570
|
+
if a in ("-W", "-X", "-Q", "--check-hash-based-pycs"):
|
|
1571
|
+
continue
|
|
1572
|
+
mod = rest[k + 1] if a == "-m" and k + 1 < len(rest) else (a[2:] if a.startswith("-m") and len(a) > 2 else None)
|
|
1573
|
+
if mod is not None:
|
|
1574
|
+
cli = _PY_MODULE_CLI.get(mod.split(".")[0])
|
|
1575
|
+
if cli:
|
|
1576
|
+
self.words([cli, *rest[k + 2:]], depth + 1)
|
|
1577
|
+
return
|
|
1578
|
+
break
|
|
1579
|
+
if not a.startswith("-"):
|
|
1580
|
+
break
|
|
1581
|
+
try:
|
|
1582
|
+
code, _why = _python_code(rest, False, self.bodies, self.cwd)
|
|
1583
|
+
except Exception:
|
|
1584
|
+
return
|
|
1585
|
+
for src in code:
|
|
1586
|
+
if src.startswith("<unreadable") or not _RUNS_PROCESSES_RE.search(src) or not GUARD_NAME_RE.search(src):
|
|
1587
|
+
continue
|
|
1588
|
+
try:
|
|
1589
|
+
tree = ast.parse(src)
|
|
1590
|
+
except (SyntaxError, ValueError):
|
|
1591
|
+
continue
|
|
1592
|
+
for node in ast.walk(tree):
|
|
1593
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str) and GUARD_NAME_RE.search(node.value):
|
|
1594
|
+
self.command(node.value, depth + 1)
|
|
1595
|
+
elif isinstance(node, (ast.List, ast.Tuple)) and node.elts and all(
|
|
1596
|
+
isinstance(e, ast.Constant) and isinstance(e.value, str) for e in node.elts):
|
|
1597
|
+
vals = [e.value for e in node.elts] # type: ignore[attr-defined]
|
|
1598
|
+
if any(_base(v) in GUARDED_CLIS for v in vals):
|
|
1599
|
+
self.words(vals, depth + 1)
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
GUARD_NAME_RE = re.compile(r"(?<![\w.-])(?:" + "|".join(re.escape(n) for n in sorted(GUARDED_CLIS, key=len, reverse=True)) +
|
|
1603
|
+
r")(?:\.sh)?(?![\w-])")
|
|
1604
|
+
DOLLAR_PROG_RE = re.compile(r"^(?:[^$/\s]*/)*(?:" + "|".join(re.escape(n) for n in sorted(GUARDED_CLIS, key=len, reverse=True)) +
|
|
1605
|
+
r")(?:\.sh)?\$")
|
|
1606
|
+
_PY_MODULE_CLI = {"awscli": "aws", "airflow": "airflow", "dbt": "dbt", "dagster": "dagster", "prefect": "prefect"}
|
|
1607
|
+
_RUNS_PROCESSES_RE = re.compile(r"subprocess|os\.system|os\.popen|Popen|check_output|check_call|pty\.spawn|\bsh\.|plumbum|asyncio\.create_subprocess")
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def parse_allow(entries: list[str] | tuple[str, ...]) -> list[list[str]]:
|
|
1611
|
+
"""`guard.cli_allow` strings -> word lists. An entry needs the CLI and a verb and no shell operators; others are ignored."""
|
|
1612
|
+
out: list[list[str]] = []
|
|
1613
|
+
for e in entries:
|
|
1614
|
+
if not isinstance(e, str) or re.search(r"[;&|<>`$()\n\\]", e):
|
|
1615
|
+
continue
|
|
1616
|
+
try:
|
|
1617
|
+
words = shlex.split(e)
|
|
1618
|
+
except ValueError:
|
|
1619
|
+
continue
|
|
1620
|
+
if len(words) >= 2:
|
|
1621
|
+
out.append(words)
|
|
1622
|
+
return out
|
|
1623
|
+
|
|
1624
|
+
|
|
1625
|
+
def classify_command(command: str, cwd: str | None = None, allow: list[str] | tuple[str, ...] = ()) -> list[Verdict]:
|
|
1626
|
+
"""Verdicts for every guarded CLI invocation in a shell command line (empty list: nothing to guard)."""
|
|
1627
|
+
w = Walker(cwd, parse_allow(allow))
|
|
1628
|
+
w.full_text = command
|
|
1629
|
+
w.command(command, 0)
|
|
1630
|
+
seen: set[Verdict] = set()
|
|
1631
|
+
out = []
|
|
1632
|
+
for v in w.out:
|
|
1633
|
+
if v not in seen:
|
|
1634
|
+
seen.add(v)
|
|
1635
|
+
out.append(v)
|
|
1636
|
+
return out
|
|
1637
|
+
|
|
1638
|
+
|
|
1639
|
+
def classify_argv(argv: list[str], allow: list[str] | tuple[str, ...] = ()) -> list[Verdict]:
|
|
1640
|
+
"""Verdicts for one argv (what a pack's `run:` list will execute). `shell=False` is assumed: no operators are interpreted."""
|
|
1641
|
+
w = Walker(None, parse_allow(allow))
|
|
1642
|
+
w.full_text = " ".join(argv)
|
|
1643
|
+
w.words(list(argv), 0)
|
|
1644
|
+
return w.out
|
|
1645
|
+
|
|
1646
|
+
|
|
1647
|
+
def first_denial(verdicts: list[Verdict]) -> Verdict | None:
|
|
1648
|
+
for v in verdicts:
|
|
1649
|
+
if v.denied:
|
|
1650
|
+
return v
|
|
1651
|
+
return None
|