muad-skill-http 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- muad_skill_http/__init__.py +47 -0
- muad_skill_http/_http.py +109 -0
- muad_skill_http/_normalize.py +49 -0
- muad_skill_http/_platform.py +404 -0
- muad_skill_http/_trace.py +216 -0
- muad_skill_http-0.1.0.dist-info/METADATA +88 -0
- muad_skill_http-0.1.0.dist-info/RECORD +9 -0
- muad_skill_http-0.1.0.dist-info/WHEEL +5 -0
- muad_skill_http-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""muad_skill_http: trace-aware requests helper and platform request client.
|
|
2
|
+
|
|
3
|
+
Usage in skills::
|
|
4
|
+
|
|
5
|
+
import muad_skill_http
|
|
6
|
+
|
|
7
|
+
response = muad_skill_http.get("http://service/api")
|
|
8
|
+
response = muad_skill_http.platform_request("mssw", "GET", "/api/v1/targets")
|
|
9
|
+
|
|
10
|
+
Without importing this module, existing skills keep their original behavior:
|
|
11
|
+
``requests`` is never patched and no trace spool is written.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from . import _platform, _trace
|
|
17
|
+
from ._http import delete, get, head, options, patch, post, put, request, session
|
|
18
|
+
from ._platform import PlatformClient, PlatformSessionError, platform, platform_request
|
|
19
|
+
from ._trace import EXIT_REQUEST_LIMIT
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"EXIT_REQUEST_LIMIT",
|
|
23
|
+
"PlatformClient",
|
|
24
|
+
"PlatformSessionError",
|
|
25
|
+
"delete",
|
|
26
|
+
"get",
|
|
27
|
+
"head",
|
|
28
|
+
"options",
|
|
29
|
+
"patch",
|
|
30
|
+
"platform",
|
|
31
|
+
"platform_request",
|
|
32
|
+
"post",
|
|
33
|
+
"put",
|
|
34
|
+
"request",
|
|
35
|
+
"session",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _reset_for_tests():
|
|
40
|
+
"""Drop the process-wide recorder and session cache (tests only)."""
|
|
41
|
+
_trace.reset()
|
|
42
|
+
_platform.reset_cache()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _flush_summary():
|
|
46
|
+
"""Write the trace summary early (tests and one-shot scripts)."""
|
|
47
|
+
_trace.flush_summary()
|
muad_skill_http/_http.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Thin requests pass-through that records every helper request."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from . import _trace
|
|
10
|
+
from ._normalize import split_target
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _duration_ms(started):
|
|
14
|
+
return int((time.monotonic() - started) * 1000)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _send(call, method, url, **kwargs):
|
|
18
|
+
method = str(method).upper()
|
|
19
|
+
extra = kwargs.pop("_trace_extra", None)
|
|
20
|
+
recorder = _trace.get_recorder()
|
|
21
|
+
host, path = split_target(url)
|
|
22
|
+
recorder.admit(host, method, path)
|
|
23
|
+
started = time.monotonic()
|
|
24
|
+
try:
|
|
25
|
+
response = call(method, url, **kwargs)
|
|
26
|
+
except requests.RequestException as exc:
|
|
27
|
+
recorder.complete(host, method, path, None, _duration_ms(started), type(exc).__name__, extra)
|
|
28
|
+
raise
|
|
29
|
+
status = response.status_code
|
|
30
|
+
error_class = "http_%d" % status if status >= 400 else ""
|
|
31
|
+
recorder.complete(host, method, path, status, _duration_ms(started), error_class, extra)
|
|
32
|
+
return response
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def request(method, url, **kwargs):
|
|
36
|
+
"""requests.request with trace + per-interface limit accounting."""
|
|
37
|
+
return _send(requests.request, method, url, **kwargs)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get(url, **kwargs):
|
|
41
|
+
return request("GET", url, **kwargs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def head(url, **kwargs):
|
|
45
|
+
return request("HEAD", url, **kwargs)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def options(url, **kwargs):
|
|
49
|
+
return request("OPTIONS", url, **kwargs)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def post(url, **kwargs):
|
|
53
|
+
return request("POST", url, **kwargs)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def put(url, **kwargs):
|
|
57
|
+
return request("PUT", url, **kwargs)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def patch(url, **kwargs):
|
|
61
|
+
return request("PATCH", url, **kwargs)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def delete(url, **kwargs):
|
|
65
|
+
return request("DELETE", url, **kwargs)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _TrackedSession:
|
|
69
|
+
def __init__(self):
|
|
70
|
+
self._session = requests.Session()
|
|
71
|
+
|
|
72
|
+
def request(self, method, url, **kwargs):
|
|
73
|
+
return _send(self._session.request, method, url, **kwargs)
|
|
74
|
+
|
|
75
|
+
def get(self, url, **kwargs):
|
|
76
|
+
return self.request("GET", url, **kwargs)
|
|
77
|
+
|
|
78
|
+
def head(self, url, **kwargs):
|
|
79
|
+
return self.request("HEAD", url, **kwargs)
|
|
80
|
+
|
|
81
|
+
def options(self, url, **kwargs):
|
|
82
|
+
return self.request("OPTIONS", url, **kwargs)
|
|
83
|
+
|
|
84
|
+
def post(self, url, **kwargs):
|
|
85
|
+
return self.request("POST", url, **kwargs)
|
|
86
|
+
|
|
87
|
+
def put(self, url, **kwargs):
|
|
88
|
+
return self.request("PUT", url, **kwargs)
|
|
89
|
+
|
|
90
|
+
def patch(self, url, **kwargs):
|
|
91
|
+
return self.request("PATCH", url, **kwargs)
|
|
92
|
+
|
|
93
|
+
def delete(self, url, **kwargs):
|
|
94
|
+
return self.request("DELETE", url, **kwargs)
|
|
95
|
+
|
|
96
|
+
def close(self):
|
|
97
|
+
self._session.close()
|
|
98
|
+
|
|
99
|
+
def __enter__(self):
|
|
100
|
+
return self
|
|
101
|
+
|
|
102
|
+
def __exit__(self, exc_type, exc, tb):
|
|
103
|
+
self.close()
|
|
104
|
+
return False
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def session():
|
|
108
|
+
"""A requests.Session wrapper (supports ``with``) that traces each call."""
|
|
109
|
+
return _TrackedSession()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Path and target normalization for interface counting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from urllib.parse import urlsplit
|
|
7
|
+
|
|
8
|
+
_UUID_RE = re.compile(
|
|
9
|
+
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
|
|
10
|
+
)
|
|
11
|
+
_HEX_RE = re.compile(r"^[0-9a-fA-F]{8,}$")
|
|
12
|
+
_DIGITS_RE = re.compile(r"^\d+$")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def normalize_path(path):
|
|
16
|
+
"""Collapse volatile path segments and drop query/fragment.
|
|
17
|
+
|
|
18
|
+
Numeric, UUID and long hex segments become ``{id}`` so one business
|
|
19
|
+
interface is counted once regardless of object ids.
|
|
20
|
+
"""
|
|
21
|
+
if not isinstance(path, str):
|
|
22
|
+
path = str(path or "")
|
|
23
|
+
if path.startswith(("http://", "https://")):
|
|
24
|
+
raw = urlsplit(path).path or "/"
|
|
25
|
+
else:
|
|
26
|
+
raw = path.split("?", 1)[0].split("#", 1)[0]
|
|
27
|
+
if not raw:
|
|
28
|
+
return "/"
|
|
29
|
+
segments = ["{id}" if _is_volatile(segment) else segment for segment in raw.split("/")]
|
|
30
|
+
return "/".join(segments) or "/"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def split_target(url):
|
|
34
|
+
"""Return (host:port, normalized path) for a request URL."""
|
|
35
|
+
parts = urlsplit(str(url))
|
|
36
|
+
host = (parts.hostname or "").lower()
|
|
37
|
+
try:
|
|
38
|
+
port = parts.port
|
|
39
|
+
except ValueError:
|
|
40
|
+
port = None
|
|
41
|
+
if port:
|
|
42
|
+
host = "%s:%d" % (host, port)
|
|
43
|
+
return host, normalize_path(parts.path or "/")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _is_volatile(segment):
|
|
47
|
+
if not segment:
|
|
48
|
+
return False
|
|
49
|
+
return bool(_DIGITS_RE.match(segment) or _UUID_RE.match(segment) or _HEX_RE.match(segment))
|
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
"""Platform request client: session state, headers, refresh and retry.
|
|
2
|
+
|
|
3
|
+
Header order follows the backend design:
|
|
4
|
+
browser defaults -> hostHeader -> staticHeaders -> caller/extra headers ->
|
|
5
|
+
Cookie -> CSRF -> Traceid -> re-assert mandatory headers. Header values never
|
|
6
|
+
reach the trace spool or diagnostics.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import uuid
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
|
|
19
|
+
import requests
|
|
20
|
+
|
|
21
|
+
from . import _http
|
|
22
|
+
from ._normalize import normalize_path
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"PlatformSessionError",
|
|
26
|
+
"normalize_path",
|
|
27
|
+
"build_platform_headers",
|
|
28
|
+
"platform_request",
|
|
29
|
+
"platform",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
SESSION_MANAGER_BIN = "session-manager"
|
|
33
|
+
GET_STATE_TIMEOUT_SECONDS = 30
|
|
34
|
+
DEFAULT_TRACE_HEADER = "Traceid"
|
|
35
|
+
IDEMPOTENT_METHODS = frozenset({"GET", "HEAD"})
|
|
36
|
+
USER_AGENT = (
|
|
37
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
38
|
+
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
_platform_session = requests.Session()
|
|
42
|
+
_cache = {}
|
|
43
|
+
_cache_lock = threading.Lock()
|
|
44
|
+
_load_lock = threading.Lock()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class PlatformSessionError(RuntimeError):
|
|
48
|
+
"""Platform credentials/state unavailable; skills should fail loud."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# --------------------------------------------------------------------------- #
|
|
52
|
+
# public API
|
|
53
|
+
# --------------------------------------------------------------------------- #
|
|
54
|
+
|
|
55
|
+
def platform_request(platform, method, path, **kwargs):
|
|
56
|
+
"""Send a business request with platform headers and trace accounting."""
|
|
57
|
+
method = str(method).upper()
|
|
58
|
+
for attempt in (0, 1):
|
|
59
|
+
call_kwargs = dict(kwargs)
|
|
60
|
+
extra_headers = call_kwargs.pop("extra_headers", None)
|
|
61
|
+
caller_headers = call_kwargs.pop("headers", None)
|
|
62
|
+
context = _get_context(platform)
|
|
63
|
+
url = _resolve_url(platform, context["profile"], path)
|
|
64
|
+
headers, trace_id = build_platform_headers(
|
|
65
|
+
context["profile"], context["cookies"], caller_headers, extra_headers, call_kwargs
|
|
66
|
+
)
|
|
67
|
+
call_kwargs["headers"] = headers
|
|
68
|
+
extra = {"platform": platform}
|
|
69
|
+
if trace_id:
|
|
70
|
+
extra["traceId"] = trace_id
|
|
71
|
+
response = _http._send(
|
|
72
|
+
_platform_session.request, method, url, _trace_extra=extra, **call_kwargs
|
|
73
|
+
)
|
|
74
|
+
if attempt == 0 and method in IDEMPOTENT_METHODS and response.status_code in (401, 403):
|
|
75
|
+
_invalidate(platform)
|
|
76
|
+
continue
|
|
77
|
+
return response
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def platform(name):
|
|
81
|
+
"""Return a reusable client bound to one platform profile."""
|
|
82
|
+
return PlatformClient(name)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class PlatformClient:
|
|
86
|
+
def __init__(self, name):
|
|
87
|
+
self.platform = name
|
|
88
|
+
|
|
89
|
+
def request(self, method, path, **kwargs):
|
|
90
|
+
return platform_request(self.platform, method, path, **kwargs)
|
|
91
|
+
|
|
92
|
+
def get(self, path, **kwargs):
|
|
93
|
+
return self.request("GET", path, **kwargs)
|
|
94
|
+
|
|
95
|
+
def head(self, path, **kwargs):
|
|
96
|
+
return self.request("HEAD", path, **kwargs)
|
|
97
|
+
|
|
98
|
+
def post(self, path, **kwargs):
|
|
99
|
+
return self.request("POST", path, **kwargs)
|
|
100
|
+
|
|
101
|
+
def put(self, path, **kwargs):
|
|
102
|
+
return self.request("PUT", path, **kwargs)
|
|
103
|
+
|
|
104
|
+
def patch(self, path, **kwargs):
|
|
105
|
+
return self.request("PATCH", path, **kwargs)
|
|
106
|
+
|
|
107
|
+
def delete(self, path, **kwargs):
|
|
108
|
+
return self.request("DELETE", path, **kwargs)
|
|
109
|
+
|
|
110
|
+
def __enter__(self):
|
|
111
|
+
return self
|
|
112
|
+
|
|
113
|
+
def __exit__(self, exc_type, exc, tb):
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# --------------------------------------------------------------------------- #
|
|
118
|
+
# headers
|
|
119
|
+
# --------------------------------------------------------------------------- #
|
|
120
|
+
|
|
121
|
+
def build_platform_headers(profile, cookies, caller_headers=None, extra_headers=None, request_kwargs=None):
|
|
122
|
+
"""Build the header dict in the design's mandated order.
|
|
123
|
+
|
|
124
|
+
Returns ``(headers, trace_id)``; ``trace_id`` is ``None`` when the profile
|
|
125
|
+
disables the trace header.
|
|
126
|
+
"""
|
|
127
|
+
headers = {}
|
|
128
|
+
_set_header(headers, "Accept", "application/json, text/plain, */*")
|
|
129
|
+
_set_header(headers, "Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
|
130
|
+
content_type = _content_type(request_kwargs or {})
|
|
131
|
+
if content_type:
|
|
132
|
+
_set_header(headers, "Content-Type", content_type)
|
|
133
|
+
_set_header(headers, "User-Agent", USER_AGENT)
|
|
134
|
+
_set_header(headers, "X-Requested-With", "XMLHttpRequest")
|
|
135
|
+
if profile.get("referer"):
|
|
136
|
+
_set_header(headers, "Referer", profile["referer"])
|
|
137
|
+
|
|
138
|
+
for name, value in profile.get("hostHeader", {}).items():
|
|
139
|
+
_set_header(headers, name, value)
|
|
140
|
+
for name, value in profile.get("staticHeaders", {}).items():
|
|
141
|
+
_set_header(headers, name, value)
|
|
142
|
+
|
|
143
|
+
if profile.get("extraAllowed", True):
|
|
144
|
+
for name, value in _header_items(caller_headers):
|
|
145
|
+
_set_header(headers, name, value)
|
|
146
|
+
for name, value in _header_items(extra_headers):
|
|
147
|
+
_set_header(headers, name, value)
|
|
148
|
+
|
|
149
|
+
cookie_values = _cookie_values(cookies)
|
|
150
|
+
cookie_header = "; ".join("%s=%s" % (name, value) for name, value in cookie_values.items())
|
|
151
|
+
if cookie_header:
|
|
152
|
+
_set_header(headers, "Cookie", cookie_header)
|
|
153
|
+
|
|
154
|
+
csrf_headers = []
|
|
155
|
+
for mapping in profile.get("csrf", []):
|
|
156
|
+
value = cookie_values.get(mapping["cookie"])
|
|
157
|
+
if value:
|
|
158
|
+
_set_header(headers, mapping["header"], value)
|
|
159
|
+
csrf_headers.append((mapping["header"], value))
|
|
160
|
+
|
|
161
|
+
trace_header = profile.get("traceHeader")
|
|
162
|
+
trace_id = None
|
|
163
|
+
if trace_header:
|
|
164
|
+
trace_id = str(uuid.uuid4())
|
|
165
|
+
_set_header(headers, trace_header, trace_id)
|
|
166
|
+
|
|
167
|
+
# Mandatory headers are written last so caller/extra headers cannot win.
|
|
168
|
+
if cookie_header:
|
|
169
|
+
_set_header(headers, "Cookie", cookie_header)
|
|
170
|
+
for name, value in csrf_headers:
|
|
171
|
+
_set_header(headers, name, value)
|
|
172
|
+
if trace_header and trace_id:
|
|
173
|
+
_set_header(headers, trace_header, trace_id)
|
|
174
|
+
return headers, trace_id
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _set_header(headers, name, value):
|
|
178
|
+
for existing in [key for key in headers if key.lower() == name.lower()]:
|
|
179
|
+
del headers[existing]
|
|
180
|
+
headers[name] = str(value)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _header_items(raw):
|
|
184
|
+
if not isinstance(raw, dict):
|
|
185
|
+
return []
|
|
186
|
+
items = []
|
|
187
|
+
for name, value in raw.items():
|
|
188
|
+
if isinstance(name, str) and name.strip() and value is not None:
|
|
189
|
+
items.append((name.strip(), value))
|
|
190
|
+
return items
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _content_type(kwargs):
|
|
194
|
+
if kwargs.get("json") is not None:
|
|
195
|
+
return "application/json;charset=UTF-8"
|
|
196
|
+
if kwargs.get("data") is not None:
|
|
197
|
+
return "application/x-www-form-urlencoded;charset=UTF-8"
|
|
198
|
+
return ""
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _cookie_values(cookies):
|
|
202
|
+
result = {}
|
|
203
|
+
if not isinstance(cookies, list):
|
|
204
|
+
return result
|
|
205
|
+
for item in cookies:
|
|
206
|
+
if not isinstance(item, dict):
|
|
207
|
+
continue
|
|
208
|
+
name = item.get("name")
|
|
209
|
+
value = item.get("value")
|
|
210
|
+
if isinstance(name, str) and name and isinstance(value, str):
|
|
211
|
+
result[name] = value
|
|
212
|
+
return result
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# --------------------------------------------------------------------------- #
|
|
216
|
+
# session state
|
|
217
|
+
# --------------------------------------------------------------------------- #
|
|
218
|
+
|
|
219
|
+
def _get_context(name):
|
|
220
|
+
with _cache_lock:
|
|
221
|
+
context = _cache.get(name)
|
|
222
|
+
if context is not None and not _expired(context["expires_at"]):
|
|
223
|
+
return context
|
|
224
|
+
with _load_lock:
|
|
225
|
+
with _cache_lock:
|
|
226
|
+
context = _cache.get(name)
|
|
227
|
+
if context is not None and not _expired(context["expires_at"]):
|
|
228
|
+
return context
|
|
229
|
+
context = _load_state(name)
|
|
230
|
+
with _cache_lock:
|
|
231
|
+
_cache[name] = context
|
|
232
|
+
return context
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _invalidate(name):
|
|
236
|
+
with _cache_lock:
|
|
237
|
+
_cache.pop(name, None)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def reset_cache():
|
|
241
|
+
with _cache_lock:
|
|
242
|
+
_cache.clear()
|
|
243
|
+
with _diagnosed_lock:
|
|
244
|
+
_diagnosed.clear()
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _load_state(name):
|
|
248
|
+
skill_name = (os.environ.get("MUAD_SKILL_NAME") or "").strip()
|
|
249
|
+
if not skill_name:
|
|
250
|
+
raise PlatformSessionError(
|
|
251
|
+
"platform session unavailable for %s: MUAD_SKILL_NAME is not set" % name
|
|
252
|
+
)
|
|
253
|
+
try:
|
|
254
|
+
completed = subprocess.run(
|
|
255
|
+
[SESSION_MANAGER_BIN, "get-state", "--skill-name", skill_name],
|
|
256
|
+
capture_output=True,
|
|
257
|
+
text=True,
|
|
258
|
+
timeout=GET_STATE_TIMEOUT_SECONDS,
|
|
259
|
+
)
|
|
260
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
261
|
+
raise PlatformSessionError(
|
|
262
|
+
"platform session unavailable for %s: session-manager could not run" % name
|
|
263
|
+
) from exc
|
|
264
|
+
if completed.returncode != 0:
|
|
265
|
+
detail = _first_line(completed.stderr)
|
|
266
|
+
message = "platform session unavailable for %s: session-manager get-state exited %d" % (
|
|
267
|
+
name,
|
|
268
|
+
completed.returncode,
|
|
269
|
+
)
|
|
270
|
+
if detail:
|
|
271
|
+
message += " (%s)" % detail
|
|
272
|
+
raise PlatformSessionError(message)
|
|
273
|
+
|
|
274
|
+
try:
|
|
275
|
+
payload = json.loads(completed.stdout)
|
|
276
|
+
state_file = payload["sessionStateFile"]
|
|
277
|
+
with open(state_file, "r", encoding="utf-8") as handle:
|
|
278
|
+
state = json.load(handle)
|
|
279
|
+
except (ValueError, KeyError, TypeError, OSError) as exc:
|
|
280
|
+
raise PlatformSessionError(
|
|
281
|
+
"platform session unavailable for %s: session state file is unreadable" % name
|
|
282
|
+
) from exc
|
|
283
|
+
|
|
284
|
+
section = (state.get("platforms") or {}).get(name)
|
|
285
|
+
if not isinstance(section, dict):
|
|
286
|
+
raise PlatformSessionError(
|
|
287
|
+
"platform session unavailable for %s: no credentials in session state" % name
|
|
288
|
+
)
|
|
289
|
+
expires_at = _parse_expiry(section.get("expiresAt"))
|
|
290
|
+
if _expired(expires_at):
|
|
291
|
+
raise PlatformSessionError(
|
|
292
|
+
"platform session unavailable for %s: session state expired after refresh" % name
|
|
293
|
+
)
|
|
294
|
+
return {
|
|
295
|
+
"platform": name,
|
|
296
|
+
"profile": _profile(section, name),
|
|
297
|
+
"cookies": section.get("cookies") or [],
|
|
298
|
+
"expires_at": expires_at,
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _profile(section, name):
|
|
303
|
+
raw = section.get("requestProfile")
|
|
304
|
+
if not isinstance(raw, dict):
|
|
305
|
+
_diagnostic(name, "request profile fallback missing=requestProfile")
|
|
306
|
+
raw = {}
|
|
307
|
+
return {
|
|
308
|
+
"baseUrl": _string(raw.get("baseUrl")),
|
|
309
|
+
"hostHeader": _string_map(raw.get("hostHeader")),
|
|
310
|
+
"referer": _string(raw.get("referer")),
|
|
311
|
+
"csrf": _csrf_mappings(raw.get("csrf")),
|
|
312
|
+
"staticHeaders": _string_map(raw.get("staticHeaders")),
|
|
313
|
+
"traceHeader": _trace_header(raw),
|
|
314
|
+
"extraAllowed": raw.get("extraAllowed") is not False,
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _trace_header(raw):
|
|
319
|
+
if "traceHeader" not in raw:
|
|
320
|
+
return DEFAULT_TRACE_HEADER
|
|
321
|
+
value = raw.get("traceHeader")
|
|
322
|
+
return value.strip() if isinstance(value, str) and value.strip() else None
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _csrf_mappings(raw):
|
|
326
|
+
mappings = []
|
|
327
|
+
if not isinstance(raw, list):
|
|
328
|
+
return mappings
|
|
329
|
+
for item in raw:
|
|
330
|
+
if not isinstance(item, dict):
|
|
331
|
+
continue
|
|
332
|
+
cookie_name = _string(item.get("cookie"))
|
|
333
|
+
header_name = _string(item.get("header"))
|
|
334
|
+
if cookie_name and header_name:
|
|
335
|
+
mappings.append({"cookie": cookie_name, "header": header_name})
|
|
336
|
+
return mappings
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _string(value):
|
|
340
|
+
return value.strip() if isinstance(value, str) else ""
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _string_map(raw):
|
|
344
|
+
result = {}
|
|
345
|
+
if not isinstance(raw, dict):
|
|
346
|
+
return result
|
|
347
|
+
for name, value in raw.items():
|
|
348
|
+
if isinstance(name, str) and name.strip() and isinstance(value, str) and value.strip():
|
|
349
|
+
result[name.strip()] = value.strip()
|
|
350
|
+
return result
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _resolve_url(name, profile, path):
|
|
354
|
+
path = str(path)
|
|
355
|
+
if path.startswith(("http://", "https://")):
|
|
356
|
+
return path
|
|
357
|
+
base_url = profile.get("baseUrl") or ""
|
|
358
|
+
if not base_url:
|
|
359
|
+
raise PlatformSessionError(
|
|
360
|
+
"platform request unavailable for %s: request profile has no baseUrl "
|
|
361
|
+
"and path is not absolute" % name
|
|
362
|
+
)
|
|
363
|
+
return base_url.rstrip("/") + "/" + path.lstrip("/")
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _parse_expiry(value):
|
|
367
|
+
if not isinstance(value, str) or not value.strip():
|
|
368
|
+
return None
|
|
369
|
+
try:
|
|
370
|
+
return datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
371
|
+
except ValueError:
|
|
372
|
+
return None
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _expired(expires_at):
|
|
376
|
+
if expires_at is None:
|
|
377
|
+
return False
|
|
378
|
+
return expires_at <= datetime.now(timezone.utc)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _first_line(text):
|
|
382
|
+
if not isinstance(text, str):
|
|
383
|
+
return ""
|
|
384
|
+
for line in text.splitlines():
|
|
385
|
+
line = line.strip()
|
|
386
|
+
if line:
|
|
387
|
+
return line[:200]
|
|
388
|
+
return ""
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
_diagnosed = set()
|
|
392
|
+
_diagnosed_lock = threading.Lock()
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _diagnostic(name, message):
|
|
396
|
+
key = (name, message)
|
|
397
|
+
with _diagnosed_lock:
|
|
398
|
+
if key in _diagnosed:
|
|
399
|
+
return
|
|
400
|
+
_diagnosed.add(key)
|
|
401
|
+
try:
|
|
402
|
+
sys.stderr.write("[muad-skill-sdk][platform] platform=%s %s\n" % (name, message))
|
|
403
|
+
except Exception:
|
|
404
|
+
return
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Per-execution trace spool and interface request limit.
|
|
2
|
+
|
|
3
|
+
The recorder is process-wide. Counting and limit enforcement stay in memory
|
|
4
|
+
under a lock; the spool file is the only observable side effect and every
|
|
5
|
+
I/O failure is swallowed so auditing never breaks a skill request.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import atexit
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
from collections import deque
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
|
|
18
|
+
EXIT_REQUEST_LIMIT = 86
|
|
19
|
+
DETAIL_LIMIT = 500
|
|
20
|
+
BURST_WINDOW_SECONDS = 10.0
|
|
21
|
+
TOP_INTERFACE_LIMIT = 10
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _terminate(code):
|
|
25
|
+
os._exit(code)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _now_iso():
|
|
29
|
+
return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _parse_limit():
|
|
33
|
+
raw = os.environ.get("MUAD_SKILL_REQUEST_LIMIT", "")
|
|
34
|
+
try:
|
|
35
|
+
value = int(str(raw).strip())
|
|
36
|
+
except (TypeError, ValueError):
|
|
37
|
+
return 0
|
|
38
|
+
return value if value > 0 else 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class TraceRecorder:
|
|
42
|
+
def __init__(self, spool_path, execution_id, limit):
|
|
43
|
+
self._spool_path = spool_path
|
|
44
|
+
self._enabled = bool(spool_path) and bool(execution_id)
|
|
45
|
+
self._limit = limit
|
|
46
|
+
self._lock = threading.Lock()
|
|
47
|
+
self._file = None
|
|
48
|
+
self._file_failed = False
|
|
49
|
+
self._counts = {}
|
|
50
|
+
self._hosts = {}
|
|
51
|
+
self._burst = deque()
|
|
52
|
+
self._max_burst = 0
|
|
53
|
+
self._total = 0
|
|
54
|
+
self._errors = 0
|
|
55
|
+
self._details = 0
|
|
56
|
+
self._truncated = False
|
|
57
|
+
self._aborted = False
|
|
58
|
+
self._summary_written = False
|
|
59
|
+
|
|
60
|
+
# -- request lifecycle ------------------------------------------------- #
|
|
61
|
+
|
|
62
|
+
def admit(self, host, method, path):
|
|
63
|
+
"""Count one attempt and terminate before sending the N+1 request."""
|
|
64
|
+
terminate_now = False
|
|
65
|
+
already_aborted = False
|
|
66
|
+
with self._lock:
|
|
67
|
+
if self._aborted:
|
|
68
|
+
already_aborted = True
|
|
69
|
+
else:
|
|
70
|
+
self._total += 1
|
|
71
|
+
key = (host, method, path)
|
|
72
|
+
count = self._counts.get(key, 0) + 1
|
|
73
|
+
self._counts[key] = count
|
|
74
|
+
self._hosts[host] = self._hosts.get(host, 0) + 1
|
|
75
|
+
self._track_burst_locked()
|
|
76
|
+
if self._limit and count > self._limit:
|
|
77
|
+
self._aborted = True
|
|
78
|
+
self._write_abort_locked(host, method, path, count)
|
|
79
|
+
self._write_summary_locked()
|
|
80
|
+
terminate_now = True
|
|
81
|
+
if terminate_now or already_aborted:
|
|
82
|
+
_terminate(EXIT_REQUEST_LIMIT)
|
|
83
|
+
|
|
84
|
+
def complete(self, host, method, path, status, duration_ms, error_class, extra=None):
|
|
85
|
+
"""Record the response (or transport failure) if the detail cap allows."""
|
|
86
|
+
event = {
|
|
87
|
+
"v": 1,
|
|
88
|
+
"ts": _now_iso(),
|
|
89
|
+
"kind": "http",
|
|
90
|
+
"host": host,
|
|
91
|
+
"method": method,
|
|
92
|
+
"path": path,
|
|
93
|
+
"status": status,
|
|
94
|
+
"durationMs": duration_ms,
|
|
95
|
+
"errorClass": error_class or "",
|
|
96
|
+
}
|
|
97
|
+
if extra:
|
|
98
|
+
event.update(extra)
|
|
99
|
+
with self._lock:
|
|
100
|
+
if status is None or status >= 400:
|
|
101
|
+
self._errors += 1
|
|
102
|
+
if self._details < DETAIL_LIMIT:
|
|
103
|
+
if self._write_locked(event):
|
|
104
|
+
self._details += 1
|
|
105
|
+
else:
|
|
106
|
+
self._truncated = True
|
|
107
|
+
|
|
108
|
+
# -- spool ------------------------------------------------------------- #
|
|
109
|
+
|
|
110
|
+
def flush_summary(self):
|
|
111
|
+
with self._lock:
|
|
112
|
+
self._write_summary_locked()
|
|
113
|
+
|
|
114
|
+
def _track_burst_locked(self):
|
|
115
|
+
now = time.monotonic()
|
|
116
|
+
self._burst.append(now)
|
|
117
|
+
while self._burst and now - self._burst[0] > BURST_WINDOW_SECONDS:
|
|
118
|
+
self._burst.popleft()
|
|
119
|
+
if len(self._burst) > self._max_burst:
|
|
120
|
+
self._max_burst = len(self._burst)
|
|
121
|
+
|
|
122
|
+
def _write_abort_locked(self, host, method, path, count):
|
|
123
|
+
self._write_locked(
|
|
124
|
+
{
|
|
125
|
+
"v": 1,
|
|
126
|
+
"ts": _now_iso(),
|
|
127
|
+
"kind": "abort",
|
|
128
|
+
"reason": "request_limit",
|
|
129
|
+
"interface": "%s %s %s" % (method, host, path),
|
|
130
|
+
"count": count,
|
|
131
|
+
"limit": self._limit,
|
|
132
|
+
}
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _write_summary_locked(self):
|
|
136
|
+
if self._summary_written:
|
|
137
|
+
return
|
|
138
|
+
self._summary_written = True
|
|
139
|
+
top = sorted(
|
|
140
|
+
(
|
|
141
|
+
{"host": host, "method": method, "path": path, "count": count}
|
|
142
|
+
for (host, method, path), count in self._counts.items()
|
|
143
|
+
),
|
|
144
|
+
key=lambda item: (-item["count"], item["host"], item["method"], item["path"]),
|
|
145
|
+
)[:TOP_INTERFACE_LIMIT]
|
|
146
|
+
self._write_locked(
|
|
147
|
+
{
|
|
148
|
+
"v": 1,
|
|
149
|
+
"ts": _now_iso(),
|
|
150
|
+
"kind": "summary",
|
|
151
|
+
"totalRequests": self._total,
|
|
152
|
+
"errorCount": self._errors,
|
|
153
|
+
"hosts": dict(self._hosts),
|
|
154
|
+
"topInterfaces": top,
|
|
155
|
+
"maxBurst10s": self._max_burst,
|
|
156
|
+
"truncated": self._truncated,
|
|
157
|
+
"aborted": self._aborted,
|
|
158
|
+
}
|
|
159
|
+
)
|
|
160
|
+
self._flush_locked()
|
|
161
|
+
|
|
162
|
+
def _write_locked(self, event):
|
|
163
|
+
if not self._enabled or self._file_failed:
|
|
164
|
+
return False
|
|
165
|
+
try:
|
|
166
|
+
if self._file is None:
|
|
167
|
+
fd = os.open(
|
|
168
|
+
self._spool_path,
|
|
169
|
+
os.O_WRONLY | os.O_CREAT | os.O_APPEND,
|
|
170
|
+
0o600,
|
|
171
|
+
)
|
|
172
|
+
self._file = os.fdopen(fd, "a", encoding="utf-8")
|
|
173
|
+
self._file.write(json.dumps(event, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
174
|
+
return True
|
|
175
|
+
except (OSError, ValueError):
|
|
176
|
+
self._disable_locked()
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
def _flush_locked(self):
|
|
180
|
+
try:
|
|
181
|
+
if self._file is not None:
|
|
182
|
+
self._file.flush()
|
|
183
|
+
os.fsync(self._file.fileno())
|
|
184
|
+
except (OSError, ValueError):
|
|
185
|
+
self._disable_locked()
|
|
186
|
+
|
|
187
|
+
def _disable_locked(self):
|
|
188
|
+
self._file_failed = True
|
|
189
|
+
self._file = None
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
_recorder = None
|
|
193
|
+
_recorder_lock = threading.Lock()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def get_recorder():
|
|
197
|
+
global _recorder
|
|
198
|
+
with _recorder_lock:
|
|
199
|
+
if _recorder is None:
|
|
200
|
+
_recorder = TraceRecorder(
|
|
201
|
+
os.environ.get("MUAD_TRACE_SPOOL", ""),
|
|
202
|
+
os.environ.get("MUAD_EXECUTION_ID", ""),
|
|
203
|
+
_parse_limit(),
|
|
204
|
+
)
|
|
205
|
+
atexit.register(_recorder.flush_summary)
|
|
206
|
+
return _recorder
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def flush_summary():
|
|
210
|
+
get_recorder().flush_summary()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def reset():
|
|
214
|
+
global _recorder
|
|
215
|
+
with _recorder_lock:
|
|
216
|
+
_recorder = None
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: muad-skill-http
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Trace-aware HTTP helpers and platform request client for muad skills
|
|
5
|
+
Author: muad-openclaw maintainers
|
|
6
|
+
Project-URL: Homepage, https://github.com/Michaelxwb/muad-openclaw
|
|
7
|
+
Project-URL: Source, https://github.com/Michaelxwb/muad-openclaw/tree/main/tools/muad-skill-sdk
|
|
8
|
+
Keywords: http,requests,tracing,skills
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: requests<3,>=2.31
|
|
22
|
+
|
|
23
|
+
# muad-skill-http
|
|
24
|
+
|
|
25
|
+
Trace-aware HTTP helpers and a platform request client for muad skills.
|
|
26
|
+
|
|
27
|
+
`platform_request()` sends business-platform requests through a platform session
|
|
28
|
+
managed by `session-manager`. It assembles the browser defaults, platform host
|
|
29
|
+
header, platform static headers, cookies, CSRF header and a fresh `Traceid`
|
|
30
|
+
UUID v4 for every request, and re-asserts the mandatory headers last so caller
|
|
31
|
+
headers cannot override them. Cookie and CSRF values never reach the trace
|
|
32
|
+
spool.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install muad-skill-http
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from muad_skill_http import get, platform_request
|
|
44
|
+
|
|
45
|
+
# Plain HTTP with trace accounting.
|
|
46
|
+
response = get("https://example.com/api/v1/items")
|
|
47
|
+
|
|
48
|
+
# Business-platform request: headers and Traceid are built by the SDK.
|
|
49
|
+
response = platform_request("demo", "GET", "/api/v1/targets")
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`get` / `post` / `put` / `patch` / `delete` / `head` / `options` / `request`
|
|
53
|
+
wrap `requests` and record request counts for the current skill execution.
|
|
54
|
+
|
|
55
|
+
### Runtime requirements for `platform_request`
|
|
56
|
+
|
|
57
|
+
`platform_request` resolves the platform session from the skill runtime, not
|
|
58
|
+
from the local machine. It needs:
|
|
59
|
+
|
|
60
|
+
- `MUAD_SKILL_NAME` set to the running skill name;
|
|
61
|
+
- a `session-manager` executable on `PATH` that answers
|
|
62
|
+
`session-manager get-state --skill-name <name>` with
|
|
63
|
+
`{"sessionStateFile": "..."}`;
|
|
64
|
+
- a session state file containing the requested platform's credentials.
|
|
65
|
+
|
|
66
|
+
Outside that environment it raises `muad_skill_http.PlatformSessionError`.
|
|
67
|
+
Plain `get` / `post` calls work anywhere.
|
|
68
|
+
|
|
69
|
+
## Environment variables
|
|
70
|
+
|
|
71
|
+
| Variable | Purpose |
|
|
72
|
+
| --- | --- |
|
|
73
|
+
| `MUAD_SKILL_NAME` | Skill name used to resolve the platform session. |
|
|
74
|
+
| `MUAD_TRACE_SPOOL` | Optional JSONL trace spool path; paired with `MUAD_EXECUTION_ID`. |
|
|
75
|
+
| `MUAD_EXECUTION_ID` | Execution id written to the trace spool. |
|
|
76
|
+
| `MUAD_SKILL_REQUEST_LIMIT` | Per-execution interface request limit; unset or `0` means unlimited. Exceeding the limit exits with code `86`. |
|
|
77
|
+
|
|
78
|
+
I/O failures while writing the trace spool are swallowed: auditing never breaks
|
|
79
|
+
a skill request.
|
|
80
|
+
|
|
81
|
+
## Development
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
python -m pytest test
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Releases are published by pushing an `sdk-v*` tag; the tag must match
|
|
88
|
+
`project.version` in `pyproject.toml`.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
muad_skill_http/__init__.py,sha256=waAP4oLLoetZdR8h8HoMwijeMG2bvWk0aeIS6MYdFek,1201
|
|
2
|
+
muad_skill_http/_http.py,sha256=qvWXMql8ikjQAhSJMIe3LYG5CcRP3cupvkKIATSnvmE,2754
|
|
3
|
+
muad_skill_http/_normalize.py,sha256=zXDahA3n-_e-Pv0C5HePupXSRB-akBSJ0QQNyhzxecI,1449
|
|
4
|
+
muad_skill_http/_platform.py,sha256=tkSTzeyeOnzXRmorq1-0pmGE35g1nhpSJCmQNO-o1d4,12745
|
|
5
|
+
muad_skill_http/_trace.py,sha256=YsnqDjkhz74ILpgHVrPrZ2cxe-XdwS3o2wx-9pcT7WQ,6676
|
|
6
|
+
muad_skill_http-0.1.0.dist-info/METADATA,sha256=lhAJw9oCP8Wes19qexTDenX7J_GNY0a5x0rQ73JHiQU,3222
|
|
7
|
+
muad_skill_http-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
muad_skill_http-0.1.0.dist-info/top_level.txt,sha256=_qe_kcb4GVh2LH_Us3dCDITuwB7uk7kyqi7ZRakmXvQ,16
|
|
9
|
+
muad_skill_http-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
muad_skill_http
|