orbit-sdk 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- orbit/__init__.py +259 -0
- orbit/_autoimport.py +121 -0
- orbit/_bootstrap/sitecustomize.py +18 -0
- orbit/_classify.py +281 -0
- orbit/_core.py +1014 -0
- orbit/_db.py +216 -0
- orbit/_inbound.py +299 -0
- orbit/_logs.py +66 -0
- orbit/_outbound.py +206 -0
- orbit/_pii.py +134 -0
- orbit/_trackers.py +103 -0
- orbit/auto.py +10 -0
- orbit/py.typed +0 -0
- orbit/run.py +28 -0
- orbit_sdk-1.0.0.dist-info/METADATA +253 -0
- orbit_sdk-1.0.0.dist-info/RECORD +17 -0
- orbit_sdk-1.0.0.dist-info/WHEEL +4 -0
orbit/__init__.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Orbit SDK by JuriCloud: runtime DPDP Act compliance tracking for Python back ends.
|
|
2
|
+
|
|
3
|
+
`import orbit.auto` (or `orbit.init(key=..., service=...)`) and the SDK records, value-free, where personal data is
|
|
4
|
+
collected, stored, sent and logged; consent checks; erasure/export/verify tasks; outbound policies. Protocol:
|
|
5
|
+
docs/SDK.md.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import atexit
|
|
12
|
+
import contextlib
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from collections.abc import Iterable, Iterator, Mapping
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Callable, Optional, TypeVar, overload
|
|
18
|
+
|
|
19
|
+
from . import _autoimport, _core
|
|
20
|
+
from ._core import (
|
|
21
|
+
VERSION,
|
|
22
|
+
OrbitError,
|
|
23
|
+
OrbitPolicyError,
|
|
24
|
+
PolicyEvent,
|
|
25
|
+
PolicyListener,
|
|
26
|
+
RightsRequest,
|
|
27
|
+
Withdrawal,
|
|
28
|
+
)
|
|
29
|
+
from ._inbound import ASGIMiddleware, DjangoMiddleware, instrument_flask
|
|
30
|
+
from ._logs import patch_logging
|
|
31
|
+
|
|
32
|
+
__version__ = VERSION
|
|
33
|
+
__all__ = [
|
|
34
|
+
"ASGIMiddleware", "DjangoMiddleware", "OrbitError", "OrbitPolicyError", "PolicyEvent", "RightsRequest",
|
|
35
|
+
"close", "consent", "flush", "health", "init", "instrument_flask", "on_erase", "on_export", "on_policy", "on_withdraw", "Withdrawal",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
S = _core.S
|
|
39
|
+
_TRUE = re.compile(r"(1|true|yes|on)", re.I)
|
|
40
|
+
_SERVICE_OK = re.compile(r"[^A-Za-z0-9._ -]+")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _truthy(v: Optional[str]) -> bool:
|
|
44
|
+
return bool(v) and bool(_TRUE.fullmatch((v or "").strip()))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _service_name(name: str) -> str:
|
|
48
|
+
n = _SERVICE_OK.sub("-", name.strip()).lstrip("._ -")[:80]
|
|
49
|
+
return n or "default"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def default_service(cwd: Optional[str] = None) -> str:
|
|
53
|
+
"""`ORBIT_SERVICE`, else the project name in the nearest pyproject.toml / setup.cfg / setup.py, else the folder."""
|
|
54
|
+
env = os.environ.get("ORBIT_SERVICE")
|
|
55
|
+
if env:
|
|
56
|
+
return _service_name(env)
|
|
57
|
+
here = Path(cwd or os.getcwd()).resolve()
|
|
58
|
+
for d in [here, *here.parents][:6]:
|
|
59
|
+
for fname, rx in (
|
|
60
|
+
("pyproject.toml", r'(?ms)^\[(?:project|tool\.poetry)\][^\[]*?^name\s*=\s*["\']([^"\']+)["\']'),
|
|
61
|
+
("setup.cfg", r"(?ms)^\[metadata\][^\[]*?^name\s*=\s*(\S+)"),
|
|
62
|
+
("setup.py", r"""name\s*=\s*["']([^"']+)["']"""),
|
|
63
|
+
):
|
|
64
|
+
f = d / fname
|
|
65
|
+
try:
|
|
66
|
+
if f.is_file():
|
|
67
|
+
m = re.search(rx, f.read_text("utf-8", "replace")[:65536])
|
|
68
|
+
if m:
|
|
69
|
+
return _service_name(m.group(1))
|
|
70
|
+
except OSError:
|
|
71
|
+
continue
|
|
72
|
+
return _service_name(here.name)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def init(
|
|
76
|
+
key: Optional[str] = None,
|
|
77
|
+
service: Optional[str] = None,
|
|
78
|
+
*,
|
|
79
|
+
environment: Optional[str] = None,
|
|
80
|
+
release: Optional[str] = None,
|
|
81
|
+
endpoint: Optional[str] = None,
|
|
82
|
+
first_party_hosts: Iterable[str] = (),
|
|
83
|
+
debug: Optional[bool] = None,
|
|
84
|
+
dry_run: bool = False,
|
|
85
|
+
disabled: bool = False,
|
|
86
|
+
enforcement: bool = True,
|
|
87
|
+
instrument: bool = True,
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Starts the SDK. Arguments win over env (ORBIT_KEY, ORBIT_SERVICE, ORBIT_ENVIRONMENT, ORBIT_ENDPOINT, ORBIT_DEBUG),
|
|
90
|
+
except ORBIT_DISABLED=1 and ORBIT_ENFORCEMENT=off, which always win. Never raises."""
|
|
91
|
+
if S.initialized and not S.closed:
|
|
92
|
+
if S.debug:
|
|
93
|
+
_core.debug("init called twice; the second call is ignored")
|
|
94
|
+
return
|
|
95
|
+
try:
|
|
96
|
+
env = os.environ
|
|
97
|
+
S.key = key if key is not None else env.get("ORBIT_KEY", "")
|
|
98
|
+
S.service = _service_name(service) if service else default_service()
|
|
99
|
+
S.environment = (environment or env.get("ORBIT_ENVIRONMENT") or "production")[:40]
|
|
100
|
+
S.release = (release or env.get("ORBIT_RELEASE") or None)
|
|
101
|
+
S.release = S.release[:60] if S.release else None
|
|
102
|
+
S.debug = debug if debug is not None else _truthy(env.get("ORBIT_DEBUG"))
|
|
103
|
+
S.dry_run = dry_run
|
|
104
|
+
S.enforcement = enforcement and (env.get("ORBIT_ENFORCEMENT", "").strip().lower() not in ("off", "0", "false", "no"))
|
|
105
|
+
S.user_first_party = [h.lower() for h in first_party_hosts]
|
|
106
|
+
S.local_disabled = disabled or _truthy(env.get("ORBIT_DISABLED")) or (not S.key and not dry_run)
|
|
107
|
+
try:
|
|
108
|
+
S.endpoint = _core.parse_endpoint(endpoint or env.get("ORBIT_ENDPOINT") or _core.DEFAULT_ENDPOINT)
|
|
109
|
+
except ValueError as e:
|
|
110
|
+
S.endpoint = None
|
|
111
|
+
S.local_disabled = True
|
|
112
|
+
_core.debug(f"disabled: {e}")
|
|
113
|
+
S.initialized = True
|
|
114
|
+
S.closed = False
|
|
115
|
+
if S.local_disabled:
|
|
116
|
+
if S.debug or (not S.key and not disabled and not _truthy(env.get("ORBIT_DISABLED"))):
|
|
117
|
+
_core.debug("disabled" + ("" if S.key else " (no key: set ORBIT_KEY or pass key=...)"))
|
|
118
|
+
return
|
|
119
|
+
patch_logging()
|
|
120
|
+
if instrument:
|
|
121
|
+
_autoimport.install()
|
|
122
|
+
_core.AUTO["mode"] = "import-hook"
|
|
123
|
+
_core.start_background()
|
|
124
|
+
if S.debug:
|
|
125
|
+
ep = S.endpoint.netloc if S.endpoint else "-"
|
|
126
|
+
_core.debug(f"started: service={S.service} environment={S.environment} endpoint={ep}"
|
|
127
|
+
f"{' (dry run)' if S.dry_run else ''}{'' if S.enforcement else ' (enforcement off)'}")
|
|
128
|
+
except Exception as e:
|
|
129
|
+
_core.fault(e)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def flush(timeout: float = 5.0) -> bool:
|
|
133
|
+
"""Sends what was recorded now, waiting at most `timeout` s. Serverless: call at the end of each invocation."""
|
|
134
|
+
try:
|
|
135
|
+
return _core.flush(timeout)
|
|
136
|
+
except Exception as e:
|
|
137
|
+
_core.fault(e)
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def close(timeout: float = 2.0) -> None:
|
|
142
|
+
"""Final flush (bounded by `timeout`), then stops background threads. Patches stay but record nothing."""
|
|
143
|
+
if not S.initialized or S.closed:
|
|
144
|
+
return
|
|
145
|
+
try:
|
|
146
|
+
if not S.local_disabled:
|
|
147
|
+
_core.flush(timeout)
|
|
148
|
+
_core.stop_background()
|
|
149
|
+
_autoimport.uninstall()
|
|
150
|
+
except Exception as e:
|
|
151
|
+
_core.fault(e)
|
|
152
|
+
S.closed = True
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
atexit.register(close)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def health() -> dict[str, Any]:
|
|
159
|
+
"""`{enabled, queued, lastFlushAt, lastError, dropped, sdkErrors, auto: {mode, detected, missed, errors}}`."""
|
|
160
|
+
return _core.health()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
H = TypeVar("H", bound=Callable[..., Any])
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _register(kind: str, fn: Any, loop: Optional[asyncio.AbstractEventLoop]) -> Any:
|
|
167
|
+
def deco(f: H) -> H:
|
|
168
|
+
setattr(S, kind, (f, loop))
|
|
169
|
+
try:
|
|
170
|
+
_core.start_tasks()
|
|
171
|
+
if S.initialized and not S.local_disabled:
|
|
172
|
+
_core._wake.set() # announce the handler so Orbit starts handing out tasks
|
|
173
|
+
except Exception as e:
|
|
174
|
+
_core.fault(e)
|
|
175
|
+
return f
|
|
176
|
+
|
|
177
|
+
return deco(fn) if fn is not None else deco
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@overload
|
|
181
|
+
def on_erase(fn: H, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> H: ...
|
|
182
|
+
@overload
|
|
183
|
+
def on_erase(fn: None = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Callable[[H], H]: ...
|
|
184
|
+
def on_erase(fn: Any = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Any:
|
|
185
|
+
"""Registers the erasure handler: `fn(RightsRequest) -> {"erased": n}` (sync or async). With `loop=`, an async
|
|
186
|
+
handler runs on that event loop (use your app's loop when the handler needs its connection pool)."""
|
|
187
|
+
return _register("erase", fn, loop)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@overload
|
|
191
|
+
def on_export(fn: H, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> H: ...
|
|
192
|
+
@overload
|
|
193
|
+
def on_export(fn: None = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Callable[[H], H]: ...
|
|
194
|
+
def on_export(fn: Any = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Any:
|
|
195
|
+
"""Registers the export handler: `fn(RightsRequest) -> {"categories": [...], "records": n}`. Also runs deletion
|
|
196
|
+
verification (`records == 0` after an erasure means verified)."""
|
|
197
|
+
return _register("export", fn, loop)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@overload
|
|
201
|
+
def on_withdraw(fn: H, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> H: ...
|
|
202
|
+
@overload
|
|
203
|
+
def on_withdraw(fn: None = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Callable[[H], H]: ...
|
|
204
|
+
def on_withdraw(fn: Any = None, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> Any:
|
|
205
|
+
"""Registers `fn(Withdrawal)` for when a person withdraws consent for a purpose (sync or async): stop and undo that
|
|
206
|
+
use in your systems, e.g. unsubscribe. Orbit already drops its cached consent for the person either way."""
|
|
207
|
+
return _register("withdraw", fn, loop)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def on_policy(fn: PolicyListener) -> PolicyListener:
|
|
211
|
+
"""Calls `fn(PolicyEvent)` for every warned / approval_needed / blocked outbound call. Usable as a decorator."""
|
|
212
|
+
S.listeners.append(fn)
|
|
213
|
+
return fn
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class _Consent:
|
|
217
|
+
"""`orbit.consent`: checks (fail-safe, cached 60 s) and records consent."""
|
|
218
|
+
|
|
219
|
+
@staticmethod
|
|
220
|
+
def allowed(principal_ref: str, purpose: str) -> bool:
|
|
221
|
+
"""True for granted/essential purposes, else False. Waits at most 1.5 s; Orbit unreachable and nothing cached:
|
|
222
|
+
False for consent purposes, True for essential ones. Never raises."""
|
|
223
|
+
return _core.consent_allowed(principal_ref, purpose)
|
|
224
|
+
|
|
225
|
+
@staticmethod
|
|
226
|
+
async def allowed_async(principal_ref: str, purpose: str) -> bool:
|
|
227
|
+
return await _core.consent_allowed_async(principal_ref, purpose)
|
|
228
|
+
|
|
229
|
+
@staticmethod
|
|
230
|
+
@contextlib.contextmanager
|
|
231
|
+
def scope() -> Iterator[None]:
|
|
232
|
+
"""A fresh consent context for work outside a web request (a job, a queue consumer): checks made inside are
|
|
233
|
+
attributed only to outbound calls made inside. Web requests get one automatically."""
|
|
234
|
+
tok = _core.consent_ctx.set(_core.ConsentCtx())
|
|
235
|
+
try:
|
|
236
|
+
yield
|
|
237
|
+
finally:
|
|
238
|
+
_core.consent_ctx.reset(tok)
|
|
239
|
+
|
|
240
|
+
@staticmethod
|
|
241
|
+
def record(principal_ref: str, choices: Iterable[Mapping[str, str]], *, channel: str = "api", language: str = "en",
|
|
242
|
+
notice_version: int = 0, email: Optional[str] = None) -> Any:
|
|
243
|
+
"""Records choices made in your own UI, e.g. `[{"purpose": "marketing", "action": "granted"}]`. Raises
|
|
244
|
+
OrbitError if Orbit did not store them. `email` (opt-in) links the consent to the privacy centre."""
|
|
245
|
+
return _core.consent_record(principal_ref, list(choices), channel=channel, language=language,
|
|
246
|
+
notice_version=notice_version, email=email)
|
|
247
|
+
|
|
248
|
+
@staticmethod
|
|
249
|
+
async def record_async(principal_ref: str, choices: Iterable[Mapping[str, str]], *, channel: str = "api",
|
|
250
|
+
language: str = "en", notice_version: int = 0, email: Optional[str] = None) -> Any:
|
|
251
|
+
ctx = _core.consent_ctx.get()
|
|
252
|
+
result = await asyncio.get_running_loop().run_in_executor(None, lambda: _core.consent_record(
|
|
253
|
+
principal_ref, list(choices), channel=channel, language=language, notice_version=notice_version, email=email))
|
|
254
|
+
if ctx is not None:
|
|
255
|
+
ctx.recorded = True
|
|
256
|
+
return result
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
consent = _Consent()
|
orbit/_autoimport.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Automatic instrumentation: a sys.meta_path finder patches supported libraries right after they are imported
|
|
2
|
+
(modules already imported are patched at install). It only observes imports: when anything about it fails, the import
|
|
3
|
+
proceeds exactly as without the SDK."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import importlib.abc
|
|
8
|
+
import importlib.machinery
|
|
9
|
+
import sys
|
|
10
|
+
import threading
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
from types import ModuleType
|
|
13
|
+
from typing import Any, Callable, Optional
|
|
14
|
+
|
|
15
|
+
from . import _core as C
|
|
16
|
+
from . import _db, _inbound, _outbound
|
|
17
|
+
|
|
18
|
+
# module name -> (distribution for the version label, patcher)
|
|
19
|
+
TARGETS: dict[str, tuple[str, Callable[[Any], None]]] = {
|
|
20
|
+
"requests": ("requests", _outbound.patch_requests),
|
|
21
|
+
"urllib3": ("urllib3", _outbound.patch_urllib3),
|
|
22
|
+
"httpx": ("httpx", _outbound.patch_httpx),
|
|
23
|
+
"aiohttp": ("aiohttp", _outbound.patch_aiohttp),
|
|
24
|
+
"urllib.request": ("", _outbound.patch_urllib),
|
|
25
|
+
"starlette.applications": ("starlette", _inbound.patch_starlette),
|
|
26
|
+
"flask": ("flask", _inbound.patch_flask),
|
|
27
|
+
"django.core.handlers.base": ("django", _inbound.patch_django_handler),
|
|
28
|
+
"django.db.backends.signals": ("django", _db.patch_django_db),
|
|
29
|
+
"psycopg2": ("psycopg2-binary", _db.patch_psycopg2),
|
|
30
|
+
"psycopg": ("psycopg", _db.patch_psycopg),
|
|
31
|
+
"asyncpg": ("asyncpg", _db.patch_asyncpg),
|
|
32
|
+
"sqlalchemy": ("sqlalchemy", _db.patch_sqlalchemy),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
_done: set[str] = set()
|
|
36
|
+
_busy = threading.local()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _label(name: str, dist: str) -> str:
|
|
40
|
+
if not dist:
|
|
41
|
+
return name
|
|
42
|
+
from importlib import metadata
|
|
43
|
+
|
|
44
|
+
for d in (dist, name, dist.replace("-binary", "")):
|
|
45
|
+
try:
|
|
46
|
+
return f"{name}@{metadata.version(d)}"
|
|
47
|
+
except Exception:
|
|
48
|
+
continue
|
|
49
|
+
return name
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def instrument(name: str, module: Any) -> None:
|
|
53
|
+
"""Runs the patcher for one loaded module, once. Never raises."""
|
|
54
|
+
if name in _done:
|
|
55
|
+
return
|
|
56
|
+
_done.add(name)
|
|
57
|
+
dist, patcher = TARGETS[name]
|
|
58
|
+
try:
|
|
59
|
+
patcher(module)
|
|
60
|
+
C.AUTO["detected"].append(_label(name, dist))
|
|
61
|
+
except Exception as e:
|
|
62
|
+
C.AUTO["missed"].append(name)
|
|
63
|
+
C.AUTO["errors"].append(f"{name}: {type(e).__name__}: {e}"[:300])
|
|
64
|
+
C.fault(e)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _Finder(importlib.abc.MetaPathFinder):
|
|
68
|
+
def find_spec(self, fullname: str, path: Optional[Sequence[str]], target: Optional[ModuleType] = None) -> Optional[importlib.machinery.ModuleSpec]:
|
|
69
|
+
if fullname not in TARGETS or fullname in _done or getattr(_busy, "on", False):
|
|
70
|
+
return None
|
|
71
|
+
_busy.on = True
|
|
72
|
+
try:
|
|
73
|
+
for finder in sys.meta_path:
|
|
74
|
+
if finder is self or not hasattr(finder, "find_spec"):
|
|
75
|
+
continue
|
|
76
|
+
spec = finder.find_spec(fullname, path, target)
|
|
77
|
+
if spec is None:
|
|
78
|
+
continue
|
|
79
|
+
loader = spec.loader
|
|
80
|
+
exec_module = getattr(loader, "exec_module", None)
|
|
81
|
+
if loader is not None and callable(exec_module):
|
|
82
|
+
def patched(module: ModuleType, _orig: Callable[[ModuleType], None] = exec_module, _loader: Any = loader) -> None:
|
|
83
|
+
try:
|
|
84
|
+
_orig(module) # the import itself: its errors propagate untouched
|
|
85
|
+
finally:
|
|
86
|
+
try:
|
|
87
|
+
del _loader.exec_module # back to the class method
|
|
88
|
+
except Exception:
|
|
89
|
+
pass
|
|
90
|
+
instrument(fullname, module)
|
|
91
|
+
|
|
92
|
+
loader.exec_module = patched # type: ignore[method-assign] # per-spec loader instance
|
|
93
|
+
return spec
|
|
94
|
+
return None
|
|
95
|
+
except Exception as e:
|
|
96
|
+
C.fault(e)
|
|
97
|
+
return None # let the normal machinery import it
|
|
98
|
+
finally:
|
|
99
|
+
_busy.on = False
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
_finder = _Finder()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def install() -> None:
|
|
106
|
+
try:
|
|
107
|
+
if _finder not in sys.meta_path:
|
|
108
|
+
sys.meta_path.insert(0, _finder)
|
|
109
|
+
for name in TARGETS:
|
|
110
|
+
mod = sys.modules.get(name)
|
|
111
|
+
if mod is not None:
|
|
112
|
+
instrument(name, mod)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
C.fault(e)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def uninstall() -> None:
|
|
118
|
+
try:
|
|
119
|
+
sys.meta_path.remove(_finder)
|
|
120
|
+
except ValueError:
|
|
121
|
+
pass
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Loaded by `python -m orbit.run`: starts Orbit, then hands over to the application's own sitecustomize (if any)."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import orbit.auto # noqa: F401
|
|
9
|
+
except Exception as e: # never stop the application from starting
|
|
10
|
+
sys.stderr.write(f"[orbit] not started: {type(e).__name__}: {e}\n")
|
|
11
|
+
|
|
12
|
+
_here = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
+
sys.path[:] = [p for p in sys.path if os.path.abspath(p or ".") != _here]
|
|
14
|
+
_self = sys.modules.pop("sitecustomize")
|
|
15
|
+
try:
|
|
16
|
+
importlib.import_module("sitecustomize")
|
|
17
|
+
except ImportError:
|
|
18
|
+
sys.modules["sitecustomize"] = _self # the import system expects the entry back
|
orbit/_classify.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""In-memory classification (port of packages/sdk-node/src/classify.ts).
|
|
2
|
+
|
|
3
|
+
Everything here returns NAMES and CATEGORIES only; values are inspected and dropped.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
from collections.abc import Iterable, Mapping
|
|
11
|
+
from typing import Any, NamedTuple, Optional
|
|
12
|
+
from urllib.parse import parse_qsl, unquote
|
|
13
|
+
|
|
14
|
+
from ._pii import categories_from_name, classify_value
|
|
15
|
+
|
|
16
|
+
MAX_NODES = 1000
|
|
17
|
+
MAX_DEPTH = 8
|
|
18
|
+
MAX_ARRAY = 20
|
|
19
|
+
BODY_CAP = 64 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Hits:
|
|
23
|
+
__slots__ = ("categories", "consent", "fields", "kinds")
|
|
24
|
+
|
|
25
|
+
def __init__(self) -> None:
|
|
26
|
+
self.fields: set[str] = set()
|
|
27
|
+
self.categories: set[str] = set()
|
|
28
|
+
self.kinds: set[str] = set()
|
|
29
|
+
self.consent: set[str] = set() # v1.2: field names that look like a consent choice
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_KEY_BAD = re.compile(r"[0-9]{4,}|@")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def safe_key(k: str) -> str:
|
|
36
|
+
"""A key that could itself be personal data (an email-keyed map, a phone number, an id) is replaced by '*'."""
|
|
37
|
+
if not k or len(k) > 64 or _KEY_BAD.search(k) or classify_value(k):
|
|
38
|
+
return "*"
|
|
39
|
+
return k.lower()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def add_name(h: Hits, name: str) -> bool:
|
|
43
|
+
cats = categories_from_name(name)
|
|
44
|
+
h.categories.update(cats)
|
|
45
|
+
return bool(cats)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _scalar_text(v: Any) -> Optional[str]:
|
|
49
|
+
if isinstance(v, str):
|
|
50
|
+
return v
|
|
51
|
+
if isinstance(v, bool):
|
|
52
|
+
return None
|
|
53
|
+
if isinstance(v, int):
|
|
54
|
+
return str(v)
|
|
55
|
+
if isinstance(v, float):
|
|
56
|
+
return str(int(v)) if v.is_integer() and abs(v) < 1e21 else repr(v)
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def add_value(h: Hits, v: Any) -> bool:
|
|
61
|
+
"""Classifies one scalar. Numbers count: an Aadhaar or a phone is often stored as a number."""
|
|
62
|
+
s = _scalar_text(v)
|
|
63
|
+
if s is None or len(s) < 5 or len(s) > 200:
|
|
64
|
+
return False
|
|
65
|
+
m = classify_value(s)
|
|
66
|
+
if not m:
|
|
67
|
+
return False
|
|
68
|
+
h.categories.add(m.category)
|
|
69
|
+
h.kinds.add(m.kind)
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
_CONSENT_NAME = re.compile(r"consent|opt_?in|agree|accept|subscribe|marketing|newsletter|whatsapp_updates|terms|tnc|permission", re.I | re.A)
|
|
74
|
+
_BOOLISH = {"true", "false", "yes", "no", "on", "off", "1", "0", "y", "n", "checked", "unchecked", "granted", "denied"}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _consent_like(leaf: str, v: Any) -> bool:
|
|
78
|
+
if not _CONSENT_NAME.search(leaf):
|
|
79
|
+
return False
|
|
80
|
+
if isinstance(v, bool) or (isinstance(v, int) and v in (0, 1)):
|
|
81
|
+
return True
|
|
82
|
+
return isinstance(v, str) and v.strip().lower() in _BOOLISH
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def flatten(value: Any, prefix: str, h: Hits, max_fields: int, depth: int = MAX_DEPTH) -> None:
|
|
86
|
+
"""Walks a parsed body: every leaf whose name or value classifies adds its dotted path (`prefix.a.b`, list indexes
|
|
87
|
+
dropped) to fields. Bounded by depth, node count and `max_fields`."""
|
|
88
|
+
nodes = 0
|
|
89
|
+
|
|
90
|
+
def walk(v: Any, path: str, leaf: str, d: int) -> None:
|
|
91
|
+
nonlocal nodes
|
|
92
|
+
nodes += 1
|
|
93
|
+
if nodes > MAX_NODES:
|
|
94
|
+
return
|
|
95
|
+
if isinstance(v, Mapping):
|
|
96
|
+
if d >= depth:
|
|
97
|
+
return
|
|
98
|
+
for k in list(v.keys())[:MAX_NODES]:
|
|
99
|
+
key = safe_key(str(k))
|
|
100
|
+
walk(v[k], f"{path}.{key}" if path else key, key, d + 1)
|
|
101
|
+
return
|
|
102
|
+
if isinstance(v, (list, tuple)):
|
|
103
|
+
if d >= depth:
|
|
104
|
+
return
|
|
105
|
+
for x in v[:MAX_ARRAY]:
|
|
106
|
+
walk(x, path, leaf, d + 1)
|
|
107
|
+
return
|
|
108
|
+
by_name = leaf != "*" and leaf != "" and add_name(h, leaf)
|
|
109
|
+
by_value = add_value(h, v)
|
|
110
|
+
if path and (by_name or by_value) and len(h.fields) < max_fields:
|
|
111
|
+
h.fields.add(path[:200])
|
|
112
|
+
if path and leaf and leaf != "*" and len(h.consent) < max_fields and _consent_like(leaf, v):
|
|
113
|
+
h.consent.add(path[:200])
|
|
114
|
+
|
|
115
|
+
walk(value, prefix, "", 0)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
_FORMISH = re.compile(r"[\s<]")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def parse_body(raw: Any) -> Any:
|
|
122
|
+
"""Parses a request body (str/bytes, capped) as JSON or urlencoded; anything else is not inspected (None)."""
|
|
123
|
+
if isinstance(raw, (bytes, bytearray, memoryview)):
|
|
124
|
+
try:
|
|
125
|
+
text = bytes(raw[:BODY_CAP]).decode("utf-8", "replace")
|
|
126
|
+
except Exception:
|
|
127
|
+
return None
|
|
128
|
+
elif isinstance(raw, str):
|
|
129
|
+
text = raw[:BODY_CAP]
|
|
130
|
+
else:
|
|
131
|
+
return None
|
|
132
|
+
t = text.lstrip()
|
|
133
|
+
if t.startswith(("{", "[")):
|
|
134
|
+
try:
|
|
135
|
+
return json.loads(t)
|
|
136
|
+
except ValueError:
|
|
137
|
+
return None
|
|
138
|
+
if "=" in t and not _FORMISH.search(t[:200]):
|
|
139
|
+
return dict(parse_qsl(t, keep_blank_values=True))
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I)
|
|
144
|
+
_HEX = re.compile(r"[0-9a-f]{16,}", re.I)
|
|
145
|
+
_IDISH = re.compile(r"[0-9]{3,}|@")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def template_path(raw: str, h: Optional[Hits] = None, max_fields: int = 50) -> str:
|
|
149
|
+
"""`/v1/customer/9876543210/orders?x=1` -> `/v1/customer/:id/orders`. Id-like or personal segments are replaced
|
|
150
|
+
(and classified into `h` when given); the query string is classified as `query.<name>` fields and dropped."""
|
|
151
|
+
q = raw.find("?")
|
|
152
|
+
path = raw if q < 0 else raw[:q]
|
|
153
|
+
out: list[str] = []
|
|
154
|
+
for seg in path.split("/"):
|
|
155
|
+
if not seg:
|
|
156
|
+
out.append(seg)
|
|
157
|
+
continue
|
|
158
|
+
d = unquote(seg)
|
|
159
|
+
if h is not None and add_value(h, d):
|
|
160
|
+
if len(h.fields) < max_fields:
|
|
161
|
+
h.fields.add("path")
|
|
162
|
+
out.append(":id")
|
|
163
|
+
continue
|
|
164
|
+
idish = _IDISH.search(d) or _UUID.fullmatch(d) or _HEX.fullmatch(d) or len(d) > 40
|
|
165
|
+
out.append(":id" if idish else seg)
|
|
166
|
+
if h is not None and q >= 0:
|
|
167
|
+
flatten(dict(parse_qsl(raw[q + 1 :], keep_blank_values=True)), "query", h, max_fields, 2)
|
|
168
|
+
return ("/".join(out) or "/")[:300]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
_ROUTE_PARAM = re.compile(r"<(?:[^:<>]+:)?([^<>]+)>|\{([^{}:]+)(?::[^{}]*)?\}")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def route_template(route: str) -> str:
|
|
175
|
+
"""Framework route syntax to the protocol's: Flask/Django `<int:id>` and Starlette `{id}` / `{p:path}` -> `:id`."""
|
|
176
|
+
return _ROUTE_PARAM.sub(lambda m: ":" + (m.group(1) or m.group(2)), route)[:300]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
_TOKEN_SPLIT = re.compile(r"[\s,;'\"`()\[\]{}<>|]+")
|
|
180
|
+
_TOKEN_TRIM = re.compile(r"^[=:]+|[.:=]+$")
|
|
181
|
+
_GROUPS = re.compile(r"\b[0-9]{4}[ -][0-9]{4}[ -][0-9]{4}(?:[ -][0-9]{4})?\b", re.A)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def classify_text(s: str, h: Hits) -> None:
|
|
185
|
+
"""Free text (a log line): each token, spaced Aadhaar/card groups and short whole strings are classified."""
|
|
186
|
+
if len(s) <= 200:
|
|
187
|
+
add_value(h, s)
|
|
188
|
+
for tok in _TOKEN_SPLIT.split(s):
|
|
189
|
+
if len(tok) >= 5:
|
|
190
|
+
add_value(h, _TOKEN_TRIM.sub("", tok))
|
|
191
|
+
eq = tok.find("=")
|
|
192
|
+
if eq > 0:
|
|
193
|
+
add_value(h, tok[eq + 1 :])
|
|
194
|
+
for m in _GROUPS.finditer(s):
|
|
195
|
+
add_value(h, m.group(0))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def classify_log_args(args: Iterable[Any], h: Hits, budget: int = 8192) -> None:
|
|
199
|
+
"""Log arguments: strings as text, containers walked <= 2 levels; values only (key names would flag every `message`)."""
|
|
200
|
+
left = budget
|
|
201
|
+
|
|
202
|
+
def visit(v: Any, d: int) -> None:
|
|
203
|
+
nonlocal left
|
|
204
|
+
if left <= 0:
|
|
205
|
+
return
|
|
206
|
+
if isinstance(v, str):
|
|
207
|
+
s = v[:left]
|
|
208
|
+
left -= len(s)
|
|
209
|
+
classify_text(s, h)
|
|
210
|
+
return
|
|
211
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool):
|
|
212
|
+
add_value(h, v)
|
|
213
|
+
left -= 16
|
|
214
|
+
return
|
|
215
|
+
if isinstance(v, BaseException):
|
|
216
|
+
visit(str(v), d)
|
|
217
|
+
return
|
|
218
|
+
if d >= 2:
|
|
219
|
+
return
|
|
220
|
+
if isinstance(v, Mapping):
|
|
221
|
+
for x in list(v.values())[:MAX_NODES]:
|
|
222
|
+
visit(x, d + 1)
|
|
223
|
+
elif isinstance(v, (list, tuple, set, frozenset)):
|
|
224
|
+
for i, x in enumerate(v):
|
|
225
|
+
if i >= MAX_ARRAY:
|
|
226
|
+
break
|
|
227
|
+
visit(x, d + 1)
|
|
228
|
+
|
|
229
|
+
for a in args:
|
|
230
|
+
visit(a, 0)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
class SqlInfo(NamedTuple):
|
|
234
|
+
operation: str
|
|
235
|
+
target: str
|
|
236
|
+
columns: list[str]
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
_ID = r'((?:"[^"]+"|`[^`]+`|[\w$]+)(?:\.(?:"[^"]+"|`[^`]+`|[\w$]+))?)'
|
|
240
|
+
_INSERT = re.compile(r"\binsert\s+(?:ignore\s+)?into\s+" + _ID + r"\s*(?:\(([^)]*)\))?", re.I | re.A)
|
|
241
|
+
_UPDATE = re.compile(r"\bupdate\s+(?:only\s+)?" + _ID + r"\s+(?:(?:as\s+)?\w+\s+)?set\s+([\s\S]*?)(?:\bwhere\b|\breturning\b|\bfrom\b|$)", re.I | re.A)
|
|
242
|
+
_DELETE = re.compile(r"\bdelete\s+from\s+" + _ID, re.I | re.A)
|
|
243
|
+
_SELECT = re.compile(r"\bselect\b[\s\S]*?\bfrom\s+" + _ID, re.I | re.A)
|
|
244
|
+
_COMMENTS = re.compile(r"--[^\n]*|/\*[\s\S]*?\*/")
|
|
245
|
+
_STRINGS = re.compile(r"'(?:[^']|'')*'")
|
|
246
|
+
_SET_COL = re.compile(r"([\w$\"`.]+)\s*=", re.A)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _clean(s: str) -> str:
|
|
250
|
+
return s.replace('"', "").replace("`", "").strip().lower()[:120]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def parse_sql(sql: str) -> Optional[SqlInfo]:
|
|
254
|
+
"""Cheap statement parse (no SQL parser): operation, first target table, INSERT/UPDATE column names."""
|
|
255
|
+
s = _STRINGS.sub("''", _COMMENTS.sub(" ", sql[:8192]))
|
|
256
|
+
m = _INSERT.search(s)
|
|
257
|
+
if m:
|
|
258
|
+
cols = [c for c in (_clean(x) for x in m.group(2).split(",")) if c] if m.group(2) else []
|
|
259
|
+
return SqlInfo("insert", _clean(m.group(1)), cols)
|
|
260
|
+
m = _UPDATE.search(s)
|
|
261
|
+
if m:
|
|
262
|
+
return SqlInfo("update", _clean(m.group(1)), [_clean(c).split(".")[-1] for c in _SET_COL.findall(m.group(2))])
|
|
263
|
+
m = _DELETE.search(s)
|
|
264
|
+
if m:
|
|
265
|
+
return SqlInfo("delete", _clean(m.group(1)), [])
|
|
266
|
+
m = _SELECT.search(s)
|
|
267
|
+
if m:
|
|
268
|
+
return SqlInfo("select", _clean(m.group(1)), [])
|
|
269
|
+
return None
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
_WS_SPLIT = re.compile(r"(\s+)")
|
|
273
|
+
_REDACT = re.compile(r"\S+@\S+|[0-9]{4,}")
|
|
274
|
+
_TRAIL = re.compile(r"[.,;:]+$")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def scrub_text(s: str) -> str:
|
|
278
|
+
"""Error text from customer handlers, made safe to send: classified tokens, emails and digit runs are redacted."""
|
|
279
|
+
parts = _WS_SPLIT.split(s[:2000])
|
|
280
|
+
out = ["[redacted]" if (_REDACT.search(t) or classify_value(_TRAIL.sub("", t))) else t for t in parts]
|
|
281
|
+
return "".join(out)[:1000]
|