webpilot-engine 2.0.3__tar.gz → 2.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/PKG-INFO +8 -3
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/browser_adapter.py +4 -9
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/live_session_manager.py +23 -67
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/builder.py +23 -18
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/cli.py +22 -4
- webpilot_engine-2.0.5/common/__init__.py +17 -0
- webpilot_engine-2.0.5/common/cookies.py +48 -0
- webpilot_engine-2.0.5/common/logging.py +52 -0
- webpilot_engine-2.0.5/common/process_manager.py +259 -0
- webpilot_engine-2.0.5/common/retry.py +31 -0
- webpilot_engine-2.0.5/common/secrets.py +120 -0
- webpilot_engine-2.0.5/common/text.py +19 -0
- webpilot_engine-2.0.5/config/device_profile.example.json +17 -0
- webpilot_engine-2.0.5/config/device_profile.json +17 -0
- webpilot_engine-2.0.5/config/settings.example.json +12 -0
- webpilot_engine-2.0.5/config/settings.json +12 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/pyproject.toml +14 -4
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/client.py +53 -53
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/master_daemon.py +43 -16
- webpilot_engine-2.0.5/supervisor/security.py +109 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/shell.py +1 -1
- webpilot_engine-2.0.5/tests/test_enterprise_integration.py +103 -0
- webpilot_engine-2.0.5/tests/test_process_manager.py +57 -0
- webpilot_engine-2.0.5/tests/test_secrets_resolution.py +91 -0
- webpilot_engine-2.0.5/tests/test_supervisor_security.py +89 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/PKG-INFO +8 -3
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/SOURCES.txt +16 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/requires.txt +1 -1
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/top_level.txt +1 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/README.md +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/base.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/dom_scripts.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/envelope_builder.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/adapters/upload_adapter.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/config/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/config/settings.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/constants/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/constants/contracts.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/constants/selectors.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/constants/timeouts.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/core/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/core/exceptions.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/core/models.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/engine.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/exceptions.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/mcp_server.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/models.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/orchestration/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/orchestration/context.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/orchestration/flow_orchestrator.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/orchestration/session_coordinator.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/auth_navigation_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/field_interaction_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/form_filler_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/inspection_formatter_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/inspection_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/services/reactive_interaction_service.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/setup.cfg +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/contracts.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/supervisor/worker_process.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_config_and_builder.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_contracts.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_engine_facade.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_live_session.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_mcp_server.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_orchestration.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_redaction.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_services.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/tests/test_supervisor.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/validators/__init__.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/validators/form_validator.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/validators/payload_validator.py +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/dependency_links.txt +0 -0
- {webpilot_engine-2.0.3 → webpilot_engine-2.0.5}/webpilot_engine.egg-info/entry_points.txt +0 -0
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: webpilot-engine
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.5
|
|
4
4
|
Summary: WebPilot: Enterprise Autonomous Browser Engine, Universal Form Inspector & AI Agent Web Runtime
|
|
5
5
|
Author: Abdulkarim Salih, WebPilot Core Team
|
|
6
6
|
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/AbdulkarimSSS/webpilot
|
|
8
|
+
Project-URL: Documentation, https://github.com/AbdulkarimSSS/webpilot#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/AbdulkarimSSS/webpilot.git
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/AbdulkarimSSS/webpilot/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/AbdulkarimSSS/webpilot/blob/master/CHANGELOG.md
|
|
7
12
|
Keywords: webpilot,browser-runtime,ai-agent,web-automation,playwright,form-filler,ats-automation,sap-successfactors,workday,browser-supervisor,process-isolation
|
|
8
|
-
Classifier: Development Status ::
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
14
|
Classifier: Intended Audience :: Developers
|
|
10
15
|
Classifier: Topic :: Software Development :: Testing
|
|
11
16
|
Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
|
|
@@ -28,7 +33,7 @@ Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
|
28
33
|
Requires-Dist: pytest-mock>=3.12.0; extra == "dev"
|
|
29
34
|
Requires-Dist: Faker>=24.0.0; extra == "dev"
|
|
30
35
|
Provides-Extra: mcp
|
|
31
|
-
Requires-Dist: mcp
|
|
36
|
+
Requires-Dist: mcp<2,>=1.0.0; extra == "mcp"
|
|
32
37
|
|
|
33
38
|
# WebPilot: Autonomous Multi-Tier Browser Automation Engine & AI Agent Web Runtime
|
|
34
39
|
|
|
@@ -253,15 +253,10 @@ class BrowserAdapter(BrowserAdapterProtocol):
|
|
|
253
253
|
args=["--disable-blink-features=AutomationControlled"],
|
|
254
254
|
)
|
|
255
255
|
try:
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
self.browser_pid = int(out)
|
|
261
|
-
else:
|
|
262
|
-
out = subprocess.check_output(["pgrep", "-P", str(os.getpid())], text=True).strip().split()
|
|
263
|
-
if out and out[0].isdigit():
|
|
264
|
-
self.browser_pid = int(out[0])
|
|
256
|
+
from common.process_manager import get_process_manager
|
|
257
|
+
children = get_process_manager().get_child_pids(os.getpid())
|
|
258
|
+
if children:
|
|
259
|
+
self.browser_pid = children[0]
|
|
265
260
|
except Exception:
|
|
266
261
|
pass
|
|
267
262
|
|
|
@@ -124,16 +124,8 @@ class LiveSessionManager:
|
|
|
124
124
|
pid = state.get("pid")
|
|
125
125
|
if pid:
|
|
126
126
|
try:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
subprocess.run(
|
|
130
|
-
["taskkill", "/F", "/T", "/PID", str(pid)],
|
|
131
|
-
stdout=subprocess.DEVNULL,
|
|
132
|
-
stderr=subprocess.DEVNULL,
|
|
133
|
-
)
|
|
134
|
-
else:
|
|
135
|
-
os.kill(pid, 9)
|
|
136
|
-
return True
|
|
127
|
+
from common.process_manager import get_process_manager
|
|
128
|
+
return get_process_manager().kill_process_tree(pid, force=True)
|
|
137
129
|
except Exception:
|
|
138
130
|
pass
|
|
139
131
|
return False
|
|
@@ -210,13 +202,16 @@ class LiveSessionManager:
|
|
|
210
202
|
time.sleep(0.2)
|
|
211
203
|
|
|
212
204
|
if not active:
|
|
213
|
-
if
|
|
205
|
+
if proc_pid:
|
|
214
206
|
try:
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
207
|
+
if sys.platform == "win32":
|
|
208
|
+
subprocess.run(
|
|
209
|
+
["taskkill", "/F", "/T", "/PID", str(proc_pid)],
|
|
210
|
+
stdout=subprocess.DEVNULL,
|
|
211
|
+
stderr=subprocess.DEVNULL,
|
|
212
|
+
)
|
|
213
|
+
else:
|
|
214
|
+
os.kill(proc_pid, 9)
|
|
220
215
|
except Exception:
|
|
221
216
|
pass
|
|
222
217
|
raise RuntimeError(f"Chromium failed to open CDP port {port} within 5 seconds.")
|
|
@@ -264,62 +259,23 @@ class LiveSessionManager:
|
|
|
264
259
|
@classmethod
|
|
265
260
|
def _launch_wmi_process(cls, cmd_str: str, cwd: Optional[str] = None) -> Optional[int]:
|
|
266
261
|
"""Launches a detached process via Windows WMI outside any parent Job Object."""
|
|
262
|
+
from common.process_manager import get_process_manager
|
|
267
263
|
working_dir = cwd or PROJECT_ROOT
|
|
268
|
-
|
|
269
|
-
$startup = ([wmiclass]"Win32_ProcessStartup").CreateInstance()
|
|
270
|
-
$startup.ShowWindow = 0
|
|
271
|
-
$proc = ([wmiclass]"Win32_Process").Create(@'
|
|
272
|
-
{cmd_str}
|
|
273
|
-
'@, $null, $startup)
|
|
274
|
-
if ($proc.ReturnValue -eq 0) {{
|
|
275
|
-
Write-Output $proc.ProcessId
|
|
276
|
-
}} else {{
|
|
277
|
-
exit 1
|
|
278
|
-
}}
|
|
279
|
-
"""
|
|
280
|
-
try:
|
|
281
|
-
encoded = base64.b64encode(ps_script.encode("utf-16le")).decode("ascii")
|
|
282
|
-
res = subprocess.run(
|
|
283
|
-
["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
|
|
284
|
-
capture_output=True,
|
|
285
|
-
text=True,
|
|
286
|
-
timeout=10,
|
|
287
|
-
)
|
|
288
|
-
out = res.stdout.strip()
|
|
289
|
-
if res.returncode == 0 and out.isdigit():
|
|
290
|
-
return int(out)
|
|
291
|
-
else:
|
|
292
|
-
print(f"[!] WMI launch non-zero/non-digit: code={res.returncode}, out='{out}', err='{res.stderr.strip()}'")
|
|
293
|
-
except Exception as exc:
|
|
294
|
-
print(f"[!] WMI launch exception: {exc}")
|
|
295
|
-
return None
|
|
264
|
+
return get_process_manager().launch_detached(cmd_str, cwd=working_dir)
|
|
296
265
|
|
|
297
266
|
@classmethod
|
|
298
267
|
def _spawn_watchdog_supervisor(cls, browser_pid: int, port: int, timeout_seconds: int) -> None:
|
|
299
268
|
"""Launches a lightweight detached background watchdog to terminate idle browser."""
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
}
|
|
311
|
-
if sys.platform == "win32":
|
|
312
|
-
popen_kwargs["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
|
313
|
-
else:
|
|
314
|
-
popen_kwargs["start_new_session"] = True
|
|
315
|
-
|
|
316
|
-
try:
|
|
317
|
-
subprocess.Popen(
|
|
318
|
-
[sys.executable, "-m", "adapters.live_session_manager", str(browser_pid), str(port), str(timeout_seconds)],
|
|
319
|
-
**popen_kwargs,
|
|
320
|
-
)
|
|
321
|
-
except Exception:
|
|
322
|
-
pass
|
|
269
|
+
from common.process_manager import get_process_manager
|
|
270
|
+
watchdog_cmd = [
|
|
271
|
+
sys.executable,
|
|
272
|
+
"-m",
|
|
273
|
+
"adapters.live_session_manager",
|
|
274
|
+
str(browser_pid),
|
|
275
|
+
str(port),
|
|
276
|
+
str(timeout_seconds),
|
|
277
|
+
]
|
|
278
|
+
get_process_manager().launch_detached(watchdog_cmd, cwd=PROJECT_ROOT)
|
|
323
279
|
|
|
324
280
|
|
|
325
281
|
if __name__ == "__main__":
|
|
@@ -34,7 +34,17 @@ class FormPayloadBuilder:
|
|
|
34
34
|
|
|
35
35
|
@staticmethod
|
|
36
36
|
def load_from_file(filepath: str) -> List[Tuple[str, Any]]:
|
|
37
|
-
"""Load field key-value pairs from an external JSON file."""
|
|
37
|
+
"""Load field key-value pairs from an external JSON file or stdin ('-' or '@stdin')."""
|
|
38
|
+
import sys
|
|
39
|
+
from common.secrets import resolve_secret_value
|
|
40
|
+
|
|
41
|
+
if filepath in ("-", "@stdin"):
|
|
42
|
+
try:
|
|
43
|
+
raw_data = json.load(sys.stdin)
|
|
44
|
+
except Exception as exc:
|
|
45
|
+
raise ValidationError(f"Invalid JSON read from standard input: {exc}") from exc
|
|
46
|
+
return FormPayloadBuilder.normalize_payload(raw_data)
|
|
47
|
+
|
|
38
48
|
if not os.path.isfile(filepath):
|
|
39
49
|
raise ConfigurationError(f"Form payload data file not found: {filepath}")
|
|
40
50
|
|
|
@@ -48,18 +58,21 @@ class FormPayloadBuilder:
|
|
|
48
58
|
|
|
49
59
|
@staticmethod
|
|
50
60
|
def normalize_payload(raw_data: Union[Dict[str, Any], List[Any]]) -> List[Tuple[str, Any]]:
|
|
51
|
-
"""Normalizes dict or list structures into a clean list of (field, value) pairs."""
|
|
61
|
+
"""Normalizes dict or list structures into a clean list of (field, value) pairs and resolves secrets."""
|
|
62
|
+
from common.secrets import resolve_secret_value
|
|
63
|
+
|
|
52
64
|
fields: List[Tuple[str, Any]] = []
|
|
53
65
|
|
|
54
66
|
if isinstance(raw_data, dict):
|
|
55
67
|
for k, v in raw_data.items():
|
|
56
|
-
fields.append((str(k), v))
|
|
68
|
+
fields.append((str(k), resolve_secret_value(v, prompt_label=str(k))))
|
|
57
69
|
elif isinstance(raw_data, list):
|
|
58
70
|
for item in raw_data:
|
|
59
71
|
if isinstance(item, (list, tuple)) and len(item) == 2:
|
|
60
|
-
fields.append((str(item[0]), item[1]))
|
|
72
|
+
fields.append((str(item[0]), resolve_secret_value(item[1], prompt_label=str(item[0]))))
|
|
61
73
|
elif isinstance(item, dict) and "key" in item and "value" in item:
|
|
62
|
-
|
|
74
|
+
k = str(item["key"])
|
|
75
|
+
fields.append((k, resolve_secret_value(item["value"], prompt_label=k)))
|
|
63
76
|
else:
|
|
64
77
|
raise ValidationError(f"Unrecognized payload entry structure: {item}")
|
|
65
78
|
else:
|
|
@@ -68,16 +81,8 @@ class FormPayloadBuilder:
|
|
|
68
81
|
return fields
|
|
69
82
|
|
|
70
83
|
@staticmethod
|
|
71
|
-
def parse_cli_fill_arguments(fill_args: Optional[List[str]]) -> List[Tuple[str,
|
|
72
|
-
"""Parses CLI --fill
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
parsed: List[Tuple[str, str]] = []
|
|
77
|
-
for item in fill_args:
|
|
78
|
-
if "=" in item:
|
|
79
|
-
k, v = item.split("=", 1)
|
|
80
|
-
parsed.append((k.strip(), v.strip()))
|
|
81
|
-
else:
|
|
82
|
-
raise ValidationError(f"Invalid fill argument format: '{item}' (Expected 'key=value')")
|
|
83
|
-
return parsed
|
|
84
|
+
def parse_cli_fill_arguments(fill_args: Optional[List[str]]) -> List[Tuple[str, Any]]:
|
|
85
|
+
"""Parses CLI --fill arguments into key-value pairs and resolves secrets (@env, @stdin, @file)."""
|
|
86
|
+
from common.secrets import parse_and_resolve_fill_args
|
|
87
|
+
return parse_and_resolve_fill_args(fill_args)
|
|
88
|
+
|
|
@@ -116,11 +116,29 @@ def apply_cmd(args: argparse.Namespace) -> None:
|
|
|
116
116
|
"""Executes the universal form filling and submission workflow."""
|
|
117
117
|
auto_inspect = not getattr(args, "no_auto_inspect", False)
|
|
118
118
|
|
|
119
|
+
from common.secrets import parse_and_resolve_fill_args
|
|
120
|
+
from builder import FormPayloadBuilder
|
|
121
|
+
|
|
122
|
+
raw_fill_args = args.fill or []
|
|
123
|
+
resolved_pairs = parse_and_resolve_fill_args(raw_fill_args)
|
|
124
|
+
effective_fill_args = [f"{k}={v}" for k, v in resolved_pairs]
|
|
125
|
+
|
|
126
|
+
effective_data_path = args.data
|
|
127
|
+
if args.data in ("-", "@stdin"):
|
|
128
|
+
try:
|
|
129
|
+
stdin_json = json.load(sys.stdin)
|
|
130
|
+
stdin_pairs = FormPayloadBuilder.normalize_payload(stdin_json)
|
|
131
|
+
effective_fill_args.extend([f"{k}={v}" for k, v in stdin_pairs])
|
|
132
|
+
effective_data_path = None
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
print(f"[!] Error reading JSON payload from stdin: {exc}", file=sys.stderr)
|
|
135
|
+
sys.exit(1)
|
|
136
|
+
|
|
119
137
|
if getattr(args, "standalone", False):
|
|
120
138
|
req = ApplyRequestContext(
|
|
121
139
|
url=args.url,
|
|
122
|
-
data_path=
|
|
123
|
-
fill_arguments=
|
|
140
|
+
data_path=effective_data_path,
|
|
141
|
+
fill_arguments=effective_fill_args,
|
|
124
142
|
press_buttons=args.press or [],
|
|
125
143
|
upload_files=args.upload or [],
|
|
126
144
|
cookies_path=args.cookies,
|
|
@@ -165,8 +183,8 @@ def apply_cmd(args: argparse.Namespace) -> None:
|
|
|
165
183
|
req = SupervisorActionRequest(
|
|
166
184
|
action="apply",
|
|
167
185
|
url=args.url,
|
|
168
|
-
data_path=
|
|
169
|
-
fill_arguments=
|
|
186
|
+
data_path=effective_data_path,
|
|
187
|
+
fill_arguments=effective_fill_args,
|
|
170
188
|
press_buttons=args.press or [],
|
|
171
189
|
upload_files=args.upload or [],
|
|
172
190
|
cookies_path=args.cookies,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Common utilities package for universal web applier."""
|
|
2
|
+
|
|
3
|
+
from common.text import clean_label, truncate_text
|
|
4
|
+
from common.cookies import sanitize_cookies, load_and_sanitize_cookies, save_cookies
|
|
5
|
+
from common.retry import with_retry
|
|
6
|
+
from common.logging import redact_sensitive, log_event
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"clean_label",
|
|
10
|
+
"truncate_text",
|
|
11
|
+
"sanitize_cookies",
|
|
12
|
+
"load_and_sanitize_cookies",
|
|
13
|
+
"save_cookies",
|
|
14
|
+
"with_retry",
|
|
15
|
+
"redact_sensitive",
|
|
16
|
+
"log_event",
|
|
17
|
+
]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Cookie sanitization and persistence helpers for Playwright."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
from typing import Any, Dict, List, Optional
|
|
6
|
+
from core.exceptions import ValidationError
|
|
7
|
+
|
|
8
|
+
INVALID_COOKIE_KEYS = ("hostOnly", "session", "storeId", "id")
|
|
9
|
+
VALID_SAME_SITE_VALUES = ("Strict", "Lax", "None")
|
|
10
|
+
|
|
11
|
+
def sanitize_cookies(raw_cookies: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
12
|
+
"""Sanitizes raw cookie objects ensuring full Playwright compatibility."""
|
|
13
|
+
if not isinstance(raw_cookies, list):
|
|
14
|
+
raise ValidationError(f"Expected list of cookies, got {type(raw_cookies).__name__}")
|
|
15
|
+
|
|
16
|
+
sanitized: List[Dict[str, Any]] = []
|
|
17
|
+
for c in raw_cookies:
|
|
18
|
+
if not isinstance(c, dict):
|
|
19
|
+
continue
|
|
20
|
+
cookie = dict(c)
|
|
21
|
+
same_site = str(cookie.get("sameSite", "")).capitalize()
|
|
22
|
+
if same_site in VALID_SAME_SITE_VALUES:
|
|
23
|
+
cookie["sameSite"] = same_site
|
|
24
|
+
else:
|
|
25
|
+
cookie.pop("sameSite", None)
|
|
26
|
+
|
|
27
|
+
for key in INVALID_COOKIE_KEYS:
|
|
28
|
+
cookie.pop(key, None)
|
|
29
|
+
|
|
30
|
+
sanitized.append(cookie)
|
|
31
|
+
|
|
32
|
+
return sanitized
|
|
33
|
+
|
|
34
|
+
def load_and_sanitize_cookies(filepath: str) -> List[Dict[str, Any]]:
|
|
35
|
+
"""Loads a JSON cookie file from disk and returns sanitized cookie dictionaries."""
|
|
36
|
+
if not os.path.isfile(filepath):
|
|
37
|
+
raise FileNotFoundError(f"Cookie file not found: {filepath}")
|
|
38
|
+
|
|
39
|
+
with open(filepath, "r", encoding="utf-8") as f:
|
|
40
|
+
raw_cookies = json.load(f)
|
|
41
|
+
|
|
42
|
+
return sanitize_cookies(raw_cookies)
|
|
43
|
+
|
|
44
|
+
def save_cookies(filepath: str, cookies: List[Dict[str, Any]]) -> None:
|
|
45
|
+
"""Persists cookie objects to disk formatted cleanly."""
|
|
46
|
+
os.makedirs(os.path.dirname(os.path.abspath(filepath)), exist_ok=True)
|
|
47
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
48
|
+
json.dump(cookies, f, indent=2, ensure_ascii=False)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Structured logging and sensitive data redaction utilities."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any, Dict, List, Set, Union
|
|
5
|
+
|
|
6
|
+
SENSITIVE_KEY_PATTERNS: Set[str] = {
|
|
7
|
+
"password",
|
|
8
|
+
"secret",
|
|
9
|
+
"token",
|
|
10
|
+
"api_key",
|
|
11
|
+
"apikey",
|
|
12
|
+
"national_id",
|
|
13
|
+
"iqama",
|
|
14
|
+
"ssn",
|
|
15
|
+
"credentials",
|
|
16
|
+
"auth",
|
|
17
|
+
"cookie",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def redact_sensitive(data: Any) -> Any:
|
|
22
|
+
"""Recursively redacts sensitive keys and values from dictionaries, lists, or strings."""
|
|
23
|
+
if isinstance(data, dict):
|
|
24
|
+
redacted = {}
|
|
25
|
+
for k, v in data.items():
|
|
26
|
+
k_lower = str(k).lower().strip()
|
|
27
|
+
if any(p in k_lower for p in SENSITIVE_KEY_PATTERNS):
|
|
28
|
+
redacted[k] = "[REDACTED]"
|
|
29
|
+
else:
|
|
30
|
+
redacted[k] = redact_sensitive(v)
|
|
31
|
+
return redacted
|
|
32
|
+
|
|
33
|
+
if isinstance(data, list):
|
|
34
|
+
return [redact_sensitive(item) for item in data]
|
|
35
|
+
|
|
36
|
+
if isinstance(data, tuple):
|
|
37
|
+
return tuple(redact_sensitive(item) for item in data)
|
|
38
|
+
|
|
39
|
+
if isinstance(data, str):
|
|
40
|
+
# Redact patterns resembling 9-10 digit national IDs (excluding all-zeros placeholders)
|
|
41
|
+
masked = re.sub(r"\b(?!0{9,10}\b)[1-9]\d{8,9}\b", "[REDACTED_ID]", data)
|
|
42
|
+
# Redact email addresses
|
|
43
|
+
masked = re.sub(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", "[REDACTED_EMAIL]", masked)
|
|
44
|
+
return masked
|
|
45
|
+
|
|
46
|
+
return data
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def log_event(level: str, operation: str, context: Dict[str, Any]) -> None:
|
|
50
|
+
"""Prints a structured log line with sensitive fields automatically redacted."""
|
|
51
|
+
safe_ctx = redact_sensitive(context)
|
|
52
|
+
print(f"[{level.upper()}] {operation} | {safe_ctx}")
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Cross-Platform Process Management and Isolation Layer.
|
|
2
|
+
|
|
3
|
+
Isolates OS-specific process lifecycle mechanisms (WMI, CIM, taskkill on Windows;
|
|
4
|
+
killpg, pgrep, signals on POSIX/Linux/macOS) behind a clean, unified ProcessManager interface.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import os
|
|
11
|
+
import signal
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
from abc import ABC, abstractmethod
|
|
15
|
+
from typing import Any, Dict, List, Optional, Union
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BaseProcessManager(ABC):
|
|
19
|
+
"""Abstract interface for system process management and detached lifecycle execution."""
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def launch_detached(
|
|
23
|
+
self,
|
|
24
|
+
cmd: Union[str, List[str]],
|
|
25
|
+
cwd: Optional[str] = None,
|
|
26
|
+
env: Optional[Dict[str, str]] = None,
|
|
27
|
+
) -> Optional[int]:
|
|
28
|
+
"""Launches a detached process decoupled from the caller's terminal and process group."""
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
@abstractmethod
|
|
32
|
+
def kill_process_tree(self, pid: int, force: bool = True) -> bool:
|
|
33
|
+
"""Kills the target process and all descendant child processes (cascading teardown)."""
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def get_child_pids(self, parent_pid: int) -> List[int]:
|
|
38
|
+
"""Returns direct child process IDs for the given parent PID."""
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
def is_process_running(self, pid: int) -> bool:
|
|
43
|
+
"""Checks if a process with the specified PID is currently active."""
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class WindowsProcessManager(BaseProcessManager):
|
|
48
|
+
"""Windows-specific implementation using PowerShell WMI/CIM and taskkill."""
|
|
49
|
+
|
|
50
|
+
def launch_detached(
|
|
51
|
+
self,
|
|
52
|
+
cmd: Union[str, List[str]],
|
|
53
|
+
cwd: Optional[str] = None,
|
|
54
|
+
env: Optional[Dict[str, str]] = None,
|
|
55
|
+
) -> Optional[int]:
|
|
56
|
+
target_cwd = cwd or os.getcwd()
|
|
57
|
+
if isinstance(cmd, list):
|
|
58
|
+
cmd_str = " ".join(f'"{arg}"' if " " in arg and not arg.startswith('"') else arg for arg in cmd)
|
|
59
|
+
else:
|
|
60
|
+
cmd_str = cmd
|
|
61
|
+
|
|
62
|
+
# Primary approach: Win32_Process.Create via WMI to break out of terminal Job Objects
|
|
63
|
+
ps_script = f"""
|
|
64
|
+
$cwd = '{target_cwd}'
|
|
65
|
+
$cmd = @'
|
|
66
|
+
{cmd_str}
|
|
67
|
+
'@
|
|
68
|
+
$startup = ([wmiclass]'Win32_ProcessStartup').CreateInstance()
|
|
69
|
+
$startup.ShowWindow = 0
|
|
70
|
+
$proc = ([wmiclass]'Win32_Process').Create($cmd, $cwd, $startup)
|
|
71
|
+
if ($proc.ReturnValue -eq 0) {{
|
|
72
|
+
Write-Output $proc.ProcessId
|
|
73
|
+
}} else {{
|
|
74
|
+
exit 1
|
|
75
|
+
}}
|
|
76
|
+
"""
|
|
77
|
+
try:
|
|
78
|
+
encoded = base64.b64encode(ps_script.encode("utf-16le")).decode("ascii")
|
|
79
|
+
res = subprocess.run(
|
|
80
|
+
["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
|
|
81
|
+
capture_output=True,
|
|
82
|
+
text=True,
|
|
83
|
+
timeout=10,
|
|
84
|
+
)
|
|
85
|
+
out = res.stdout.strip()
|
|
86
|
+
if res.returncode == 0 and out.isdigit():
|
|
87
|
+
return int(out)
|
|
88
|
+
except Exception:
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
# Fallback approach: subprocess.Popen with DETACHED_PROCESS
|
|
92
|
+
try:
|
|
93
|
+
cmd_args = cmd if isinstance(cmd, list) else cmd_str
|
|
94
|
+
creation_flags = (
|
|
95
|
+
getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
|
|
96
|
+
| getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
|
|
97
|
+
)
|
|
98
|
+
proc = subprocess.Popen(
|
|
99
|
+
cmd_args,
|
|
100
|
+
cwd=target_cwd,
|
|
101
|
+
env=env,
|
|
102
|
+
stdout=subprocess.DEVNULL,
|
|
103
|
+
stderr=subprocess.DEVNULL,
|
|
104
|
+
stdin=subprocess.DEVNULL,
|
|
105
|
+
creationflags=creation_flags,
|
|
106
|
+
)
|
|
107
|
+
return proc.pid
|
|
108
|
+
except Exception:
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
def kill_process_tree(self, pid: int, force: bool = True) -> bool:
|
|
112
|
+
if not pid or pid <= 0:
|
|
113
|
+
return False
|
|
114
|
+
args = ["taskkill"]
|
|
115
|
+
if force:
|
|
116
|
+
args.append("/F")
|
|
117
|
+
args.extend(["/T", "/PID", str(pid)])
|
|
118
|
+
try:
|
|
119
|
+
res = subprocess.run(
|
|
120
|
+
args,
|
|
121
|
+
stdout=subprocess.DEVNULL,
|
|
122
|
+
stderr=subprocess.DEVNULL,
|
|
123
|
+
timeout=10,
|
|
124
|
+
)
|
|
125
|
+
return res.returncode == 0
|
|
126
|
+
except Exception:
|
|
127
|
+
return False
|
|
128
|
+
|
|
129
|
+
def get_child_pids(self, parent_pid: int) -> List[int]:
|
|
130
|
+
if not parent_pid or parent_pid <= 0:
|
|
131
|
+
return []
|
|
132
|
+
try:
|
|
133
|
+
ps_c = f"(Get-CimInstance Win32_Process | Where-Object {{ $_.ParentProcessId -eq {parent_pid} }}).ProcessId"
|
|
134
|
+
out = subprocess.check_output(
|
|
135
|
+
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps_c],
|
|
136
|
+
text=True,
|
|
137
|
+
stderr=subprocess.DEVNULL,
|
|
138
|
+
timeout=8,
|
|
139
|
+
).strip()
|
|
140
|
+
pids = []
|
|
141
|
+
for line in out.splitlines():
|
|
142
|
+
line = line.strip()
|
|
143
|
+
if line.isdigit():
|
|
144
|
+
pids.append(int(line))
|
|
145
|
+
return pids
|
|
146
|
+
except Exception:
|
|
147
|
+
return []
|
|
148
|
+
|
|
149
|
+
def is_process_running(self, pid: int) -> bool:
|
|
150
|
+
if not pid or pid <= 0:
|
|
151
|
+
return False
|
|
152
|
+
try:
|
|
153
|
+
ps_c = f"(Get-Process -Id {pid} -ErrorAction SilentlyContinue) -ne $null"
|
|
154
|
+
out = subprocess.check_output(
|
|
155
|
+
["powershell", "-NoProfile", "-NonInteractive", "-Command", ps_c],
|
|
156
|
+
text=True,
|
|
157
|
+
stderr=subprocess.DEVNULL,
|
|
158
|
+
timeout=5,
|
|
159
|
+
).strip()
|
|
160
|
+
return out.lower() == "true"
|
|
161
|
+
except Exception:
|
|
162
|
+
return False
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class PosixProcessManager(BaseProcessManager):
|
|
166
|
+
"""POSIX/Linux/macOS implementation using setsid, signal.SIGKILL, and pgrep."""
|
|
167
|
+
|
|
168
|
+
def launch_detached(
|
|
169
|
+
self,
|
|
170
|
+
cmd: Union[str, List[str]],
|
|
171
|
+
cwd: Optional[str] = None,
|
|
172
|
+
env: Optional[Dict[str, str]] = None,
|
|
173
|
+
) -> Optional[int]:
|
|
174
|
+
target_cwd = cwd or os.getcwd()
|
|
175
|
+
try:
|
|
176
|
+
cmd_args = cmd if isinstance(cmd, list) else cmd.split()
|
|
177
|
+
proc = subprocess.Popen(
|
|
178
|
+
cmd_args,
|
|
179
|
+
cwd=target_cwd,
|
|
180
|
+
env=env,
|
|
181
|
+
stdout=subprocess.DEVNULL,
|
|
182
|
+
stderr=subprocess.DEVNULL,
|
|
183
|
+
stdin=subprocess.DEVNULL,
|
|
184
|
+
start_new_session=True, # setsid decoupling
|
|
185
|
+
)
|
|
186
|
+
return proc.pid
|
|
187
|
+
except Exception:
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
def kill_process_tree(self, pid: int, force: bool = True) -> bool:
|
|
191
|
+
if not pid or pid <= 0:
|
|
192
|
+
return False
|
|
193
|
+
sig = signal.SIGKILL if force else signal.SIGTERM
|
|
194
|
+
killed = False
|
|
195
|
+
# Strategy 1: Process group kill if leader
|
|
196
|
+
try:
|
|
197
|
+
pgid = os.getpgid(pid)
|
|
198
|
+
os.killpg(pgid, sig)
|
|
199
|
+
killed = True
|
|
200
|
+
except Exception:
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
# Strategy 2: pkill children then kill target
|
|
204
|
+
try:
|
|
205
|
+
subprocess.run(["pkill", "-P", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
206
|
+
os.kill(pid, sig)
|
|
207
|
+
killed = True
|
|
208
|
+
except Exception:
|
|
209
|
+
pass
|
|
210
|
+
|
|
211
|
+
return killed
|
|
212
|
+
|
|
213
|
+
def get_child_pids(self, parent_pid: int) -> List[int]:
|
|
214
|
+
if not parent_pid or parent_pid <= 0:
|
|
215
|
+
return []
|
|
216
|
+
try:
|
|
217
|
+
out = subprocess.check_output(
|
|
218
|
+
["pgrep", "-P", str(parent_pid)],
|
|
219
|
+
text=True,
|
|
220
|
+
stderr=subprocess.DEVNULL,
|
|
221
|
+
timeout=5,
|
|
222
|
+
).strip()
|
|
223
|
+
pids = []
|
|
224
|
+
for line in out.splitlines():
|
|
225
|
+
line = line.strip()
|
|
226
|
+
if line.isdigit():
|
|
227
|
+
pids.append(int(line))
|
|
228
|
+
return pids
|
|
229
|
+
except Exception:
|
|
230
|
+
return []
|
|
231
|
+
|
|
232
|
+
def is_process_running(self, pid: int) -> bool:
|
|
233
|
+
if not pid or pid <= 0:
|
|
234
|
+
return False
|
|
235
|
+
try:
|
|
236
|
+
os.kill(pid, 0)
|
|
237
|
+
return True
|
|
238
|
+
except OSError:
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
_GLOBAL_PROCESS_MANAGER: Optional[BaseProcessManager] = None
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def get_process_manager() -> BaseProcessManager:
|
|
246
|
+
"""Returns the platform-appropriate ProcessManager singleton."""
|
|
247
|
+
global _GLOBAL_PROCESS_MANAGER
|
|
248
|
+
if _GLOBAL_PROCESS_MANAGER is None:
|
|
249
|
+
if sys.platform == "win32":
|
|
250
|
+
_GLOBAL_PROCESS_MANAGER = WindowsProcessManager()
|
|
251
|
+
else:
|
|
252
|
+
_GLOBAL_PROCESS_MANAGER = PosixProcessManager()
|
|
253
|
+
return _GLOBAL_PROCESS_MANAGER
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def set_process_manager(manager: Optional[BaseProcessManager]) -> None:
|
|
257
|
+
"""Allows injecting custom or mock ProcessManager implementations for testing."""
|
|
258
|
+
global _GLOBAL_PROCESS_MANAGER
|
|
259
|
+
_GLOBAL_PROCESS_MANAGER = manager
|