pipulate 2.58__tar.gz → 2.60__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pipulate-2.58/pipulate.egg-info → pipulate-2.60}/PKG-INFO +1 -1
- {pipulate-2.58 → pipulate-2.60}/__init__.py +2 -2
- {pipulate-2.58 → pipulate-2.60}/imports/voice_synthesis.py +62 -12
- {pipulate-2.58 → pipulate-2.60/pipulate.egg-info}/PKG-INFO +1 -1
- {pipulate-2.58 → pipulate-2.60}/pyproject.toml +1 -1
- {pipulate-2.58 → pipulate-2.60}/tools/scraper_tools.py +93 -41
- {pipulate-2.58 → pipulate-2.60}/LICENSE +0 -0
- {pipulate-2.58 → pipulate-2.60}/MANIFEST.in +0 -0
- {pipulate-2.58 → pipulate-2.60}/README.md +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/010_introduction.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/015_config.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/020_profiles.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/025_aspect.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/030_roles.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/040_hello_workflow.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/050_documentation.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/060_tasks.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/070_history.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/110_parameter_buster.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/120_link_graph.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/130_gap_analysis.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/200_workflow_genesis.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/210_widget_examples.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/220_roadmap.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/230_dev_assistant.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/240_simon_mcp.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/300_blank_placeholder.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/400_botify_trifecta.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/440_browser_automation.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/450_stream_simulator.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/510_text_field.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/520_text_area.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/530_dropdown.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/540_checkboxes.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/550_radios.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/560_range.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/570_switch.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/580_upload.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/610_markdown.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/620_mermaid.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/630_prism.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/640_javascript.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/710_pandas.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/720_rich.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/730_matplotlib.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/810_webbrowser.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/820_selenium.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/apps/830_pico_slider.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/cli.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/config.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/__init__.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/ai_dictdb.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/ai_tool_discovery_simple_parser.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/append_only_conversation.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/ascii_displays.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/botify/__init__.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/botify/code_generators.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/botify/true_schema_discoverer.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/botify_code_generation.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/crud.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/database_safety_wrapper.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/dom_processing/__init__.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/dom_processing/ai_dom_beautifier.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/dom_processing/enhanced_dom_processor.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/durable_backup_system.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/mcp_orchestrator.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/server_logging.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/imports/stream_orchestrator.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate/__init__.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate/core.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate.egg-info/SOURCES.txt +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate.egg-info/dependency_links.txt +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate.egg-info/entry_points.txt +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate.egg-info/requires.txt +0 -0
- {pipulate-2.58 → pipulate-2.60}/pipulate.egg-info/top_level.txt +0 -0
- {pipulate-2.58 → pipulate-2.60}/server.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/setup.cfg +0 -0
- {pipulate-2.58 → pipulate-2.60}/tests/test_apply_airlock.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tests/test_mck_rep2.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/__init__.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/advanced_automation_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/botify_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/connector_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/conversation_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/dom_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/keychain_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/llm_optics.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/mcp_tools.py +0 -0
- {pipulate-2.58 → pipulate-2.60}/tools/system_tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pipulate
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.60
|
|
4
4
|
Summary: The context compiler that helps you prompt well — local-first, Nix-reproducible, replayable. The successor to AI SEO software.
|
|
5
5
|
Author-email: Mike Levin <pipulate@gmail.com>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -12,7 +12,7 @@ Usage:
|
|
|
12
12
|
pipulate
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
-
__version__ = "2.
|
|
15
|
+
__version__ = "2.60"
|
|
16
16
|
# APOSTROPHES RESTORED (2026-08-04). They were stripped as a workaround for
|
|
17
17
|
# flake.nix's descMatch regex, whose character class excluded ' from the
|
|
18
18
|
# CAPTURE and truncated the banner to "(So)". That regex was fixed in the same
|
|
@@ -23,7 +23,7 @@ __version__ = "2.58"
|
|
|
23
23
|
# is not a property of the system; the regex is. Blast radius is one banner:
|
|
24
24
|
# nothing but flake.nix reads this name -- version_sync.py syncs __version__
|
|
25
25
|
# and __description__, never this. So'wI' chu' -- "engage the cloaking device."
|
|
26
|
-
__version_description__ = "
|
|
26
|
+
__version_description__ = "Putting the v in NPvg"
|
|
27
27
|
# SPDX expression, single source of truth, synced into pyproject.toml by
|
|
28
28
|
# scripts/release/version_sync.py. "-or-later" (not bare AGPL-3.0, which is
|
|
29
29
|
# deprecated SPDX) because the header below grants "any later version".
|
|
@@ -15,7 +15,9 @@ import signal
|
|
|
15
15
|
import sys
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
from typing import Optional, Dict, Any
|
|
18
|
+
import contextlib
|
|
18
19
|
import logging
|
|
20
|
+
import warnings
|
|
19
21
|
|
|
20
22
|
# Try to import voice synthesis dependencies
|
|
21
23
|
try:
|
|
@@ -35,6 +37,42 @@ except ImportError:
|
|
|
35
37
|
|
|
36
38
|
logger = logging.getLogger(__name__)
|
|
37
39
|
|
|
40
|
+
|
|
41
|
+
@contextlib.contextmanager
|
|
42
|
+
def _quiet_hub():
|
|
43
|
+
"""Silence the hub for the length of the one download it ever makes.
|
|
44
|
+
|
|
45
|
+
THE QUIET WALK (2026-09-18, Mac receipt, deed 1480): the first consent
|
|
46
|
+
printed a three-line deprecation notice, a token nag and three progress
|
|
47
|
+
bars between the card and the first spoken step. The hub speaks through
|
|
48
|
+
two channels, its logger and the warnings module, and draws its bars
|
|
49
|
+
through its own switch; all three are turned down here and restored on
|
|
50
|
+
the way out, so nothing outside this block changes. The download itself
|
|
51
|
+
is announced by one plain line in the caller.
|
|
52
|
+
"""
|
|
53
|
+
try:
|
|
54
|
+
from huggingface_hub.utils import (
|
|
55
|
+
are_progress_bars_disabled,
|
|
56
|
+
disable_progress_bars,
|
|
57
|
+
enable_progress_bars,
|
|
58
|
+
)
|
|
59
|
+
except ImportError:
|
|
60
|
+
are_progress_bars_disabled = disable_progress_bars = enable_progress_bars = None
|
|
61
|
+
hub_logger = logging.getLogger("huggingface_hub")
|
|
62
|
+
previous_level = hub_logger.level
|
|
63
|
+
hub_logger.setLevel(logging.ERROR)
|
|
64
|
+
bars_were_disabled = True if disable_progress_bars is None else are_progress_bars_disabled()
|
|
65
|
+
if not bars_were_disabled:
|
|
66
|
+
disable_progress_bars()
|
|
67
|
+
try:
|
|
68
|
+
with warnings.catch_warnings():
|
|
69
|
+
warnings.simplefilter("ignore")
|
|
70
|
+
yield
|
|
71
|
+
finally:
|
|
72
|
+
hub_logger.setLevel(previous_level)
|
|
73
|
+
if not bars_were_disabled:
|
|
74
|
+
enable_progress_bars()
|
|
75
|
+
|
|
38
76
|
# ---------------------------------------------------------------------------
|
|
39
77
|
# THE VOICE ASKS BEFORE IT SPEAKS (2026-09-17). Two functions, one file.
|
|
40
78
|
# voice_consent() is the BARRIER: every path to sound reads it, fresh, on every
|
|
@@ -157,29 +195,35 @@ class ChipVoiceSystem:
|
|
|
157
195
|
self.model_path = str(local_model_dir / model_path_in_repo)
|
|
158
196
|
self.config_path = str(local_model_dir / config_path_in_repo)
|
|
159
197
|
|
|
198
|
+
downloaded = False
|
|
160
199
|
# Download files if they don't exist
|
|
161
200
|
if not os.path.exists(self.model_path):
|
|
201
|
+
print("Downloading the voice, about 60 MB, once...", flush=True)
|
|
162
202
|
logger.info(f"🎤 Downloading voice model: {model_path_in_repo}...")
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
203
|
+
with _quiet_hub():
|
|
204
|
+
hf_hub_download(
|
|
205
|
+
repo_id=repo_id,
|
|
206
|
+
filename=model_path_in_repo,
|
|
207
|
+
local_dir=local_model_dir,
|
|
208
|
+
)
|
|
209
|
+
downloaded = True
|
|
169
210
|
|
|
170
211
|
if not os.path.exists(self.config_path):
|
|
171
212
|
logger.info(f"🎤 Downloading voice config: {config_path_in_repo}...")
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
213
|
+
with _quiet_hub():
|
|
214
|
+
hf_hub_download(
|
|
215
|
+
repo_id=repo_id,
|
|
216
|
+
filename=config_path_in_repo,
|
|
217
|
+
local_dir=local_model_dir,
|
|
218
|
+
)
|
|
219
|
+
downloaded = True
|
|
178
220
|
|
|
179
221
|
# Load the voice model
|
|
180
222
|
self.voice = PiperVoice.load(self.model_path, config_path=self.config_path)
|
|
181
223
|
self.voice_ready = True
|
|
182
224
|
logger.info("🎤 Voice model loaded successfully")
|
|
225
|
+
if downloaded:
|
|
226
|
+
print("Voice ready.", flush=True)
|
|
183
227
|
|
|
184
228
|
except Exception as e:
|
|
185
229
|
logger.error(f"🎤 Failed to setup voice model: {e}")
|
|
@@ -197,6 +241,12 @@ class ChipVoiceSystem:
|
|
|
197
241
|
if not VOICE_SYNTHESIS_AVAILABLE:
|
|
198
242
|
self.last_error = f"voice synthesis not installed: {IMPORT_ERROR}"
|
|
199
243
|
return False
|
|
244
|
+
# THE PHONEME LINE (TODO of 2026-09-15, discharged 2026-09-18): piper
|
|
245
|
+
# warns, through a logger nobody configured, about one syllabic mark
|
|
246
|
+
# in the word "written", and Python's last-resort handler prints the
|
|
247
|
+
# bare message on every walk. The flake's door-1 greeting already sets
|
|
248
|
+
# this level; the speaker now sets it for every caller.
|
|
249
|
+
logging.getLogger("piper").setLevel(logging.ERROR)
|
|
200
250
|
self.setup_voice_model()
|
|
201
251
|
return self.voice_ready
|
|
202
252
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pipulate
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.60
|
|
4
4
|
Summary: The context compiler that helps you prompt well — local-first, Nix-reproducible, replayable. The successor to AI SEO software.
|
|
5
5
|
Author-email: Mike Levin <pipulate@gmail.com>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pipulate"
|
|
7
7
|
# VERSION NOTE: Synced from pipulate/__init__.py.__version__ via python version_sync.py
|
|
8
|
-
version = "2.
|
|
8
|
+
version = "2.60"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="Mike Levin", email="pipulate@gmail.com" },
|
|
11
11
|
]
|
|
@@ -3,6 +3,7 @@ import asyncio
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
|
+
import shlex
|
|
6
7
|
import sys
|
|
7
8
|
import shutil
|
|
8
9
|
import tempfile
|
|
@@ -21,6 +22,18 @@ from selenium.webdriver.common.by import By
|
|
|
21
22
|
from tools import auto_tool
|
|
22
23
|
from . import dom_tools
|
|
23
24
|
|
|
25
|
+
# THE HEAL HAPPENS ONCE PER RUN (2026-09-18, Mac receipt, deed 1480). The
|
|
26
|
+
# flake pins no Chrome on a Mac, so the driver is matched to the host's
|
|
27
|
+
# Chrome at launch: the first uc.Chrome raises on the version mismatch, the
|
|
28
|
+
# window it had opened closes, and the retry below relaunches with
|
|
29
|
+
# version_main set to the browser's major version. That version was thrown
|
|
30
|
+
# away after every stop, so a three-stop walk paid the close-and-reopen at
|
|
31
|
+
# every stop and printed a WARNING three times for the ordinary case. It is
|
|
32
|
+
# remembered here for the life of the process: the next launch passes it in
|
|
33
|
+
# and never mismatches. A dict, not a bare name, so no function needs a
|
|
34
|
+
# global statement to write it.
|
|
35
|
+
_HEAL = {"version_main": None}
|
|
36
|
+
|
|
24
37
|
async def generate_optics_subprocess(target_dir_path: str):
|
|
25
38
|
"""Isolated wrapper to call llm_optics.py as a subprocess, protecting the event loop."""
|
|
26
39
|
script_path = Path(__file__).resolve().parent / "llm_optics.py"
|
|
@@ -59,35 +72,38 @@ def _guided_path_component(url: str) -> tuple[str, str]:
|
|
|
59
72
|
|
|
60
73
|
|
|
61
74
|
# --- Optional audio during browser startup ---
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
# never touch any other aplay pipeline. The wav lives in repo negative space
|
|
66
|
-
# (gitignored) or ~/.local/share/pipulate/; absent file = silent no-op.
|
|
75
|
+
# Operator-owned courtesy sounds live outside the repo. tick.wav loops while
|
|
76
|
+
# the browser is opening; ding.wav plays once when the page-load phase ends.
|
|
77
|
+
# No file means no sound. Linux uses aplay; Darwin uses afplay.
|
|
67
78
|
SCRAPE_MUSIC_MARKER = "pipulate-scrape-music"
|
|
79
|
+
SOUND_DIR = Path.home() / ".local/share/pipulate"
|
|
68
80
|
|
|
69
81
|
|
|
70
|
-
def
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
82
|
+
def _sound_file(name):
|
|
83
|
+
path = SOUND_DIR / name
|
|
84
|
+
return path if path.is_file() else None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _player_args(sound):
|
|
88
|
+
if sound is None:
|
|
89
|
+
return None
|
|
90
|
+
if sys.platform == "darwin":
|
|
91
|
+
player = shutil.which("afplay")
|
|
92
|
+
return [player, str(sound)] if player else None
|
|
93
|
+
player = shutil.which("aplay")
|
|
94
|
+
return [player, "-q", "-D", "default", str(sound)] if player else None
|
|
80
95
|
|
|
81
96
|
|
|
82
97
|
def _start_scrape_music(verbose=True):
|
|
83
98
|
import subprocess
|
|
84
|
-
music = _find_music_file()
|
|
85
|
-
if not music:
|
|
86
|
-
return None
|
|
87
99
|
|
|
100
|
+
args = _player_args(_sound_file("tick.wav"))
|
|
101
|
+
if not args:
|
|
102
|
+
return None
|
|
103
|
+
command = " ".join(shlex.quote(part) for part in args)
|
|
88
104
|
try:
|
|
89
105
|
return subprocess.Popen(
|
|
90
|
-
["sh", "-c", f
|
|
106
|
+
["sh", "-c", f"while :; do {command}; done # {SCRAPE_MUSIC_MARKER}"],
|
|
91
107
|
stdout=subprocess.DEVNULL,
|
|
92
108
|
stderr=subprocess.DEVNULL,
|
|
93
109
|
preexec_fn=os.setsid,
|
|
@@ -96,18 +112,36 @@ def _start_scrape_music(verbose=True):
|
|
|
96
112
|
return None
|
|
97
113
|
|
|
98
114
|
|
|
99
|
-
def _stop_scrape_music(proc):
|
|
115
|
+
def _stop_scrape_music(proc, ding=False):
|
|
100
116
|
import signal
|
|
101
117
|
import subprocess
|
|
118
|
+
|
|
102
119
|
if proc is not None:
|
|
103
120
|
try:
|
|
104
121
|
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
|
105
122
|
except Exception:
|
|
106
123
|
pass
|
|
107
124
|
try:
|
|
108
|
-
subprocess.run(
|
|
125
|
+
subprocess.run(
|
|
126
|
+
["pkill", "-f", SCRAPE_MUSIC_MARKER],
|
|
127
|
+
check=False,
|
|
128
|
+
stdout=subprocess.DEVNULL,
|
|
129
|
+
stderr=subprocess.DEVNULL,
|
|
130
|
+
)
|
|
109
131
|
except Exception:
|
|
110
132
|
pass
|
|
133
|
+
if ding:
|
|
134
|
+
args = _player_args(_sound_file("ding.wav"))
|
|
135
|
+
if args:
|
|
136
|
+
try:
|
|
137
|
+
subprocess.run(
|
|
138
|
+
args,
|
|
139
|
+
check=False,
|
|
140
|
+
stdout=subprocess.DEVNULL,
|
|
141
|
+
stderr=subprocess.DEVNULL,
|
|
142
|
+
)
|
|
143
|
+
except Exception:
|
|
144
|
+
pass
|
|
111
145
|
|
|
112
146
|
|
|
113
147
|
def _simplify_html_for_llm(html_content, default_title=""):
|
|
@@ -223,7 +257,7 @@ def _document_candidates(cdp_events: list, domain: str, final_url: str = "") ->
|
|
|
223
257
|
return on_host
|
|
224
258
|
|
|
225
259
|
|
|
226
|
-
def _capture_checkpoint(stdin=None, stdout=None) -> dict:
|
|
260
|
+
def _capture_checkpoint(stdin=None, stdout=None, before_prompt=None) -> dict:
|
|
227
261
|
"""Require an explicit CAPTURE token from the human's keyboard.
|
|
228
262
|
|
|
229
263
|
/DEV/TTY PREFERENCE (convicted 2026-07-29, public_walk stop 1): the
|
|
@@ -265,8 +299,16 @@ def _capture_checkpoint(stdin=None, stdout=None) -> dict:
|
|
|
265
299
|
}
|
|
266
300
|
|
|
267
301
|
try:
|
|
302
|
+
if before_prompt is not None:
|
|
303
|
+
try:
|
|
304
|
+
before_prompt()
|
|
305
|
+
except Exception as exc:
|
|
306
|
+
output_stream.write(f"\n(checkpoint narration unavailable: {exc})\n")
|
|
307
|
+
else:
|
|
308
|
+
output_stream.write(
|
|
309
|
+
"\nWhen the page you want is ready, type CAPTURE and press Enter.\n"
|
|
310
|
+
)
|
|
268
311
|
output_stream.write(
|
|
269
|
-
"\nWhen the page you want is ready, type CAPTURE and press Enter.\n"
|
|
270
312
|
"Any other response aborts without capturing artifacts.\n"
|
|
271
313
|
"CAPTURE> "
|
|
272
314
|
)
|
|
@@ -560,7 +602,8 @@ async def _selenium_capture(params: dict, checkpoint=None) -> dict:
|
|
|
560
602
|
driver = uc.Chrome(options=options,
|
|
561
603
|
user_data_dir=str(profile_path),
|
|
562
604
|
browser_executable_path=browser_path,
|
|
563
|
-
driver_executable_path=driver_path
|
|
605
|
+
driver_executable_path=driver_path,
|
|
606
|
+
version_main=_HEAL["version_main"])
|
|
564
607
|
except Exception as e:
|
|
565
608
|
error_msg = str(e)
|
|
566
609
|
if "Current browser version is" in error_msg:
|
|
@@ -568,7 +611,10 @@ async def _selenium_capture(params: dict, checkpoint=None) -> dict:
|
|
|
568
611
|
match = re.search(r'Current browser version is (\d+)', error_msg)
|
|
569
612
|
if match:
|
|
570
613
|
fallback_version = int(match.group(1))
|
|
571
|
-
|
|
614
|
+
_HEAL["version_main"] = fallback_version
|
|
615
|
+
logger.info(f"Chrome {fallback_version}: driver matched; relaunching once for this run")
|
|
616
|
+
if verbose:
|
|
617
|
+
print(f"Chrome closed and is reopening once while its driver is matched to Chrome {fallback_version}; that is normal.", flush=True)
|
|
572
618
|
# UC consumes the options object. We must forge a fresh one.
|
|
573
619
|
fresh_options = uc.ChromeOptions()
|
|
574
620
|
fresh_options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
|
|
@@ -592,20 +638,25 @@ async def _selenium_capture(params: dict, checkpoint=None) -> dict:
|
|
|
592
638
|
logger.info(f"Navigating to: {url}")
|
|
593
639
|
driver.get(url)
|
|
594
640
|
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
641
|
+
# A human-guided ride already has the CAPTURE fence as its settle
|
|
642
|
+
# detector. The 20-second staleness wait exists for unattended sigil
|
|
643
|
+
# scrapes, where a security interstitial may reload underneath us.
|
|
644
|
+
if not interactive:
|
|
645
|
+
try:
|
|
646
|
+
if verbose: logger.info("Waiting for security challenge to trigger a reload (Stage 1)...")
|
|
647
|
+
initial_body = driver.find_element(By.TAG_NAME, 'body')
|
|
648
|
+
WebDriverWait(driver, 20).until(EC.staleness_of(initial_body))
|
|
649
|
+
if verbose: logger.success("✅ Page reload detected!")
|
|
650
|
+
|
|
651
|
+
if verbose: logger.info("Waiting for main content to appear after reload (Stage 2)...")
|
|
652
|
+
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "body")))
|
|
653
|
+
if verbose: logger.success("✅ Main content located!")
|
|
654
|
+
except Exception:
|
|
655
|
+
if verbose: logger.info("Did not detect a page reload for security challenge. Proceeding anyway.")
|
|
606
656
|
|
|
607
|
-
# The
|
|
608
|
-
|
|
657
|
+
# The page-load phase is over: stop ticking and, if the operator
|
|
658
|
+
# supplied one, play the ready sound exactly once.
|
|
659
|
+
_stop_scrape_music(music_proc, ding=True)
|
|
609
660
|
music_proc = None
|
|
610
661
|
|
|
611
662
|
if checkpoint is not None:
|
|
@@ -737,8 +788,8 @@ async def _selenium_capture(params: dict, checkpoint=None) -> dict:
|
|
|
737
788
|
# ledger-only so it can be replayed offline over any cached capture.
|
|
738
789
|
#
|
|
739
790
|
# [-1] SURVIVES, and its rationale is still UNWITNESSED: the
|
|
740
|
-
# staleness_of() wait above exists because a security
|
|
741
|
-
# serves an interstitial Document and then reloads, making the LAST
|
|
791
|
+
# non-interactive staleness_of() wait above exists because a security
|
|
792
|
+
# challenge serves an interstitial Document and then reloads, making the LAST
|
|
742
793
|
# candidate the real page. No capture on this machine has ever been
|
|
743
794
|
# challenged -- every scrape logs "Did not detect a page reload."
|
|
744
795
|
# The reasoning is sound from the code and has no receipt behind it.
|
|
@@ -956,7 +1007,7 @@ async def selenium_automation(params: dict) -> dict:
|
|
|
956
1007
|
return await _selenium_capture(params)
|
|
957
1008
|
|
|
958
1009
|
|
|
959
|
-
async def guided_browser_capture(params: dict, stdin=None, stdout=None) -> dict:
|
|
1010
|
+
async def guided_browser_capture(params: dict, stdin=None, stdout=None, before_prompt=None) -> dict:
|
|
960
1011
|
"""Capture one human-guided page through one visible persistent driver."""
|
|
961
1012
|
if not isinstance(params, dict):
|
|
962
1013
|
return {
|
|
@@ -1029,5 +1080,6 @@ async def guided_browser_capture(params: dict, stdin=None, stdout=None) -> dict:
|
|
|
1029
1080
|
checkpoint=lambda: _capture_checkpoint(
|
|
1030
1081
|
stdin=input_stream,
|
|
1031
1082
|
stdout=stdout,
|
|
1083
|
+
before_prompt=before_prompt,
|
|
1032
1084
|
),
|
|
1033
1085
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|