pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/__init__.py +357 -11
- biolib/_data_record/data_record.py +380 -0
- biolib/_index/__init__.py +0 -0
- biolib/_index/index.py +55 -0
- biolib/_index/query_result.py +103 -0
- biolib/_internal/__init__.py +0 -0
- biolib/_internal/add_copilot_prompts.py +58 -0
- biolib/_internal/add_gui_files.py +81 -0
- biolib/_internal/data_record/__init__.py +1 -0
- biolib/_internal/data_record/data_record.py +85 -0
- biolib/_internal/data_record/push_data.py +116 -0
- biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
- biolib/_internal/errors.py +5 -0
- biolib/_internal/file_utils.py +125 -0
- biolib/_internal/fuse_mount/__init__.py +1 -0
- biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
- biolib/_internal/http_client.py +159 -0
- biolib/_internal/lfs/__init__.py +1 -0
- biolib/_internal/lfs/cache.py +51 -0
- biolib/_internal/libs/__init__.py +1 -0
- biolib/_internal/libs/fusepy/__init__.py +1257 -0
- biolib/_internal/push_application.py +488 -0
- biolib/_internal/runtime.py +22 -0
- biolib/_internal/string_utils.py +13 -0
- biolib/_internal/templates/__init__.py +1 -0
- biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
- biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
- biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
- biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
- biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
- biolib/_internal/templates/gitignore_template/.gitignore +10 -0
- biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
- biolib/_internal/templates/gui_template/App.tsx +53 -0
- biolib/_internal/templates/gui_template/Dockerfile +27 -0
- biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
- biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
- biolib/_internal/templates/gui_template/index.css +5 -0
- biolib/_internal/templates/gui_template/index.html +13 -0
- biolib/_internal/templates/gui_template/index.tsx +10 -0
- biolib/_internal/templates/gui_template/package.json +27 -0
- biolib/_internal/templates/gui_template/tsconfig.json +24 -0
- biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
- biolib/_internal/templates/gui_template/vite.config.mts +10 -0
- biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
- biolib/_internal/templates/init_template/Dockerfile +14 -0
- biolib/_internal/templates/init_template/requirements.txt +1 -0
- biolib/_internal/templates/init_template/run.py +12 -0
- biolib/_internal/templates/init_template/run.sh +4 -0
- biolib/_internal/templates/templates.py +25 -0
- biolib/_internal/tree_utils.py +106 -0
- biolib/_internal/utils/__init__.py +65 -0
- biolib/_internal/utils/auth.py +46 -0
- biolib/_internal/utils/job_url.py +33 -0
- biolib/_internal/utils/multinode.py +263 -0
- biolib/_runtime/runtime.py +157 -0
- biolib/_session/session.py +44 -0
- biolib/_shared/__init__.py +0 -0
- biolib/_shared/types/__init__.py +74 -0
- biolib/_shared/types/account.py +12 -0
- biolib/_shared/types/account_member.py +8 -0
- biolib/_shared/types/app.py +9 -0
- biolib/_shared/types/data_record.py +40 -0
- biolib/_shared/types/experiment.py +32 -0
- biolib/_shared/types/file_node.py +17 -0
- biolib/_shared/types/push.py +6 -0
- biolib/_shared/types/resource.py +37 -0
- biolib/_shared/types/resource_deploy_key.py +11 -0
- biolib/_shared/types/resource_permission.py +14 -0
- biolib/_shared/types/resource_version.py +19 -0
- biolib/_shared/types/result.py +14 -0
- biolib/_shared/types/typing.py +10 -0
- biolib/_shared/types/user.py +19 -0
- biolib/_shared/utils/__init__.py +7 -0
- biolib/_shared/utils/resource_uri.py +75 -0
- biolib/api/__init__.py +6 -0
- biolib/api/client.py +168 -0
- biolib/app/app.py +252 -49
- biolib/app/search_apps.py +45 -0
- biolib/biolib_api_client/api_client.py +126 -31
- biolib/biolib_api_client/app_types.py +24 -4
- biolib/biolib_api_client/auth.py +31 -8
- biolib/biolib_api_client/biolib_app_api.py +147 -52
- biolib/biolib_api_client/biolib_job_api.py +161 -141
- biolib/biolib_api_client/job_types.py +21 -5
- biolib/biolib_api_client/lfs_types.py +7 -23
- biolib/biolib_api_client/user_state.py +56 -0
- biolib/biolib_binary_format/__init__.py +1 -4
- biolib/biolib_binary_format/file_in_container.py +105 -0
- biolib/biolib_binary_format/module_input.py +24 -7
- biolib/biolib_binary_format/module_output_v2.py +149 -0
- biolib/biolib_binary_format/remote_endpoints.py +34 -0
- biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
- biolib/biolib_binary_format/saved_job.py +3 -2
- biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
- biolib/biolib_binary_format/system_status_update.py +3 -2
- biolib/biolib_binary_format/utils.py +175 -0
- biolib/biolib_docker_client/__init__.py +11 -2
- biolib/biolib_errors.py +36 -0
- biolib/biolib_logging.py +27 -10
- biolib/cli/__init__.py +38 -0
- biolib/cli/auth.py +46 -0
- biolib/cli/data_record.py +164 -0
- biolib/cli/index.py +32 -0
- biolib/cli/init.py +421 -0
- biolib/cli/lfs.py +101 -0
- biolib/cli/push.py +50 -0
- biolib/cli/run.py +63 -0
- biolib/cli/runtime.py +14 -0
- biolib/cli/sdk.py +16 -0
- biolib/cli/start.py +56 -0
- biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
- biolib/compute_node/job_worker/cache_state.py +66 -88
- biolib/compute_node/job_worker/cache_types.py +1 -6
- biolib/compute_node/job_worker/docker_image_cache.py +112 -37
- biolib/compute_node/job_worker/executors/__init__.py +0 -3
- biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
- biolib/compute_node/job_worker/executors/docker_types.py +9 -1
- biolib/compute_node/job_worker/executors/types.py +19 -9
- biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
- biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
- biolib/compute_node/job_worker/job_storage.py +108 -0
- biolib/compute_node/job_worker/job_worker.py +397 -212
- biolib/compute_node/job_worker/large_file_system.py +87 -38
- biolib/compute_node/job_worker/network_alloc.py +99 -0
- biolib/compute_node/job_worker/network_buffer.py +240 -0
- biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
- biolib/compute_node/job_worker/utils.py +9 -24
- biolib/compute_node/remote_host_proxy.py +400 -98
- biolib/compute_node/utils.py +31 -9
- biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
- biolib/compute_node/webserver/proxy_utils.py +28 -0
- biolib/compute_node/webserver/webserver.py +130 -44
- biolib/compute_node/webserver/webserver_types.py +2 -6
- biolib/compute_node/webserver/webserver_utils.py +77 -12
- biolib/compute_node/webserver/worker_thread.py +183 -42
- biolib/experiments/__init__.py +0 -0
- biolib/experiments/experiment.py +356 -0
- biolib/jobs/__init__.py +1 -0
- biolib/jobs/job.py +741 -0
- biolib/jobs/job_result.py +185 -0
- biolib/jobs/types.py +50 -0
- biolib/py.typed +0 -0
- biolib/runtime/__init__.py +14 -0
- biolib/sdk/__init__.py +91 -0
- biolib/tables.py +34 -0
- biolib/typing_utils.py +2 -7
- biolib/user/__init__.py +1 -0
- biolib/user/sign_in.py +54 -0
- biolib/utils/__init__.py +162 -0
- biolib/utils/cache_state.py +94 -0
- biolib/utils/multipart_uploader.py +194 -0
- biolib/utils/seq_util.py +150 -0
- biolib/utils/zip/remote_zip.py +640 -0
- pybiolib-1.2.1890.dist-info/METADATA +41 -0
- pybiolib-1.2.1890.dist-info/RECORD +177 -0
- {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
- pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
- README.md +0 -17
- biolib/app/app_result.py +0 -68
- biolib/app/utils.py +0 -62
- biolib/biolib-js/0-biolib.worker.js +0 -1
- biolib/biolib-js/1-biolib.worker.js +0 -1
- biolib/biolib-js/2-biolib.worker.js +0 -1
- biolib/biolib-js/3-biolib.worker.js +0 -1
- biolib/biolib-js/4-biolib.worker.js +0 -1
- biolib/biolib-js/5-biolib.worker.js +0 -1
- biolib/biolib-js/6-biolib.worker.js +0 -1
- biolib/biolib-js/index.html +0 -10
- biolib/biolib-js/main-biolib.js +0 -1
- biolib/biolib_api_client/biolib_account_api.py +0 -21
- biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
- biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
- biolib/biolib_binary_format/module_output.py +0 -58
- biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
- biolib/biolib_push.py +0 -114
- biolib/cli.py +0 -203
- biolib/cli_utils.py +0 -273
- biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
- biolib/compute_node/enclave/__init__.py +0 -2
- biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
- biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
- biolib/compute_node/job_worker/executors/base_executor.py +0 -18
- biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
- biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
- biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
- biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
- biolib/lfs.py +0 -196
- biolib/pyppeteer/.circleci/config.yml +0 -100
- biolib/pyppeteer/.coveragerc +0 -3
- biolib/pyppeteer/.gitignore +0 -89
- biolib/pyppeteer/.pre-commit-config.yaml +0 -28
- biolib/pyppeteer/CHANGES.md +0 -253
- biolib/pyppeteer/CONTRIBUTING.md +0 -26
- biolib/pyppeteer/LICENSE +0 -12
- biolib/pyppeteer/README.md +0 -137
- biolib/pyppeteer/docs/Makefile +0 -177
- biolib/pyppeteer/docs/_static/custom.css +0 -28
- biolib/pyppeteer/docs/_templates/layout.html +0 -10
- biolib/pyppeteer/docs/changes.md +0 -1
- biolib/pyppeteer/docs/conf.py +0 -299
- biolib/pyppeteer/docs/index.md +0 -21
- biolib/pyppeteer/docs/make.bat +0 -242
- biolib/pyppeteer/docs/reference.md +0 -211
- biolib/pyppeteer/docs/server.py +0 -60
- biolib/pyppeteer/poetry.lock +0 -1699
- biolib/pyppeteer/pyppeteer/__init__.py +0 -135
- biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
- biolib/pyppeteer/pyppeteer/browser.py +0 -401
- biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
- biolib/pyppeteer/pyppeteer/command.py +0 -22
- biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
- biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
- biolib/pyppeteer/pyppeteer/coverage.py +0 -346
- biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
- biolib/pyppeteer/pyppeteer/dialog.py +0 -79
- biolib/pyppeteer/pyppeteer/domworld.py +0 -597
- biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
- biolib/pyppeteer/pyppeteer/errors.py +0 -48
- biolib/pyppeteer/pyppeteer/events.py +0 -63
- biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
- biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
- biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
- biolib/pyppeteer/pyppeteer/helpers.py +0 -245
- biolib/pyppeteer/pyppeteer/input.py +0 -371
- biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
- biolib/pyppeteer/pyppeteer/launcher.py +0 -683
- biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
- biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
- biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
- biolib/pyppeteer/pyppeteer/multimap.py +0 -82
- biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
- biolib/pyppeteer/pyppeteer/options.py +0 -8
- biolib/pyppeteer/pyppeteer/page.py +0 -1728
- biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
- biolib/pyppeteer/pyppeteer/target.py +0 -147
- biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
- biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
- biolib/pyppeteer/pyppeteer/tracing.py +0 -93
- biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
- biolib/pyppeteer/pyppeteer/util.py +0 -18
- biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
- biolib/pyppeteer/pyppeteer/worker.py +0 -101
- biolib/pyppeteer/pyproject.toml +0 -97
- biolib/pyppeteer/spell.txt +0 -137
- biolib/pyppeteer/tox.ini +0 -72
- biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
- biolib/start_cli.py +0 -7
- biolib/utils.py +0 -47
- biolib/validators/validate_app_version.py +0 -183
- biolib/validators/validate_argument.py +0 -134
- biolib/validators/validate_module.py +0 -323
- biolib/validators/validate_zip_file.py +0 -40
- biolib/validators/validator_utils.py +0 -103
- pybiolib-0.2.951.dist-info/LICENSE +0 -21
- pybiolib-0.2.951.dist-info/METADATA +0 -61
- pybiolib-0.2.951.dist-info/RECORD +0 -153
- pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
- /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
|
@@ -1,683 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import atexit
|
|
3
|
-
import json
|
|
4
|
-
import logging
|
|
5
|
-
import os
|
|
6
|
-
import re
|
|
7
|
-
import subprocess
|
|
8
|
-
import sys
|
|
9
|
-
import tempfile
|
|
10
|
-
import time
|
|
11
|
-
from contextlib import suppress
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
from signal import SIG_DFL, SIGINT, SIGTERM, signal
|
|
14
|
-
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
|
|
15
|
-
from urllib.error import URLError
|
|
16
|
-
from urllib.request import urlopen
|
|
17
|
-
|
|
18
|
-
from biolib.pyppeteer.pyppeteer import __chromium_revision__
|
|
19
|
-
from biolib.pyppeteer.pyppeteer.browser import Browser
|
|
20
|
-
from biolib.pyppeteer.pyppeteer.browser_fetcher import BrowserFetcher
|
|
21
|
-
from biolib.pyppeteer.pyppeteer.connection import Connection
|
|
22
|
-
from biolib.pyppeteer.pyppeteer.errors import BrowserError
|
|
23
|
-
from biolib.pyppeteer.pyppeteer.models import BrowserOptions, ChromeArgOptions, LaunchOptions, Protocol
|
|
24
|
-
from biolib.pyppeteer.pyppeteer.util import get_free_port
|
|
25
|
-
from biolib.pyppeteer.pyppeteer.websocket_transport import WebsocketTransport
|
|
26
|
-
|
|
27
|
-
if not sys.platform.startswith('win'):
|
|
28
|
-
from signal import SIGHUP
|
|
29
|
-
|
|
30
|
-
if sys.version_info < (3, 8):
|
|
31
|
-
pass
|
|
32
|
-
else:
|
|
33
|
-
pass
|
|
34
|
-
|
|
35
|
-
logger = logging.getLogger(__name__)
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class BrowserRunner:
|
|
39
|
-
def __init__(
|
|
40
|
-
self, executable_path: str, process_args: Sequence[str], temp_dir: tempfile.TemporaryDirectory = None,
|
|
41
|
-
):
|
|
42
|
-
self.executable_path = executable_path
|
|
43
|
-
self.process_args = process_args or []
|
|
44
|
-
self.temp_dir = temp_dir
|
|
45
|
-
|
|
46
|
-
self.proc: Optional[subprocess.Popen] = None
|
|
47
|
-
self.connection: Optional[Connection] = None
|
|
48
|
-
|
|
49
|
-
self._altered_sighandlers: Set[int] = set()
|
|
50
|
-
self._closed = True
|
|
51
|
-
|
|
52
|
-
def start(self, **kwargs: LaunchOptions) -> None:
|
|
53
|
-
process_opts = {}
|
|
54
|
-
if kwargs.get('pipe'):
|
|
55
|
-
raise NotImplementedError('Communication via pipe not supported')
|
|
56
|
-
if kwargs.get('env'):
|
|
57
|
-
process_opts['env'] = kwargs['env']
|
|
58
|
-
|
|
59
|
-
if not kwargs.get('dumpio'):
|
|
60
|
-
# we read stdout to check it for the ws endpoint
|
|
61
|
-
process_opts['stdout'] = subprocess.PIPE
|
|
62
|
-
process_opts['stderr'] = subprocess.STDOUT
|
|
63
|
-
else:
|
|
64
|
-
# todo: dumpio. See: https://pptr.dev/#?product=Puppeteer&version=v2.1.1&show=api-puppeteerlaunchoptions
|
|
65
|
-
# we need to tee proc stdout to both PIPE (so we can read it) and stdout (so users can see dumped IO)
|
|
66
|
-
# see these SO threads:
|
|
67
|
-
# https://stackoverflow.com/q/2996887/
|
|
68
|
-
# https://stackoverflow.com/q/17190221/
|
|
69
|
-
raise NotImplementedError(f'dumpio argument currently not implemented')
|
|
70
|
-
|
|
71
|
-
assert self.proc is None, 'This process has previously been started'
|
|
72
|
-
|
|
73
|
-
logger.debug(f'Calling process: {self.executable_path} {" ".join(self.process_args)}')
|
|
74
|
-
self.proc = subprocess.Popen([str(self.executable_path), *self.process_args], **process_opts)
|
|
75
|
-
self._closed = False
|
|
76
|
-
|
|
77
|
-
# ignore args from signals
|
|
78
|
-
def close_proc_wrapper(*_: Any) -> None:
|
|
79
|
-
asyncio.get_event_loop().run_until_complete(self._close_proc())
|
|
80
|
-
|
|
81
|
-
if kwargs.get('autoClose'):
|
|
82
|
-
atexit.register(close_proc_wrapper)
|
|
83
|
-
if kwargs.get('handleSIGINT'):
|
|
84
|
-
signal(SIGINT, close_proc_wrapper)
|
|
85
|
-
self._altered_sighandlers.add(SIGINT)
|
|
86
|
-
if kwargs.get('handleSIGTERM'):
|
|
87
|
-
signal(SIGTERM, close_proc_wrapper)
|
|
88
|
-
self._altered_sighandlers.add(SIGTERM)
|
|
89
|
-
if kwargs.get('handleSIGHUP'):
|
|
90
|
-
# SIGHUP is not defined on windows
|
|
91
|
-
if not sys.platform.startswith('win'):
|
|
92
|
-
signal(SIGHUP, close_proc_wrapper)
|
|
93
|
-
self._altered_sighandlers.add(SIGHUP)
|
|
94
|
-
else:
|
|
95
|
-
logger.warning(f'SIGHUP is not available on win32')
|
|
96
|
-
|
|
97
|
-
async def _close_proc(self) -> None:
|
|
98
|
-
if not self._closed:
|
|
99
|
-
if self.connection and self.connection._connected:
|
|
100
|
-
await self.connection.send('Browser.close')
|
|
101
|
-
await self.connection.dispose()
|
|
102
|
-
if self.temp_dir:
|
|
103
|
-
self._wait_for_proc_to_close()
|
|
104
|
-
self.temp_dir.cleanup()
|
|
105
|
-
|
|
106
|
-
def _wait_for_proc_to_close(self) -> None:
|
|
107
|
-
if self.proc is not None and self.proc.poll() is None and not self._closed:
|
|
108
|
-
try:
|
|
109
|
-
self.proc.terminate()
|
|
110
|
-
self.proc.wait()
|
|
111
|
-
except Exception as e:
|
|
112
|
-
logger.warning(f'error occurred on proc close: {e}')
|
|
113
|
-
|
|
114
|
-
async def close(self) -> None:
|
|
115
|
-
if not self._closed:
|
|
116
|
-
self._restore_default_signal_handlers()
|
|
117
|
-
if self.temp_dir:
|
|
118
|
-
self.kill()
|
|
119
|
-
elif self.connection:
|
|
120
|
-
try:
|
|
121
|
-
await self.connection.send('Browser.close')
|
|
122
|
-
except Exception as e:
|
|
123
|
-
logger.error(f'An exception occurred: {e}')
|
|
124
|
-
self.kill()
|
|
125
|
-
return await self._close_proc()
|
|
126
|
-
|
|
127
|
-
def kill(self) -> None:
|
|
128
|
-
if self.proc and not self._closed and self.proc.returncode is not None:
|
|
129
|
-
self._restore_default_signal_handlers()
|
|
130
|
-
try:
|
|
131
|
-
self.proc.kill()
|
|
132
|
-
if sys.platform.startswith('win'):
|
|
133
|
-
subprocess.Popen(['taskkill', '/PID', str(self.proc.pid), '/F'], shell=True).communicate()
|
|
134
|
-
|
|
135
|
-
except Exception as e:
|
|
136
|
-
logger.warning(f'Exception occurred while killing process {e}')
|
|
137
|
-
try:
|
|
138
|
-
if self.temp_dir:
|
|
139
|
-
self.temp_dir.cleanup()
|
|
140
|
-
except Exception as e:
|
|
141
|
-
logger.warning(f'Failed to cleanup {self.temp_dir}: {e}')
|
|
142
|
-
|
|
143
|
-
def _restore_default_signal_handlers(self) -> None:
|
|
144
|
-
"""
|
|
145
|
-
Restores the signals that were previously altered. This is contrary to
|
|
146
|
-
restoring all handlers, which panics in multithreaded applications
|
|
147
|
-
"""
|
|
148
|
-
for signum in self._altered_sighandlers:
|
|
149
|
-
signal(signum, SIG_DFL)
|
|
150
|
-
self._altered_sighandlers = set()
|
|
151
|
-
|
|
152
|
-
async def setupConnection(
|
|
153
|
-
self, usePipe: bool = None, timeout: float = None, slowMo: float = 0, preferredRevision: str = None,
|
|
154
|
-
) -> Connection:
|
|
155
|
-
|
|
156
|
-
if usePipe:
|
|
157
|
-
raise NotImplementedError('Communication via pipe not supported at this time')
|
|
158
|
-
# not currently support as we have no implementation for piping the stdout
|
|
159
|
-
# while also dumping to console
|
|
160
|
-
# may need to transition to asyncio.subprocess
|
|
161
|
-
# transport = PipeTransport(write_stream, read_stream)
|
|
162
|
-
# self.connection = Connection('', transport, delay=slowMo)
|
|
163
|
-
if self.proc is None:
|
|
164
|
-
raise RuntimeError('class process not initialized (self.proc = None)')
|
|
165
|
-
browser_ws_endpoint = waitForWSEndpoint(self.proc, timeout, preferredRevision)
|
|
166
|
-
transport = await WebsocketTransport.create(uri=browser_ws_endpoint)
|
|
167
|
-
self.connection = Connection(url=browser_ws_endpoint, transport=transport, delay=slowMo)
|
|
168
|
-
return self.connection
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
class BaseBrowserLauncher:
|
|
172
|
-
"""
|
|
173
|
-
Implements common BrowserLauncher operations
|
|
174
|
-
"""
|
|
175
|
-
|
|
176
|
-
def __init__(self, projectRoot: Union[Path, str] = None, preferredRevision: str = None):
|
|
177
|
-
self.projectRoot = Path(projectRoot) if projectRoot else None
|
|
178
|
-
self.preferredRevision = preferredRevision
|
|
179
|
-
|
|
180
|
-
async def connect(
|
|
181
|
-
self,
|
|
182
|
-
browserWSEndpoint: str = None,
|
|
183
|
-
browserURL: str = None,
|
|
184
|
-
transport: WebsocketTransport = None,
|
|
185
|
-
ignoreHTTPSErrors: bool = False,
|
|
186
|
-
slowMo: float = 0,
|
|
187
|
-
defaultViewport: Protocol.Page.Viewport = None,
|
|
188
|
-
) -> Browser:
|
|
189
|
-
if defaultViewport is None:
|
|
190
|
-
defaultViewport = {'width': 800, 'height': 600}
|
|
191
|
-
|
|
192
|
-
assert (
|
|
193
|
-
len([x for x in (browserWSEndpoint, browserURL, transport) if x]) == 1
|
|
194
|
-
), 'exactly one of browserWSEndpoint, browserURL, and transport must be specified'
|
|
195
|
-
|
|
196
|
-
if transport:
|
|
197
|
-
connection = Connection('', transport, slowMo)
|
|
198
|
-
else:
|
|
199
|
-
if browserURL:
|
|
200
|
-
browserWSEndpoint = getWSEndpoint(browserURL)
|
|
201
|
-
transport = await WebsocketTransport.create(uri=browserWSEndpoint)
|
|
202
|
-
connection = Connection(browserWSEndpoint, transport=transport, delay=slowMo)
|
|
203
|
-
|
|
204
|
-
async def close_callback() -> None:
|
|
205
|
-
await connection.send('Browser.close')
|
|
206
|
-
|
|
207
|
-
context_ids = await connection.send('Target.getBrowserContexts')
|
|
208
|
-
return await Browser.create(
|
|
209
|
-
connection=connection,
|
|
210
|
-
contextIds=context_ids,
|
|
211
|
-
ignoreHTTPSErrors=ignoreHTTPSErrors,
|
|
212
|
-
defaultViewport=defaultViewport,
|
|
213
|
-
process=None,
|
|
214
|
-
closeCallback=close_callback,
|
|
215
|
-
)
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
class ChromeLauncher(BaseBrowserLauncher):
|
|
219
|
-
DEFAULT_ARGS = [
|
|
220
|
-
'--disable-background-networking',
|
|
221
|
-
'--enable-features=NetworkService,NetworkServiceInProcess',
|
|
222
|
-
'--disable-background-timer-throttling',
|
|
223
|
-
'--disable-backgrounding-occluded-windows',
|
|
224
|
-
'--disable-breakpad',
|
|
225
|
-
'--disable-client-side-phishing-detection',
|
|
226
|
-
'--disable-component-extensions-with-background-pages',
|
|
227
|
-
'--disable-default-apps',
|
|
228
|
-
'--disable-dev-shm-usage',
|
|
229
|
-
'--disable-extensions',
|
|
230
|
-
'--disable-features=TranslateUI',
|
|
231
|
-
'--disable-hang-monitor',
|
|
232
|
-
'--disable-ipc-flooding-protection',
|
|
233
|
-
'--disable-popup-blocking',
|
|
234
|
-
'--disable-prompt-on-repost',
|
|
235
|
-
'--disable-renderer-backgrounding',
|
|
236
|
-
'--disable-sync',
|
|
237
|
-
'--force-color-profile=srgb',
|
|
238
|
-
'--metrics-recording-only',
|
|
239
|
-
'--no-first-run',
|
|
240
|
-
'--enable-automation',
|
|
241
|
-
'--password-store=basic',
|
|
242
|
-
'--use-mock-keychain',
|
|
243
|
-
]
|
|
244
|
-
product = 'chrome'
|
|
245
|
-
|
|
246
|
-
def __init__(self, projectRoot: str = None, preferredRevision: str = None):
|
|
247
|
-
if not preferredRevision:
|
|
248
|
-
preferredRevision = __chromium_revision__
|
|
249
|
-
super().__init__(projectRoot, preferredRevision)
|
|
250
|
-
|
|
251
|
-
@property
|
|
252
|
-
def executable_path(self) -> Optional[str]:
|
|
253
|
-
return resolveExecutablePath(self.projectRoot, self.preferredRevision)[0]
|
|
254
|
-
|
|
255
|
-
async def launch(self, **kwargs: Union[LaunchOptions, ChromeArgOptions, BrowserOptions]) -> Browser:
|
|
256
|
-
ignoreDefaultArgs = kwargs.get('ignoreDefaultArgs', False)
|
|
257
|
-
args: Sequence[str] = kwargs.get('args', [])
|
|
258
|
-
dumpio = kwargs.get('dumpio', False)
|
|
259
|
-
executablePath = kwargs.get('executablePath', None)
|
|
260
|
-
env = kwargs.get('env', os.environ)
|
|
261
|
-
handleSIGINT = kwargs.get('handleSIGINT', True)
|
|
262
|
-
handleSIGTERM = kwargs.get('handleSIGTERM', True)
|
|
263
|
-
handleSIGHUP = kwargs.get('handleSIGHUP', not sys.platform.startswith('win'))
|
|
264
|
-
ignoreHTTPSErrors = kwargs.get('ignoreHTTPSErrors', False)
|
|
265
|
-
defaultViewport = kwargs.get('defaultViewport', {'width': 800, 'height': 600})
|
|
266
|
-
slowMo = kwargs.get('slowMo', 0)
|
|
267
|
-
timeout = kwargs.get('timeout', 30_000)
|
|
268
|
-
profile_path = None
|
|
269
|
-
|
|
270
|
-
chrome_args = []
|
|
271
|
-
if not ignoreDefaultArgs:
|
|
272
|
-
chrome_args.extend(self.default_args(**kwargs))
|
|
273
|
-
elif isinstance(ignoreDefaultArgs, list):
|
|
274
|
-
chrome_args.extend([x for x in self.default_args(**kwargs) if x not in ignoreDefaultArgs])
|
|
275
|
-
else:
|
|
276
|
-
chrome_args.extend(args)
|
|
277
|
-
|
|
278
|
-
if not any(x.startswith('--remote-debugging-') for x in chrome_args):
|
|
279
|
-
chrome_args.append(f'--remote-debugging-port={get_free_port()}')
|
|
280
|
-
if not any(x.startswith(f'--user-data-dir') for x in chrome_args):
|
|
281
|
-
profile_path = tempfile.TemporaryDirectory(prefix='pyppeteer_chrome_profile_')
|
|
282
|
-
chrome_args.append(f'--user-data-dir={profile_path.name}')
|
|
283
|
-
|
|
284
|
-
if not executablePath:
|
|
285
|
-
chrome_executable, missing_text = resolveExecutablePath(self.projectRoot, self.preferredRevision)
|
|
286
|
-
if missing_text:
|
|
287
|
-
raise RuntimeError(missing_text)
|
|
288
|
-
else:
|
|
289
|
-
chrome_executable = executablePath
|
|
290
|
-
|
|
291
|
-
usePipe = '--remote-debugging-pipe' in chrome_args
|
|
292
|
-
|
|
293
|
-
runner = BrowserRunner(chrome_executable, chrome_args, profile_path)
|
|
294
|
-
runner.start(
|
|
295
|
-
handleSIGINT=handleSIGINT, handleSIGHUP=handleSIGHUP, handleSIGTERM=handleSIGTERM, env=env, dumpio=dumpio, autoClose=kwargs.get('autoClose')
|
|
296
|
-
)
|
|
297
|
-
|
|
298
|
-
try:
|
|
299
|
-
connection = await runner.setupConnection(
|
|
300
|
-
usePipe=usePipe, timeout=timeout, slowMo=slowMo, preferredRevision=self.preferredRevision,
|
|
301
|
-
)
|
|
302
|
-
browser = await Browser.create(
|
|
303
|
-
connection=connection,
|
|
304
|
-
contextIds=[],
|
|
305
|
-
ignoreHTTPSErrors=ignoreHTTPSErrors,
|
|
306
|
-
defaultViewport=defaultViewport,
|
|
307
|
-
process=runner.proc,
|
|
308
|
-
closeCallback=runner.close,
|
|
309
|
-
)
|
|
310
|
-
await browser.waitForTarget(lambda x: x.type == 'page')
|
|
311
|
-
return browser
|
|
312
|
-
except Exception as original_exception:
|
|
313
|
-
logger.error(f'error occurred while trying to launch browser: {original_exception}')
|
|
314
|
-
with suppress(Exception):
|
|
315
|
-
runner.kill()
|
|
316
|
-
raise original_exception
|
|
317
|
-
|
|
318
|
-
def default_args(
|
|
319
|
-
self,
|
|
320
|
-
args: Sequence[str] = None,
|
|
321
|
-
devtools: bool = False,
|
|
322
|
-
headless: bool = None,
|
|
323
|
-
userDataDir: str = None,
|
|
324
|
-
**_: Any,
|
|
325
|
-
) -> List[str]:
|
|
326
|
-
if headless is None:
|
|
327
|
-
headless = not devtools
|
|
328
|
-
if args is None:
|
|
329
|
-
args = []
|
|
330
|
-
chrome_args = self.DEFAULT_ARGS[:]
|
|
331
|
-
if isinstance(args, Sequence):
|
|
332
|
-
chrome_args.extend(args)
|
|
333
|
-
if userDataDir:
|
|
334
|
-
chrome_args.append(f'--user-data-dir={userDataDir}')
|
|
335
|
-
if devtools:
|
|
336
|
-
chrome_args.append('--auto-open-devtools-for-tabs')
|
|
337
|
-
if headless:
|
|
338
|
-
chrome_args.extend(('--headless', '--hide-scrollbars', '--mute-audio'))
|
|
339
|
-
if all(x.startswith('-') for x in args):
|
|
340
|
-
chrome_args.append('about:blank')
|
|
341
|
-
return chrome_args
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
class FirefoxLauncher(BaseBrowserLauncher):
|
|
345
|
-
DEFAULT_ARGS = [
|
|
346
|
-
'--remote-debugging-port=0',
|
|
347
|
-
'--no-remote',
|
|
348
|
-
'--foreground',
|
|
349
|
-
]
|
|
350
|
-
_server = 'dummy.test'
|
|
351
|
-
DEFAULT_PROFILE_PREFS = {
|
|
352
|
-
# Make sure Shield doesn't hit the network.
|
|
353
|
-
'app.normandy.api_url': '',
|
|
354
|
-
# Disable Firefox old build background check
|
|
355
|
-
'app.update.checkInstallTime': False,
|
|
356
|
-
# Disable automatically upgrading Firefox
|
|
357
|
-
'app.update.disabledForTesting': True,
|
|
358
|
-
# Increase the APZ content response timeout to 2 minute
|
|
359
|
-
'apz.content_response_timeout': 60000,
|
|
360
|
-
# Prevent various error message on the console
|
|
361
|
-
# jest-puppeteer asserts that no error message is emitted by the console
|
|
362
|
-
'browser.contentblocking.features.standard': '-tp,tpPrivate,cookieBehavior0,-cm,-fp',
|
|
363
|
-
# Enable the dump function: which sends messages to the system
|
|
364
|
-
# console
|
|
365
|
-
# https://bugzilla.mozilla.org/show_bug.cgi?id=1543115
|
|
366
|
-
'browser.dom.window.dump.enabled': True,
|
|
367
|
-
# Disable topstories
|
|
368
|
-
'browser.newtabpage.activity-stream.feeds.section.topstories': False,
|
|
369
|
-
# Always display a blank page
|
|
370
|
-
'browser.newtabpage.enabled': False,
|
|
371
|
-
# Background thumbnails in particular cause grief: and disabling
|
|
372
|
-
# thumbnails in general cannot hurt
|
|
373
|
-
'browser.pagethumbnails.capturing_disabled': True,
|
|
374
|
-
# Disable safebrowsing components.
|
|
375
|
-
'browser.safebrowsing.blockedURIs.enabled': False,
|
|
376
|
-
'browser.safebrowsing.downloads.enabled': False,
|
|
377
|
-
'browser.safebrowsing.malware.enabled': False,
|
|
378
|
-
'browser.safebrowsing.passwords.enabled': False,
|
|
379
|
-
'browser.safebrowsing.phishing.enabled': False,
|
|
380
|
-
# Disable updates to search engines.
|
|
381
|
-
'browser.search.update': False,
|
|
382
|
-
# Do not restore the last open set of tabs if the browser has crashed
|
|
383
|
-
'browser.sessionstore.resume_from_crash': False,
|
|
384
|
-
# Skip check for default browser on startup
|
|
385
|
-
'browser.shell.checkDefaultBrowser': False,
|
|
386
|
-
# Disable newtabpage
|
|
387
|
-
'browser.startup.homepage': 'about:blank',
|
|
388
|
-
# Do not redirect user when a milstone upgrade of Firefox is detected
|
|
389
|
-
'browser.startup.homepage_override.mstone': 'ignore',
|
|
390
|
-
# Start with a blank page about:blank
|
|
391
|
-
'browser.startup.page': 0,
|
|
392
|
-
# Do not allow background tabs to be zombified on Android: otherwise for
|
|
393
|
-
# tests that open additional tabs: the test harness tab itself might get
|
|
394
|
-
# unloaded
|
|
395
|
-
'browser.tabs.disableBackgroundZombification': False,
|
|
396
|
-
# Do not warn when closing all other open tabs
|
|
397
|
-
'browser.tabs.warnOnCloseOtherTabs': False,
|
|
398
|
-
# Do not warn when multiple tabs will be opened
|
|
399
|
-
'browser.tabs.warnOnOpen': False,
|
|
400
|
-
# Disable the UI tour.
|
|
401
|
-
'browser.uitour.enabled': False,
|
|
402
|
-
# Turn off search suggestions in the location bar so as not to trigger
|
|
403
|
-
# network connections.
|
|
404
|
-
'browser.urlbar.suggest.searches': False,
|
|
405
|
-
# Disable first run splash page on Windows 10
|
|
406
|
-
'browser.usedOnWindows10.introURL': '',
|
|
407
|
-
# Do not warn on quitting Firefox
|
|
408
|
-
'browser.warnOnQuit': False,
|
|
409
|
-
# Do not show datareporting policy notifications which can
|
|
410
|
-
# interfere with tests
|
|
411
|
-
'datareporting.healthreport.about.reportUrl': f'http://{_server}/dummy/abouthealthreport/',
|
|
412
|
-
'datareporting.healthreport.documentServerURI': f'http://{_server}/dummy/healthreport/',
|
|
413
|
-
'datareporting.healthreport.logging.consoleEnabled': False,
|
|
414
|
-
'datareporting.healthreport.service.enabled': False,
|
|
415
|
-
'datareporting.healthreport.service.firstRun': False,
|
|
416
|
-
'datareporting.healthreport.uploadEnabled': False,
|
|
417
|
-
'datareporting.policy.dataSubmissionEnabled': False,
|
|
418
|
-
'datareporting.policy.dataSubmissionPolicyAccepted': False,
|
|
419
|
-
'datareporting.policy.dataSubmissionPolicyBypassNotification': True,
|
|
420
|
-
# DevTools JSONViewer sometimes fails to load dependencies with its require.js.
|
|
421
|
-
# This doesn't affect Puppeteer but spams console (Bug 1424372)
|
|
422
|
-
'devtools.jsonview.enabled': False,
|
|
423
|
-
# Disable popup-blocker
|
|
424
|
-
'dom.disable_open_during_load': False,
|
|
425
|
-
# Enable the support for File object creation in the content process
|
|
426
|
-
# Required for |Page.setFileInputFiles| protocol method.
|
|
427
|
-
'dom.file.createInChild': True,
|
|
428
|
-
# Disable the ProcessHangMonitor
|
|
429
|
-
'dom.ipc.reportProcessHangs': False,
|
|
430
|
-
# Disable slow script dialogues
|
|
431
|
-
'dom.max_chrome_script_run_time': 0,
|
|
432
|
-
'dom.max_script_run_time': 0,
|
|
433
|
-
# Only load extensions from the application and user profile
|
|
434
|
-
# AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION
|
|
435
|
-
'extensions.autoDisableScopes': 0,
|
|
436
|
-
'extensions.enabledScopes': 5,
|
|
437
|
-
# Disable metadata caching for installed add-ons by default
|
|
438
|
-
'extensions.getAddons.cache.enabled': False,
|
|
439
|
-
# Disable installing any distribution extensions or add-ons.
|
|
440
|
-
'extensions.installDistroAddons': False,
|
|
441
|
-
# Disabled screenshots extension
|
|
442
|
-
'extensions.screenshots.disabled': True,
|
|
443
|
-
# Turn off extension updates so they do not bother tests
|
|
444
|
-
'extensions.update.enabled': False,
|
|
445
|
-
# Turn off extension updates so they do not bother tests
|
|
446
|
-
'extensions.update.notifyUser': False,
|
|
447
|
-
# Make sure opening about:addons will not hit the network
|
|
448
|
-
'extensions.webservice.discoverURL': f'http://{_server}/dummy/discoveryURL',
|
|
449
|
-
# Allow the application to have focus even it runs in the background
|
|
450
|
-
'focusmanager.testmode': True,
|
|
451
|
-
# Disable useragent updates
|
|
452
|
-
'general.useragent.updates.enabled': False,
|
|
453
|
-
# Always use network provider for geolocation tests so we bypass the
|
|
454
|
-
# macOS dialog raised by the corelocation provider
|
|
455
|
-
'geo.provider.testing': True,
|
|
456
|
-
# Do not scan Wifi
|
|
457
|
-
'geo.wifi.scan': False,
|
|
458
|
-
# No hang monitor
|
|
459
|
-
'hangmonitor.timeout': 0,
|
|
460
|
-
# Show chrome errors and warnings in the error console
|
|
461
|
-
'javascript.options.showInConsole': True,
|
|
462
|
-
# Disable download and usage of OpenH264: and Widevine plugins
|
|
463
|
-
'media.gmp-manager.updateEnabled': False,
|
|
464
|
-
# Prevent various error message on the console
|
|
465
|
-
# jest-puppeteer asserts that no error message is emitted by the console
|
|
466
|
-
'network.cookie.cookieBehavior': 0,
|
|
467
|
-
# Do not prompt for temporary redirects
|
|
468
|
-
'network.http.prompt-temp-redirect': False,
|
|
469
|
-
# Disable speculative connections so they are not reported as leaking
|
|
470
|
-
# when they are hanging around
|
|
471
|
-
'network.http.speculative-parallel-limit': 0,
|
|
472
|
-
# Do not automatically switch between offline and online
|
|
473
|
-
'network.manage-offline-status': False,
|
|
474
|
-
# Make sure SNTP requests do not hit the network
|
|
475
|
-
'network.sntp.pools': _server,
|
|
476
|
-
# Disable Flash.
|
|
477
|
-
'plugin.state.flash': 0,
|
|
478
|
-
'privacy.trackingprotection.enabled': False,
|
|
479
|
-
# Enable Remote Agent
|
|
480
|
-
# https://bugzilla.mozilla.org/show_bug.cgi?id=1544393
|
|
481
|
-
'remote.enabled': True,
|
|
482
|
-
# Don't do network connections for mitm priming
|
|
483
|
-
'security.certerrors.mitm.priming.enabled': False,
|
|
484
|
-
# Local documents have access to all other local documents,
|
|
485
|
-
# including directory listings
|
|
486
|
-
'security.fileuri.strict_origin_policy': False,
|
|
487
|
-
# Do not wait for the notification button security delay
|
|
488
|
-
'security.notification_enable_delay': 0,
|
|
489
|
-
# Ensure blocklist updates do not hit the network
|
|
490
|
-
'services.settings.server': f'http://{_server}/dummy/blocklist/',
|
|
491
|
-
# Do not automatically fill sign-in forms with known usernames and
|
|
492
|
-
# passwords
|
|
493
|
-
'signon.autofillForms': False,
|
|
494
|
-
# Disable password capture, so that tests that include forms are not
|
|
495
|
-
# influenced by the presence of the persistent doorhanger notification
|
|
496
|
-
'signon.rememberSignons': False,
|
|
497
|
-
# Disable first-run welcome page
|
|
498
|
-
'startup.homepage_welcome_url': 'about:blank',
|
|
499
|
-
# Disable first-run welcome page
|
|
500
|
-
'startup.homepage_welcome_url.additional': '',
|
|
501
|
-
# Disable browser animations (tabs, fullscreen, sliding alerts)
|
|
502
|
-
'toolkit.cosmeticAnimations.enabled': False,
|
|
503
|
-
# We want to collect telemetry, but we don't want to send in the results
|
|
504
|
-
'toolkit.telemetry.server': f'https://{_server}/dummy/telemetry/',
|
|
505
|
-
# Prevent starting into safe mode after application crashes
|
|
506
|
-
'toolkit.startup.max_resumed_crashes': -1,
|
|
507
|
-
}
|
|
508
|
-
product = 'firefox'
|
|
509
|
-
|
|
510
|
-
def __init__(self, projectRoot: str = None, preferredRevision: str = None):
|
|
511
|
-
super().__init__(projectRoot, preferredRevision)
|
|
512
|
-
|
|
513
|
-
@property
|
|
514
|
-
def executablePath(self) -> Optional[Path]:
|
|
515
|
-
raise NotImplementedError('executablePath method not implemented')
|
|
516
|
-
|
|
517
|
-
async def launch(self, **kwargs: Union[LaunchOptions, ChromeArgOptions, BrowserOptions]) -> Browser:
|
|
518
|
-
ignoreDefaultArgs = kwargs.get('ignoreDefaultArgs', False)
|
|
519
|
-
args: Sequence[str] = kwargs.get('args', [])
|
|
520
|
-
dumpio = kwargs.get('dumpio', False)
|
|
521
|
-
executablePath = kwargs.get('executablePath', None)
|
|
522
|
-
pipe = kwargs.get('pipe', False)
|
|
523
|
-
env = kwargs.get('env', os.environ)
|
|
524
|
-
handleSIGINT = kwargs.get('handleSIGINT', True)
|
|
525
|
-
handleSIGTERM = kwargs.get('handleSIGTERM', True)
|
|
526
|
-
handleSIGHUP = kwargs.get('handleSIGHUP', not sys.platform.startswith('win'))
|
|
527
|
-
ignoreHTTPSErrors = kwargs.get('ignoreHTTPSErrors', False)
|
|
528
|
-
defaultViewport = kwargs.get('defaultViewport', {'width': 800, 'height': 600})
|
|
529
|
-
slowMo = kwargs.get('slowMo', 0)
|
|
530
|
-
timeout = kwargs.get('timeout', 30_000)
|
|
531
|
-
profile_path = None
|
|
532
|
-
|
|
533
|
-
firefox_args = []
|
|
534
|
-
if not ignoreDefaultArgs:
|
|
535
|
-
firefox_args.extend(self.default_args(**kwargs))
|
|
536
|
-
elif isinstance(ignoreDefaultArgs, Sequence):
|
|
537
|
-
firefox_args.extend([x for x in self.default_args()])
|
|
538
|
-
else:
|
|
539
|
-
firefox_args.extend(args)
|
|
540
|
-
|
|
541
|
-
if '-profile' not in firefox_args and '--profile' not in firefox_args:
|
|
542
|
-
profile_path = tempfile.TemporaryDirectory('pyppyeteer2_firefox_profile_')
|
|
543
|
-
firefox_args.extend(('--profile', profile_path.name))
|
|
544
|
-
|
|
545
|
-
if not executablePath:
|
|
546
|
-
missing_text, executablePath = resolveExecutablePath(self.projectRoot, self.preferredRevision)
|
|
547
|
-
if missing_text:
|
|
548
|
-
raise RuntimeError(missing_text)
|
|
549
|
-
|
|
550
|
-
runner = BrowserRunner(executable_path=executablePath, process_args=firefox_args, temp_dir=profile_path)
|
|
551
|
-
runner.start(
|
|
552
|
-
handleSIGHUP=handleSIGHUP,
|
|
553
|
-
handleSIGTERM=handleSIGTERM,
|
|
554
|
-
handleSIGINT=handleSIGINT,
|
|
555
|
-
dumpio=dumpio,
|
|
556
|
-
env=env,
|
|
557
|
-
pipe=pipe,
|
|
558
|
-
)
|
|
559
|
-
|
|
560
|
-
try:
|
|
561
|
-
connection = await runner.setupConnection(
|
|
562
|
-
usePipe=pipe, timeout=timeout, slowMo=slowMo, preferredRevision=self.preferredRevision,
|
|
563
|
-
)
|
|
564
|
-
browser = await Browser.create(
|
|
565
|
-
connection=connection,
|
|
566
|
-
contextIds=[],
|
|
567
|
-
ignoreHTTPSErrors=ignoreHTTPSErrors,
|
|
568
|
-
defaultViewport=defaultViewport,
|
|
569
|
-
process=runner.proc,
|
|
570
|
-
closeCallback=runner.close,
|
|
571
|
-
)
|
|
572
|
-
await browser.waitForTarget(lambda t: t.type == 'page')
|
|
573
|
-
return browser
|
|
574
|
-
except Exception as original_exception:
|
|
575
|
-
logger.error(f'error occurred while trying to launch browser: {original_exception}')
|
|
576
|
-
with suppress(Exception):
|
|
577
|
-
runner.kill()
|
|
578
|
-
raise original_exception
|
|
579
|
-
|
|
580
|
-
def default_args(
|
|
581
|
-
self,
|
|
582
|
-
args: Sequence[str] = None,
|
|
583
|
-
devtools: bool = False,
|
|
584
|
-
headless: bool = None,
|
|
585
|
-
userDataDir: str = None,
|
|
586
|
-
**_: Any,
|
|
587
|
-
) -> List[str]:
|
|
588
|
-
if headless is None:
|
|
589
|
-
headless = not devtools
|
|
590
|
-
proc_args = self.DEFAULT_ARGS[:]
|
|
591
|
-
if isinstance(args, Sequence):
|
|
592
|
-
proc_args.extend(args)
|
|
593
|
-
if userDataDir:
|
|
594
|
-
proc_args.extend(['--profile', userDataDir])
|
|
595
|
-
if headless:
|
|
596
|
-
proc_args.append('--headless')
|
|
597
|
-
if devtools:
|
|
598
|
-
proc_args.append('--devtools')
|
|
599
|
-
if all(x.startswith('-') for x in proc_args):
|
|
600
|
-
proc_args.append('about:blank')
|
|
601
|
-
|
|
602
|
-
return proc_args
|
|
603
|
-
|
|
604
|
-
def _create_profile(self, extra_prefs: Dict[str, Any]) -> tempfile.TemporaryDirectory:
|
|
605
|
-
profile_path = tempfile.TemporaryDirectory(prefix='pyppeteer_firefox_profile')
|
|
606
|
-
prefs = {**self.DEFAULT_PROFILE_PREFS, **extra_prefs}
|
|
607
|
-
serialized_prefs = [f'user_pref({json.dumps(key)}, {json.dumps(val)}' for key, val in prefs.items()]
|
|
608
|
-
with open(Path(profile_path.name).joinpath('user.js'), 'w') as user:
|
|
609
|
-
user.write('\n'.join(serialized_prefs))
|
|
610
|
-
return profile_path
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
def waitForWSEndpoint(proc: subprocess.Popen, timeout: Optional[float], preferredRevision: Optional[str]) -> str:
|
|
614
|
-
assert proc.stdout is not None, 'process STDOUT wasn\'t piped'
|
|
615
|
-
start = time.perf_counter()
|
|
616
|
-
buffer = ''
|
|
617
|
-
for line in iter(proc.stdout.readline, b''):
|
|
618
|
-
line = line.decode()
|
|
619
|
-
buffer += '\n' + line
|
|
620
|
-
if timeout and (start - time.perf_counter()) > timeout:
|
|
621
|
-
raise TimeoutError(
|
|
622
|
-
f'Timed out after {timeout * 1000:.0f}ms while trying to connect to the browser! '
|
|
623
|
-
f'Only Chrome at revision {preferredRevision} is guaranteed to work.'
|
|
624
|
-
)
|
|
625
|
-
potential_match = re.match(r'DevTools listening on (ws://[\w.:/-]*)+', line)
|
|
626
|
-
if potential_match:
|
|
627
|
-
return potential_match.group(1)
|
|
628
|
-
raise RuntimeError(
|
|
629
|
-
buffer + '\nProcess ended before WebSockets endpoint could be found.'
|
|
630
|
-
f'Only Chrome at revision {preferredRevision} is guaranteed to work.'
|
|
631
|
-
)
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
def getWSEndpoint(url: str) -> str:
|
|
635
|
-
url += '/json/version'
|
|
636
|
-
timeout = time.perf_counter() + 30
|
|
637
|
-
while True:
|
|
638
|
-
if time.perf_counter() > timeout:
|
|
639
|
-
raise BrowserError('Browser closed unexpectedly:\n')
|
|
640
|
-
try:
|
|
641
|
-
with urlopen(url) as f:
|
|
642
|
-
data = json.loads(f.read().decode())
|
|
643
|
-
break
|
|
644
|
-
except URLError:
|
|
645
|
-
time.sleep(0.1)
|
|
646
|
-
try:
|
|
647
|
-
return data['webSocketDebuggerUrl']
|
|
648
|
-
except KeyError:
|
|
649
|
-
raise RuntimeError(f'webSocketDebuggerUrl not found')
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
def resolveExecutablePath(projectRoot: Path, preferred_revision: str) -> Tuple[Optional[Path], Optional[str]]:
|
|
653
|
-
missing_text = None
|
|
654
|
-
exec_path_env_var = 'PYPPETEER_EXECUTABLE_PATH'
|
|
655
|
-
revision_env_var = 'PYPPETEER_CHROMIUM_REVISION'
|
|
656
|
-
executable = os.environ.get(exec_path_env_var)
|
|
657
|
-
if executable:
|
|
658
|
-
if not Path(executable).is_file():
|
|
659
|
-
missing_text = f'Tried to use env variable ({exec_path_env_var}) to launch browser, but no executable was found at {executable}'
|
|
660
|
-
return None, missing_text
|
|
661
|
-
browser_fetcher = BrowserFetcher(projectRoot)
|
|
662
|
-
revision = os.environ.get(revision_env_var)
|
|
663
|
-
if revision:
|
|
664
|
-
revision_info = browser_fetcher.revision_info(revision)
|
|
665
|
-
if not revision_info['local']:
|
|
666
|
-
missing_text = f'Tried to use env variables ({revision_env_var}) to launch browser, but did not find executable at {revision_info["executablePath"]}'
|
|
667
|
-
return None, missing_text
|
|
668
|
-
revision_info = browser_fetcher.revision_info(preferred_revision)
|
|
669
|
-
if not revision_info['local']:
|
|
670
|
-
missing_text = 'Browser is not downloaded. Try running pyppeteer-install'
|
|
671
|
-
return revision_info['executablePath'], missing_text
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
def launcher(
|
|
675
|
-
projectRoot: str = None, preferredRevision: str = None, product: str = None
|
|
676
|
-
) -> Union[FirefoxLauncher, ChromeLauncher]:
|
|
677
|
-
"""Returns the appropriate browser launcher class instance"""
|
|
678
|
-
product = product or os.environ.get('PYPPETEER_PRODUCT') or 'chrome'
|
|
679
|
-
if product == 'firefox':
|
|
680
|
-
return FirefoxLauncher(projectRoot, preferredRevision)
|
|
681
|
-
elif product in ('chrome', 'chromium'):
|
|
682
|
-
return ChromeLauncher(projectRoot, preferredRevision)
|
|
683
|
-
raise ValueError(f'Support for {product} has not been implemented')
|