pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. biolib/__init__.py +357 -11
  2. biolib/_data_record/data_record.py +380 -0
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +55 -0
  5. biolib/_index/query_result.py +103 -0
  6. biolib/_internal/__init__.py +0 -0
  7. biolib/_internal/add_copilot_prompts.py +58 -0
  8. biolib/_internal/add_gui_files.py +81 -0
  9. biolib/_internal/data_record/__init__.py +1 -0
  10. biolib/_internal/data_record/data_record.py +85 -0
  11. biolib/_internal/data_record/push_data.py +116 -0
  12. biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
  13. biolib/_internal/errors.py +5 -0
  14. biolib/_internal/file_utils.py +125 -0
  15. biolib/_internal/fuse_mount/__init__.py +1 -0
  16. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  17. biolib/_internal/http_client.py +159 -0
  18. biolib/_internal/lfs/__init__.py +1 -0
  19. biolib/_internal/lfs/cache.py +51 -0
  20. biolib/_internal/libs/__init__.py +1 -0
  21. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  22. biolib/_internal/push_application.py +488 -0
  23. biolib/_internal/runtime.py +22 -0
  24. biolib/_internal/string_utils.py +13 -0
  25. biolib/_internal/templates/__init__.py +1 -0
  26. biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
  27. biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
  28. biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
  29. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  30. biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
  31. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  32. biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
  33. biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
  34. biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
  35. biolib/_internal/templates/gitignore_template/.gitignore +10 -0
  36. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  37. biolib/_internal/templates/gui_template/App.tsx +53 -0
  38. biolib/_internal/templates/gui_template/Dockerfile +27 -0
  39. biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
  40. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  41. biolib/_internal/templates/gui_template/index.css +5 -0
  42. biolib/_internal/templates/gui_template/index.html +13 -0
  43. biolib/_internal/templates/gui_template/index.tsx +10 -0
  44. biolib/_internal/templates/gui_template/package.json +27 -0
  45. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  46. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
  47. biolib/_internal/templates/gui_template/vite.config.mts +10 -0
  48. biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
  49. biolib/_internal/templates/init_template/Dockerfile +14 -0
  50. biolib/_internal/templates/init_template/requirements.txt +1 -0
  51. biolib/_internal/templates/init_template/run.py +12 -0
  52. biolib/_internal/templates/init_template/run.sh +4 -0
  53. biolib/_internal/templates/templates.py +25 -0
  54. biolib/_internal/tree_utils.py +106 -0
  55. biolib/_internal/utils/__init__.py +65 -0
  56. biolib/_internal/utils/auth.py +46 -0
  57. biolib/_internal/utils/job_url.py +33 -0
  58. biolib/_internal/utils/multinode.py +263 -0
  59. biolib/_runtime/runtime.py +157 -0
  60. biolib/_session/session.py +44 -0
  61. biolib/_shared/__init__.py +0 -0
  62. biolib/_shared/types/__init__.py +74 -0
  63. biolib/_shared/types/account.py +12 -0
  64. biolib/_shared/types/account_member.py +8 -0
  65. biolib/_shared/types/app.py +9 -0
  66. biolib/_shared/types/data_record.py +40 -0
  67. biolib/_shared/types/experiment.py +32 -0
  68. biolib/_shared/types/file_node.py +17 -0
  69. biolib/_shared/types/push.py +6 -0
  70. biolib/_shared/types/resource.py +37 -0
  71. biolib/_shared/types/resource_deploy_key.py +11 -0
  72. biolib/_shared/types/resource_permission.py +14 -0
  73. biolib/_shared/types/resource_version.py +19 -0
  74. biolib/_shared/types/result.py +14 -0
  75. biolib/_shared/types/typing.py +10 -0
  76. biolib/_shared/types/user.py +19 -0
  77. biolib/_shared/utils/__init__.py +7 -0
  78. biolib/_shared/utils/resource_uri.py +75 -0
  79. biolib/api/__init__.py +6 -0
  80. biolib/api/client.py +168 -0
  81. biolib/app/app.py +252 -49
  82. biolib/app/search_apps.py +45 -0
  83. biolib/biolib_api_client/api_client.py +126 -31
  84. biolib/biolib_api_client/app_types.py +24 -4
  85. biolib/biolib_api_client/auth.py +31 -8
  86. biolib/biolib_api_client/biolib_app_api.py +147 -52
  87. biolib/biolib_api_client/biolib_job_api.py +161 -141
  88. biolib/biolib_api_client/job_types.py +21 -5
  89. biolib/biolib_api_client/lfs_types.py +7 -23
  90. biolib/biolib_api_client/user_state.py +56 -0
  91. biolib/biolib_binary_format/__init__.py +1 -4
  92. biolib/biolib_binary_format/file_in_container.py +105 -0
  93. biolib/biolib_binary_format/module_input.py +24 -7
  94. biolib/biolib_binary_format/module_output_v2.py +149 -0
  95. biolib/biolib_binary_format/remote_endpoints.py +34 -0
  96. biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
  97. biolib/biolib_binary_format/saved_job.py +3 -2
  98. biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
  99. biolib/biolib_binary_format/system_status_update.py +3 -2
  100. biolib/biolib_binary_format/utils.py +175 -0
  101. biolib/biolib_docker_client/__init__.py +11 -2
  102. biolib/biolib_errors.py +36 -0
  103. biolib/biolib_logging.py +27 -10
  104. biolib/cli/__init__.py +38 -0
  105. biolib/cli/auth.py +46 -0
  106. biolib/cli/data_record.py +164 -0
  107. biolib/cli/index.py +32 -0
  108. biolib/cli/init.py +421 -0
  109. biolib/cli/lfs.py +101 -0
  110. biolib/cli/push.py +50 -0
  111. biolib/cli/run.py +63 -0
  112. biolib/cli/runtime.py +14 -0
  113. biolib/cli/sdk.py +16 -0
  114. biolib/cli/start.py +56 -0
  115. biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
  116. biolib/compute_node/job_worker/cache_state.py +66 -88
  117. biolib/compute_node/job_worker/cache_types.py +1 -6
  118. biolib/compute_node/job_worker/docker_image_cache.py +112 -37
  119. biolib/compute_node/job_worker/executors/__init__.py +0 -3
  120. biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
  121. biolib/compute_node/job_worker/executors/docker_types.py +9 -1
  122. biolib/compute_node/job_worker/executors/types.py +19 -9
  123. biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
  124. biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
  125. biolib/compute_node/job_worker/job_storage.py +108 -0
  126. biolib/compute_node/job_worker/job_worker.py +397 -212
  127. biolib/compute_node/job_worker/large_file_system.py +87 -38
  128. biolib/compute_node/job_worker/network_alloc.py +99 -0
  129. biolib/compute_node/job_worker/network_buffer.py +240 -0
  130. biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
  131. biolib/compute_node/job_worker/utils.py +9 -24
  132. biolib/compute_node/remote_host_proxy.py +400 -98
  133. biolib/compute_node/utils.py +31 -9
  134. biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
  135. biolib/compute_node/webserver/proxy_utils.py +28 -0
  136. biolib/compute_node/webserver/webserver.py +130 -44
  137. biolib/compute_node/webserver/webserver_types.py +2 -6
  138. biolib/compute_node/webserver/webserver_utils.py +77 -12
  139. biolib/compute_node/webserver/worker_thread.py +183 -42
  140. biolib/experiments/__init__.py +0 -0
  141. biolib/experiments/experiment.py +356 -0
  142. biolib/jobs/__init__.py +1 -0
  143. biolib/jobs/job.py +741 -0
  144. biolib/jobs/job_result.py +185 -0
  145. biolib/jobs/types.py +50 -0
  146. biolib/py.typed +0 -0
  147. biolib/runtime/__init__.py +14 -0
  148. biolib/sdk/__init__.py +91 -0
  149. biolib/tables.py +34 -0
  150. biolib/typing_utils.py +2 -7
  151. biolib/user/__init__.py +1 -0
  152. biolib/user/sign_in.py +54 -0
  153. biolib/utils/__init__.py +162 -0
  154. biolib/utils/cache_state.py +94 -0
  155. biolib/utils/multipart_uploader.py +194 -0
  156. biolib/utils/seq_util.py +150 -0
  157. biolib/utils/zip/remote_zip.py +640 -0
  158. pybiolib-1.2.1890.dist-info/METADATA +41 -0
  159. pybiolib-1.2.1890.dist-info/RECORD +177 -0
  160. {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
  161. pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
  162. README.md +0 -17
  163. biolib/app/app_result.py +0 -68
  164. biolib/app/utils.py +0 -62
  165. biolib/biolib-js/0-biolib.worker.js +0 -1
  166. biolib/biolib-js/1-biolib.worker.js +0 -1
  167. biolib/biolib-js/2-biolib.worker.js +0 -1
  168. biolib/biolib-js/3-biolib.worker.js +0 -1
  169. biolib/biolib-js/4-biolib.worker.js +0 -1
  170. biolib/biolib-js/5-biolib.worker.js +0 -1
  171. biolib/biolib-js/6-biolib.worker.js +0 -1
  172. biolib/biolib-js/index.html +0 -10
  173. biolib/biolib-js/main-biolib.js +0 -1
  174. biolib/biolib_api_client/biolib_account_api.py +0 -21
  175. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
  176. biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
  177. biolib/biolib_binary_format/module_output.py +0 -58
  178. biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
  179. biolib/biolib_push.py +0 -114
  180. biolib/cli.py +0 -203
  181. biolib/cli_utils.py +0 -273
  182. biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
  183. biolib/compute_node/enclave/__init__.py +0 -2
  184. biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
  185. biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
  186. biolib/compute_node/job_worker/executors/base_executor.py +0 -18
  187. biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
  188. biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
  189. biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
  190. biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
  191. biolib/lfs.py +0 -196
  192. biolib/pyppeteer/.circleci/config.yml +0 -100
  193. biolib/pyppeteer/.coveragerc +0 -3
  194. biolib/pyppeteer/.gitignore +0 -89
  195. biolib/pyppeteer/.pre-commit-config.yaml +0 -28
  196. biolib/pyppeteer/CHANGES.md +0 -253
  197. biolib/pyppeteer/CONTRIBUTING.md +0 -26
  198. biolib/pyppeteer/LICENSE +0 -12
  199. biolib/pyppeteer/README.md +0 -137
  200. biolib/pyppeteer/docs/Makefile +0 -177
  201. biolib/pyppeteer/docs/_static/custom.css +0 -28
  202. biolib/pyppeteer/docs/_templates/layout.html +0 -10
  203. biolib/pyppeteer/docs/changes.md +0 -1
  204. biolib/pyppeteer/docs/conf.py +0 -299
  205. biolib/pyppeteer/docs/index.md +0 -21
  206. biolib/pyppeteer/docs/make.bat +0 -242
  207. biolib/pyppeteer/docs/reference.md +0 -211
  208. biolib/pyppeteer/docs/server.py +0 -60
  209. biolib/pyppeteer/poetry.lock +0 -1699
  210. biolib/pyppeteer/pyppeteer/__init__.py +0 -135
  211. biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
  212. biolib/pyppeteer/pyppeteer/browser.py +0 -401
  213. biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
  214. biolib/pyppeteer/pyppeteer/command.py +0 -22
  215. biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
  216. biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
  217. biolib/pyppeteer/pyppeteer/coverage.py +0 -346
  218. biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
  219. biolib/pyppeteer/pyppeteer/dialog.py +0 -79
  220. biolib/pyppeteer/pyppeteer/domworld.py +0 -597
  221. biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
  222. biolib/pyppeteer/pyppeteer/errors.py +0 -48
  223. biolib/pyppeteer/pyppeteer/events.py +0 -63
  224. biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
  225. biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
  226. biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
  227. biolib/pyppeteer/pyppeteer/helpers.py +0 -245
  228. biolib/pyppeteer/pyppeteer/input.py +0 -371
  229. biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
  230. biolib/pyppeteer/pyppeteer/launcher.py +0 -683
  231. biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
  232. biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
  233. biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
  234. biolib/pyppeteer/pyppeteer/multimap.py +0 -82
  235. biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
  236. biolib/pyppeteer/pyppeteer/options.py +0 -8
  237. biolib/pyppeteer/pyppeteer/page.py +0 -1728
  238. biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
  239. biolib/pyppeteer/pyppeteer/target.py +0 -147
  240. biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
  241. biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
  242. biolib/pyppeteer/pyppeteer/tracing.py +0 -93
  243. biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
  244. biolib/pyppeteer/pyppeteer/util.py +0 -18
  245. biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
  246. biolib/pyppeteer/pyppeteer/worker.py +0 -101
  247. biolib/pyppeteer/pyproject.toml +0 -97
  248. biolib/pyppeteer/spell.txt +0 -137
  249. biolib/pyppeteer/tox.ini +0 -72
  250. biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
  251. biolib/start_cli.py +0 -7
  252. biolib/utils.py +0 -47
  253. biolib/validators/validate_app_version.py +0 -183
  254. biolib/validators/validate_argument.py +0 -134
  255. biolib/validators/validate_module.py +0 -323
  256. biolib/validators/validate_zip_file.py +0 -40
  257. biolib/validators/validator_utils.py +0 -103
  258. pybiolib-0.2.951.dist-info/LICENSE +0 -21
  259. pybiolib-0.2.951.dist-info/METADATA +0 -61
  260. pybiolib-0.2.951.dist-info/RECORD +0 -153
  261. pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
  262. /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
@@ -1,683 +0,0 @@
1
- import asyncio
2
- import atexit
3
- import json
4
- import logging
5
- import os
6
- import re
7
- import subprocess
8
- import sys
9
- import tempfile
10
- import time
11
- from contextlib import suppress
12
- from pathlib import Path
13
- from signal import SIG_DFL, SIGINT, SIGTERM, signal
14
- from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
15
- from urllib.error import URLError
16
- from urllib.request import urlopen
17
-
18
- from biolib.pyppeteer.pyppeteer import __chromium_revision__
19
- from biolib.pyppeteer.pyppeteer.browser import Browser
20
- from biolib.pyppeteer.pyppeteer.browser_fetcher import BrowserFetcher
21
- from biolib.pyppeteer.pyppeteer.connection import Connection
22
- from biolib.pyppeteer.pyppeteer.errors import BrowserError
23
- from biolib.pyppeteer.pyppeteer.models import BrowserOptions, ChromeArgOptions, LaunchOptions, Protocol
24
- from biolib.pyppeteer.pyppeteer.util import get_free_port
25
- from biolib.pyppeteer.pyppeteer.websocket_transport import WebsocketTransport
26
-
27
- if not sys.platform.startswith('win'):
28
- from signal import SIGHUP
29
-
30
- if sys.version_info < (3, 8):
31
- pass
32
- else:
33
- pass
34
-
35
- logger = logging.getLogger(__name__)
36
-
37
-
38
- class BrowserRunner:
39
- def __init__(
40
- self, executable_path: str, process_args: Sequence[str], temp_dir: tempfile.TemporaryDirectory = None,
41
- ):
42
- self.executable_path = executable_path
43
- self.process_args = process_args or []
44
- self.temp_dir = temp_dir
45
-
46
- self.proc: Optional[subprocess.Popen] = None
47
- self.connection: Optional[Connection] = None
48
-
49
- self._altered_sighandlers: Set[int] = set()
50
- self._closed = True
51
-
52
- def start(self, **kwargs: LaunchOptions) -> None:
53
- process_opts = {}
54
- if kwargs.get('pipe'):
55
- raise NotImplementedError('Communication via pipe not supported')
56
- if kwargs.get('env'):
57
- process_opts['env'] = kwargs['env']
58
-
59
- if not kwargs.get('dumpio'):
60
- # we read stdout to check it for the ws endpoint
61
- process_opts['stdout'] = subprocess.PIPE
62
- process_opts['stderr'] = subprocess.STDOUT
63
- else:
64
- # todo: dumpio. See: https://pptr.dev/#?product=Puppeteer&version=v2.1.1&show=api-puppeteerlaunchoptions
65
- # we need to tee proc stdout to both PIPE (so we can read it) and stdout (so users can see dumped IO)
66
- # see these SO threads:
67
- # https://stackoverflow.com/q/2996887/
68
- # https://stackoverflow.com/q/17190221/
69
- raise NotImplementedError(f'dumpio argument currently not implemented')
70
-
71
- assert self.proc is None, 'This process has previously been started'
72
-
73
- logger.debug(f'Calling process: {self.executable_path} {" ".join(self.process_args)}')
74
- self.proc = subprocess.Popen([str(self.executable_path), *self.process_args], **process_opts)
75
- self._closed = False
76
-
77
- # ignore args from signals
78
- def close_proc_wrapper(*_: Any) -> None:
79
- asyncio.get_event_loop().run_until_complete(self._close_proc())
80
-
81
- if kwargs.get('autoClose'):
82
- atexit.register(close_proc_wrapper)
83
- if kwargs.get('handleSIGINT'):
84
- signal(SIGINT, close_proc_wrapper)
85
- self._altered_sighandlers.add(SIGINT)
86
- if kwargs.get('handleSIGTERM'):
87
- signal(SIGTERM, close_proc_wrapper)
88
- self._altered_sighandlers.add(SIGTERM)
89
- if kwargs.get('handleSIGHUP'):
90
- # SIGHUP is not defined on windows
91
- if not sys.platform.startswith('win'):
92
- signal(SIGHUP, close_proc_wrapper)
93
- self._altered_sighandlers.add(SIGHUP)
94
- else:
95
- logger.warning(f'SIGHUP is not available on win32')
96
-
97
- async def _close_proc(self) -> None:
98
- if not self._closed:
99
- if self.connection and self.connection._connected:
100
- await self.connection.send('Browser.close')
101
- await self.connection.dispose()
102
- if self.temp_dir:
103
- self._wait_for_proc_to_close()
104
- self.temp_dir.cleanup()
105
-
106
- def _wait_for_proc_to_close(self) -> None:
107
- if self.proc is not None and self.proc.poll() is None and not self._closed:
108
- try:
109
- self.proc.terminate()
110
- self.proc.wait()
111
- except Exception as e:
112
- logger.warning(f'error occurred on proc close: {e}')
113
-
114
- async def close(self) -> None:
115
- if not self._closed:
116
- self._restore_default_signal_handlers()
117
- if self.temp_dir:
118
- self.kill()
119
- elif self.connection:
120
- try:
121
- await self.connection.send('Browser.close')
122
- except Exception as e:
123
- logger.error(f'An exception occurred: {e}')
124
- self.kill()
125
- return await self._close_proc()
126
-
127
- def kill(self) -> None:
128
- if self.proc and not self._closed and self.proc.returncode is not None:
129
- self._restore_default_signal_handlers()
130
- try:
131
- self.proc.kill()
132
- if sys.platform.startswith('win'):
133
- subprocess.Popen(['taskkill', '/PID', str(self.proc.pid), '/F'], shell=True).communicate()
134
-
135
- except Exception as e:
136
- logger.warning(f'Exception occurred while killing process {e}')
137
- try:
138
- if self.temp_dir:
139
- self.temp_dir.cleanup()
140
- except Exception as e:
141
- logger.warning(f'Failed to cleanup {self.temp_dir}: {e}')
142
-
143
- def _restore_default_signal_handlers(self) -> None:
144
- """
145
- Restores the signals that were previously altered. This is contrary to
146
- restoring all handlers, which panics in multithreaded applications
147
- """
148
- for signum in self._altered_sighandlers:
149
- signal(signum, SIG_DFL)
150
- self._altered_sighandlers = set()
151
-
152
- async def setupConnection(
153
- self, usePipe: bool = None, timeout: float = None, slowMo: float = 0, preferredRevision: str = None,
154
- ) -> Connection:
155
-
156
- if usePipe:
157
- raise NotImplementedError('Communication via pipe not supported at this time')
158
- # not currently support as we have no implementation for piping the stdout
159
- # while also dumping to console
160
- # may need to transition to asyncio.subprocess
161
- # transport = PipeTransport(write_stream, read_stream)
162
- # self.connection = Connection('', transport, delay=slowMo)
163
- if self.proc is None:
164
- raise RuntimeError('class process not initialized (self.proc = None)')
165
- browser_ws_endpoint = waitForWSEndpoint(self.proc, timeout, preferredRevision)
166
- transport = await WebsocketTransport.create(uri=browser_ws_endpoint)
167
- self.connection = Connection(url=browser_ws_endpoint, transport=transport, delay=slowMo)
168
- return self.connection
169
-
170
-
171
- class BaseBrowserLauncher:
172
- """
173
- Implements common BrowserLauncher operations
174
- """
175
-
176
- def __init__(self, projectRoot: Union[Path, str] = None, preferredRevision: str = None):
177
- self.projectRoot = Path(projectRoot) if projectRoot else None
178
- self.preferredRevision = preferredRevision
179
-
180
- async def connect(
181
- self,
182
- browserWSEndpoint: str = None,
183
- browserURL: str = None,
184
- transport: WebsocketTransport = None,
185
- ignoreHTTPSErrors: bool = False,
186
- slowMo: float = 0,
187
- defaultViewport: Protocol.Page.Viewport = None,
188
- ) -> Browser:
189
- if defaultViewport is None:
190
- defaultViewport = {'width': 800, 'height': 600}
191
-
192
- assert (
193
- len([x for x in (browserWSEndpoint, browserURL, transport) if x]) == 1
194
- ), 'exactly one of browserWSEndpoint, browserURL, and transport must be specified'
195
-
196
- if transport:
197
- connection = Connection('', transport, slowMo)
198
- else:
199
- if browserURL:
200
- browserWSEndpoint = getWSEndpoint(browserURL)
201
- transport = await WebsocketTransport.create(uri=browserWSEndpoint)
202
- connection = Connection(browserWSEndpoint, transport=transport, delay=slowMo)
203
-
204
- async def close_callback() -> None:
205
- await connection.send('Browser.close')
206
-
207
- context_ids = await connection.send('Target.getBrowserContexts')
208
- return await Browser.create(
209
- connection=connection,
210
- contextIds=context_ids,
211
- ignoreHTTPSErrors=ignoreHTTPSErrors,
212
- defaultViewport=defaultViewport,
213
- process=None,
214
- closeCallback=close_callback,
215
- )
216
-
217
-
218
- class ChromeLauncher(BaseBrowserLauncher):
219
- DEFAULT_ARGS = [
220
- '--disable-background-networking',
221
- '--enable-features=NetworkService,NetworkServiceInProcess',
222
- '--disable-background-timer-throttling',
223
- '--disable-backgrounding-occluded-windows',
224
- '--disable-breakpad',
225
- '--disable-client-side-phishing-detection',
226
- '--disable-component-extensions-with-background-pages',
227
- '--disable-default-apps',
228
- '--disable-dev-shm-usage',
229
- '--disable-extensions',
230
- '--disable-features=TranslateUI',
231
- '--disable-hang-monitor',
232
- '--disable-ipc-flooding-protection',
233
- '--disable-popup-blocking',
234
- '--disable-prompt-on-repost',
235
- '--disable-renderer-backgrounding',
236
- '--disable-sync',
237
- '--force-color-profile=srgb',
238
- '--metrics-recording-only',
239
- '--no-first-run',
240
- '--enable-automation',
241
- '--password-store=basic',
242
- '--use-mock-keychain',
243
- ]
244
- product = 'chrome'
245
-
246
- def __init__(self, projectRoot: str = None, preferredRevision: str = None):
247
- if not preferredRevision:
248
- preferredRevision = __chromium_revision__
249
- super().__init__(projectRoot, preferredRevision)
250
-
251
- @property
252
- def executable_path(self) -> Optional[str]:
253
- return resolveExecutablePath(self.projectRoot, self.preferredRevision)[0]
254
-
255
- async def launch(self, **kwargs: Union[LaunchOptions, ChromeArgOptions, BrowserOptions]) -> Browser:
256
- ignoreDefaultArgs = kwargs.get('ignoreDefaultArgs', False)
257
- args: Sequence[str] = kwargs.get('args', [])
258
- dumpio = kwargs.get('dumpio', False)
259
- executablePath = kwargs.get('executablePath', None)
260
- env = kwargs.get('env', os.environ)
261
- handleSIGINT = kwargs.get('handleSIGINT', True)
262
- handleSIGTERM = kwargs.get('handleSIGTERM', True)
263
- handleSIGHUP = kwargs.get('handleSIGHUP', not sys.platform.startswith('win'))
264
- ignoreHTTPSErrors = kwargs.get('ignoreHTTPSErrors', False)
265
- defaultViewport = kwargs.get('defaultViewport', {'width': 800, 'height': 600})
266
- slowMo = kwargs.get('slowMo', 0)
267
- timeout = kwargs.get('timeout', 30_000)
268
- profile_path = None
269
-
270
- chrome_args = []
271
- if not ignoreDefaultArgs:
272
- chrome_args.extend(self.default_args(**kwargs))
273
- elif isinstance(ignoreDefaultArgs, list):
274
- chrome_args.extend([x for x in self.default_args(**kwargs) if x not in ignoreDefaultArgs])
275
- else:
276
- chrome_args.extend(args)
277
-
278
- if not any(x.startswith('--remote-debugging-') for x in chrome_args):
279
- chrome_args.append(f'--remote-debugging-port={get_free_port()}')
280
- if not any(x.startswith(f'--user-data-dir') for x in chrome_args):
281
- profile_path = tempfile.TemporaryDirectory(prefix='pyppeteer_chrome_profile_')
282
- chrome_args.append(f'--user-data-dir={profile_path.name}')
283
-
284
- if not executablePath:
285
- chrome_executable, missing_text = resolveExecutablePath(self.projectRoot, self.preferredRevision)
286
- if missing_text:
287
- raise RuntimeError(missing_text)
288
- else:
289
- chrome_executable = executablePath
290
-
291
- usePipe = '--remote-debugging-pipe' in chrome_args
292
-
293
- runner = BrowserRunner(chrome_executable, chrome_args, profile_path)
294
- runner.start(
295
- handleSIGINT=handleSIGINT, handleSIGHUP=handleSIGHUP, handleSIGTERM=handleSIGTERM, env=env, dumpio=dumpio, autoClose=kwargs.get('autoClose')
296
- )
297
-
298
- try:
299
- connection = await runner.setupConnection(
300
- usePipe=usePipe, timeout=timeout, slowMo=slowMo, preferredRevision=self.preferredRevision,
301
- )
302
- browser = await Browser.create(
303
- connection=connection,
304
- contextIds=[],
305
- ignoreHTTPSErrors=ignoreHTTPSErrors,
306
- defaultViewport=defaultViewport,
307
- process=runner.proc,
308
- closeCallback=runner.close,
309
- )
310
- await browser.waitForTarget(lambda x: x.type == 'page')
311
- return browser
312
- except Exception as original_exception:
313
- logger.error(f'error occurred while trying to launch browser: {original_exception}')
314
- with suppress(Exception):
315
- runner.kill()
316
- raise original_exception
317
-
318
- def default_args(
319
- self,
320
- args: Sequence[str] = None,
321
- devtools: bool = False,
322
- headless: bool = None,
323
- userDataDir: str = None,
324
- **_: Any,
325
- ) -> List[str]:
326
- if headless is None:
327
- headless = not devtools
328
- if args is None:
329
- args = []
330
- chrome_args = self.DEFAULT_ARGS[:]
331
- if isinstance(args, Sequence):
332
- chrome_args.extend(args)
333
- if userDataDir:
334
- chrome_args.append(f'--user-data-dir={userDataDir}')
335
- if devtools:
336
- chrome_args.append('--auto-open-devtools-for-tabs')
337
- if headless:
338
- chrome_args.extend(('--headless', '--hide-scrollbars', '--mute-audio'))
339
- if all(x.startswith('-') for x in args):
340
- chrome_args.append('about:blank')
341
- return chrome_args
342
-
343
-
344
- class FirefoxLauncher(BaseBrowserLauncher):
345
- DEFAULT_ARGS = [
346
- '--remote-debugging-port=0',
347
- '--no-remote',
348
- '--foreground',
349
- ]
350
- _server = 'dummy.test'
351
- DEFAULT_PROFILE_PREFS = {
352
- # Make sure Shield doesn't hit the network.
353
- 'app.normandy.api_url': '',
354
- # Disable Firefox old build background check
355
- 'app.update.checkInstallTime': False,
356
- # Disable automatically upgrading Firefox
357
- 'app.update.disabledForTesting': True,
358
- # Increase the APZ content response timeout to 2 minute
359
- 'apz.content_response_timeout': 60000,
360
- # Prevent various error message on the console
361
- # jest-puppeteer asserts that no error message is emitted by the console
362
- 'browser.contentblocking.features.standard': '-tp,tpPrivate,cookieBehavior0,-cm,-fp',
363
- # Enable the dump function: which sends messages to the system
364
- # console
365
- # https://bugzilla.mozilla.org/show_bug.cgi?id=1543115
366
- 'browser.dom.window.dump.enabled': True,
367
- # Disable topstories
368
- 'browser.newtabpage.activity-stream.feeds.section.topstories': False,
369
- # Always display a blank page
370
- 'browser.newtabpage.enabled': False,
371
- # Background thumbnails in particular cause grief: and disabling
372
- # thumbnails in general cannot hurt
373
- 'browser.pagethumbnails.capturing_disabled': True,
374
- # Disable safebrowsing components.
375
- 'browser.safebrowsing.blockedURIs.enabled': False,
376
- 'browser.safebrowsing.downloads.enabled': False,
377
- 'browser.safebrowsing.malware.enabled': False,
378
- 'browser.safebrowsing.passwords.enabled': False,
379
- 'browser.safebrowsing.phishing.enabled': False,
380
- # Disable updates to search engines.
381
- 'browser.search.update': False,
382
- # Do not restore the last open set of tabs if the browser has crashed
383
- 'browser.sessionstore.resume_from_crash': False,
384
- # Skip check for default browser on startup
385
- 'browser.shell.checkDefaultBrowser': False,
386
- # Disable newtabpage
387
- 'browser.startup.homepage': 'about:blank',
388
- # Do not redirect user when a milstone upgrade of Firefox is detected
389
- 'browser.startup.homepage_override.mstone': 'ignore',
390
- # Start with a blank page about:blank
391
- 'browser.startup.page': 0,
392
- # Do not allow background tabs to be zombified on Android: otherwise for
393
- # tests that open additional tabs: the test harness tab itself might get
394
- # unloaded
395
- 'browser.tabs.disableBackgroundZombification': False,
396
- # Do not warn when closing all other open tabs
397
- 'browser.tabs.warnOnCloseOtherTabs': False,
398
- # Do not warn when multiple tabs will be opened
399
- 'browser.tabs.warnOnOpen': False,
400
- # Disable the UI tour.
401
- 'browser.uitour.enabled': False,
402
- # Turn off search suggestions in the location bar so as not to trigger
403
- # network connections.
404
- 'browser.urlbar.suggest.searches': False,
405
- # Disable first run splash page on Windows 10
406
- 'browser.usedOnWindows10.introURL': '',
407
- # Do not warn on quitting Firefox
408
- 'browser.warnOnQuit': False,
409
- # Do not show datareporting policy notifications which can
410
- # interfere with tests
411
- 'datareporting.healthreport.about.reportUrl': f'http://{_server}/dummy/abouthealthreport/',
412
- 'datareporting.healthreport.documentServerURI': f'http://{_server}/dummy/healthreport/',
413
- 'datareporting.healthreport.logging.consoleEnabled': False,
414
- 'datareporting.healthreport.service.enabled': False,
415
- 'datareporting.healthreport.service.firstRun': False,
416
- 'datareporting.healthreport.uploadEnabled': False,
417
- 'datareporting.policy.dataSubmissionEnabled': False,
418
- 'datareporting.policy.dataSubmissionPolicyAccepted': False,
419
- 'datareporting.policy.dataSubmissionPolicyBypassNotification': True,
420
- # DevTools JSONViewer sometimes fails to load dependencies with its require.js.
421
- # This doesn't affect Puppeteer but spams console (Bug 1424372)
422
- 'devtools.jsonview.enabled': False,
423
- # Disable popup-blocker
424
- 'dom.disable_open_during_load': False,
425
- # Enable the support for File object creation in the content process
426
- # Required for |Page.setFileInputFiles| protocol method.
427
- 'dom.file.createInChild': True,
428
- # Disable the ProcessHangMonitor
429
- 'dom.ipc.reportProcessHangs': False,
430
- # Disable slow script dialogues
431
- 'dom.max_chrome_script_run_time': 0,
432
- 'dom.max_script_run_time': 0,
433
- # Only load extensions from the application and user profile
434
- # AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION
435
- 'extensions.autoDisableScopes': 0,
436
- 'extensions.enabledScopes': 5,
437
- # Disable metadata caching for installed add-ons by default
438
- 'extensions.getAddons.cache.enabled': False,
439
- # Disable installing any distribution extensions or add-ons.
440
- 'extensions.installDistroAddons': False,
441
- # Disabled screenshots extension
442
- 'extensions.screenshots.disabled': True,
443
- # Turn off extension updates so they do not bother tests
444
- 'extensions.update.enabled': False,
445
- # Turn off extension updates so they do not bother tests
446
- 'extensions.update.notifyUser': False,
447
- # Make sure opening about:addons will not hit the network
448
- 'extensions.webservice.discoverURL': f'http://{_server}/dummy/discoveryURL',
449
- # Allow the application to have focus even it runs in the background
450
- 'focusmanager.testmode': True,
451
- # Disable useragent updates
452
- 'general.useragent.updates.enabled': False,
453
- # Always use network provider for geolocation tests so we bypass the
454
- # macOS dialog raised by the corelocation provider
455
- 'geo.provider.testing': True,
456
- # Do not scan Wifi
457
- 'geo.wifi.scan': False,
458
- # No hang monitor
459
- 'hangmonitor.timeout': 0,
460
- # Show chrome errors and warnings in the error console
461
- 'javascript.options.showInConsole': True,
462
- # Disable download and usage of OpenH264: and Widevine plugins
463
- 'media.gmp-manager.updateEnabled': False,
464
- # Prevent various error message on the console
465
- # jest-puppeteer asserts that no error message is emitted by the console
466
- 'network.cookie.cookieBehavior': 0,
467
- # Do not prompt for temporary redirects
468
- 'network.http.prompt-temp-redirect': False,
469
- # Disable speculative connections so they are not reported as leaking
470
- # when they are hanging around
471
- 'network.http.speculative-parallel-limit': 0,
472
- # Do not automatically switch between offline and online
473
- 'network.manage-offline-status': False,
474
- # Make sure SNTP requests do not hit the network
475
- 'network.sntp.pools': _server,
476
- # Disable Flash.
477
- 'plugin.state.flash': 0,
478
- 'privacy.trackingprotection.enabled': False,
479
- # Enable Remote Agent
480
- # https://bugzilla.mozilla.org/show_bug.cgi?id=1544393
481
- 'remote.enabled': True,
482
- # Don't do network connections for mitm priming
483
- 'security.certerrors.mitm.priming.enabled': False,
484
- # Local documents have access to all other local documents,
485
- # including directory listings
486
- 'security.fileuri.strict_origin_policy': False,
487
- # Do not wait for the notification button security delay
488
- 'security.notification_enable_delay': 0,
489
- # Ensure blocklist updates do not hit the network
490
- 'services.settings.server': f'http://{_server}/dummy/blocklist/',
491
- # Do not automatically fill sign-in forms with known usernames and
492
- # passwords
493
- 'signon.autofillForms': False,
494
- # Disable password capture, so that tests that include forms are not
495
- # influenced by the presence of the persistent doorhanger notification
496
- 'signon.rememberSignons': False,
497
- # Disable first-run welcome page
498
- 'startup.homepage_welcome_url': 'about:blank',
499
- # Disable first-run welcome page
500
- 'startup.homepage_welcome_url.additional': '',
501
- # Disable browser animations (tabs, fullscreen, sliding alerts)
502
- 'toolkit.cosmeticAnimations.enabled': False,
503
- # We want to collect telemetry, but we don't want to send in the results
504
- 'toolkit.telemetry.server': f'https://{_server}/dummy/telemetry/',
505
- # Prevent starting into safe mode after application crashes
506
- 'toolkit.startup.max_resumed_crashes': -1,
507
- }
508
- product = 'firefox'
509
-
510
- def __init__(self, projectRoot: str = None, preferredRevision: str = None):
511
- super().__init__(projectRoot, preferredRevision)
512
-
513
- @property
514
- def executablePath(self) -> Optional[Path]:
515
- raise NotImplementedError('executablePath method not implemented')
516
-
517
- async def launch(self, **kwargs: Union[LaunchOptions, ChromeArgOptions, BrowserOptions]) -> Browser:
518
- ignoreDefaultArgs = kwargs.get('ignoreDefaultArgs', False)
519
- args: Sequence[str] = kwargs.get('args', [])
520
- dumpio = kwargs.get('dumpio', False)
521
- executablePath = kwargs.get('executablePath', None)
522
- pipe = kwargs.get('pipe', False)
523
- env = kwargs.get('env', os.environ)
524
- handleSIGINT = kwargs.get('handleSIGINT', True)
525
- handleSIGTERM = kwargs.get('handleSIGTERM', True)
526
- handleSIGHUP = kwargs.get('handleSIGHUP', not sys.platform.startswith('win'))
527
- ignoreHTTPSErrors = kwargs.get('ignoreHTTPSErrors', False)
528
- defaultViewport = kwargs.get('defaultViewport', {'width': 800, 'height': 600})
529
- slowMo = kwargs.get('slowMo', 0)
530
- timeout = kwargs.get('timeout', 30_000)
531
- profile_path = None
532
-
533
- firefox_args = []
534
- if not ignoreDefaultArgs:
535
- firefox_args.extend(self.default_args(**kwargs))
536
- elif isinstance(ignoreDefaultArgs, Sequence):
537
- firefox_args.extend([x for x in self.default_args()])
538
- else:
539
- firefox_args.extend(args)
540
-
541
- if '-profile' not in firefox_args and '--profile' not in firefox_args:
542
- profile_path = tempfile.TemporaryDirectory('pyppyeteer2_firefox_profile_')
543
- firefox_args.extend(('--profile', profile_path.name))
544
-
545
- if not executablePath:
546
- missing_text, executablePath = resolveExecutablePath(self.projectRoot, self.preferredRevision)
547
- if missing_text:
548
- raise RuntimeError(missing_text)
549
-
550
- runner = BrowserRunner(executable_path=executablePath, process_args=firefox_args, temp_dir=profile_path)
551
- runner.start(
552
- handleSIGHUP=handleSIGHUP,
553
- handleSIGTERM=handleSIGTERM,
554
- handleSIGINT=handleSIGINT,
555
- dumpio=dumpio,
556
- env=env,
557
- pipe=pipe,
558
- )
559
-
560
- try:
561
- connection = await runner.setupConnection(
562
- usePipe=pipe, timeout=timeout, slowMo=slowMo, preferredRevision=self.preferredRevision,
563
- )
564
- browser = await Browser.create(
565
- connection=connection,
566
- contextIds=[],
567
- ignoreHTTPSErrors=ignoreHTTPSErrors,
568
- defaultViewport=defaultViewport,
569
- process=runner.proc,
570
- closeCallback=runner.close,
571
- )
572
- await browser.waitForTarget(lambda t: t.type == 'page')
573
- return browser
574
- except Exception as original_exception:
575
- logger.error(f'error occurred while trying to launch browser: {original_exception}')
576
- with suppress(Exception):
577
- runner.kill()
578
- raise original_exception
579
-
580
- def default_args(
581
- self,
582
- args: Sequence[str] = None,
583
- devtools: bool = False,
584
- headless: bool = None,
585
- userDataDir: str = None,
586
- **_: Any,
587
- ) -> List[str]:
588
- if headless is None:
589
- headless = not devtools
590
- proc_args = self.DEFAULT_ARGS[:]
591
- if isinstance(args, Sequence):
592
- proc_args.extend(args)
593
- if userDataDir:
594
- proc_args.extend(['--profile', userDataDir])
595
- if headless:
596
- proc_args.append('--headless')
597
- if devtools:
598
- proc_args.append('--devtools')
599
- if all(x.startswith('-') for x in proc_args):
600
- proc_args.append('about:blank')
601
-
602
- return proc_args
603
-
604
- def _create_profile(self, extra_prefs: Dict[str, Any]) -> tempfile.TemporaryDirectory:
605
- profile_path = tempfile.TemporaryDirectory(prefix='pyppeteer_firefox_profile')
606
- prefs = {**self.DEFAULT_PROFILE_PREFS, **extra_prefs}
607
- serialized_prefs = [f'user_pref({json.dumps(key)}, {json.dumps(val)}' for key, val in prefs.items()]
608
- with open(Path(profile_path.name).joinpath('user.js'), 'w') as user:
609
- user.write('\n'.join(serialized_prefs))
610
- return profile_path
611
-
612
-
613
- def waitForWSEndpoint(proc: subprocess.Popen, timeout: Optional[float], preferredRevision: Optional[str]) -> str:
614
- assert proc.stdout is not None, 'process STDOUT wasn\'t piped'
615
- start = time.perf_counter()
616
- buffer = ''
617
- for line in iter(proc.stdout.readline, b''):
618
- line = line.decode()
619
- buffer += '\n' + line
620
- if timeout and (start - time.perf_counter()) > timeout:
621
- raise TimeoutError(
622
- f'Timed out after {timeout * 1000:.0f}ms while trying to connect to the browser! '
623
- f'Only Chrome at revision {preferredRevision} is guaranteed to work.'
624
- )
625
- potential_match = re.match(r'DevTools listening on (ws://[\w.:/-]*)+', line)
626
- if potential_match:
627
- return potential_match.group(1)
628
- raise RuntimeError(
629
- buffer + '\nProcess ended before WebSockets endpoint could be found.'
630
- f'Only Chrome at revision {preferredRevision} is guaranteed to work.'
631
- )
632
-
633
-
634
- def getWSEndpoint(url: str) -> str:
635
- url += '/json/version'
636
- timeout = time.perf_counter() + 30
637
- while True:
638
- if time.perf_counter() > timeout:
639
- raise BrowserError('Browser closed unexpectedly:\n')
640
- try:
641
- with urlopen(url) as f:
642
- data = json.loads(f.read().decode())
643
- break
644
- except URLError:
645
- time.sleep(0.1)
646
- try:
647
- return data['webSocketDebuggerUrl']
648
- except KeyError:
649
- raise RuntimeError(f'webSocketDebuggerUrl not found')
650
-
651
-
652
- def resolveExecutablePath(projectRoot: Path, preferred_revision: str) -> Tuple[Optional[Path], Optional[str]]:
653
- missing_text = None
654
- exec_path_env_var = 'PYPPETEER_EXECUTABLE_PATH'
655
- revision_env_var = 'PYPPETEER_CHROMIUM_REVISION'
656
- executable = os.environ.get(exec_path_env_var)
657
- if executable:
658
- if not Path(executable).is_file():
659
- missing_text = f'Tried to use env variable ({exec_path_env_var}) to launch browser, but no executable was found at {executable}'
660
- return None, missing_text
661
- browser_fetcher = BrowserFetcher(projectRoot)
662
- revision = os.environ.get(revision_env_var)
663
- if revision:
664
- revision_info = browser_fetcher.revision_info(revision)
665
- if not revision_info['local']:
666
- missing_text = f'Tried to use env variables ({revision_env_var}) to launch browser, but did not find executable at {revision_info["executablePath"]}'
667
- return None, missing_text
668
- revision_info = browser_fetcher.revision_info(preferred_revision)
669
- if not revision_info['local']:
670
- missing_text = 'Browser is not downloaded. Try running pyppeteer-install'
671
- return revision_info['executablePath'], missing_text
672
-
673
-
674
- def launcher(
675
- projectRoot: str = None, preferredRevision: str = None, product: str = None
676
- ) -> Union[FirefoxLauncher, ChromeLauncher]:
677
- """Returns the appropriate browser launcher class instance"""
678
- product = product or os.environ.get('PYPPETEER_PRODUCT') or 'chrome'
679
- if product == 'firefox':
680
- return FirefoxLauncher(projectRoot, preferredRevision)
681
- elif product in ('chrome', 'chromium'):
682
- return ChromeLauncher(projectRoot, preferredRevision)
683
- raise ValueError(f'Support for {product} has not been implemented')