pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. biolib/__init__.py +357 -11
  2. biolib/_data_record/data_record.py +380 -0
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +55 -0
  5. biolib/_index/query_result.py +103 -0
  6. biolib/_internal/__init__.py +0 -0
  7. biolib/_internal/add_copilot_prompts.py +58 -0
  8. biolib/_internal/add_gui_files.py +81 -0
  9. biolib/_internal/data_record/__init__.py +1 -0
  10. biolib/_internal/data_record/data_record.py +85 -0
  11. biolib/_internal/data_record/push_data.py +116 -0
  12. biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
  13. biolib/_internal/errors.py +5 -0
  14. biolib/_internal/file_utils.py +125 -0
  15. biolib/_internal/fuse_mount/__init__.py +1 -0
  16. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  17. biolib/_internal/http_client.py +159 -0
  18. biolib/_internal/lfs/__init__.py +1 -0
  19. biolib/_internal/lfs/cache.py +51 -0
  20. biolib/_internal/libs/__init__.py +1 -0
  21. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  22. biolib/_internal/push_application.py +488 -0
  23. biolib/_internal/runtime.py +22 -0
  24. biolib/_internal/string_utils.py +13 -0
  25. biolib/_internal/templates/__init__.py +1 -0
  26. biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
  27. biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
  28. biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
  29. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  30. biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
  31. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  32. biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
  33. biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
  34. biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
  35. biolib/_internal/templates/gitignore_template/.gitignore +10 -0
  36. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  37. biolib/_internal/templates/gui_template/App.tsx +53 -0
  38. biolib/_internal/templates/gui_template/Dockerfile +27 -0
  39. biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
  40. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  41. biolib/_internal/templates/gui_template/index.css +5 -0
  42. biolib/_internal/templates/gui_template/index.html +13 -0
  43. biolib/_internal/templates/gui_template/index.tsx +10 -0
  44. biolib/_internal/templates/gui_template/package.json +27 -0
  45. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  46. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
  47. biolib/_internal/templates/gui_template/vite.config.mts +10 -0
  48. biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
  49. biolib/_internal/templates/init_template/Dockerfile +14 -0
  50. biolib/_internal/templates/init_template/requirements.txt +1 -0
  51. biolib/_internal/templates/init_template/run.py +12 -0
  52. biolib/_internal/templates/init_template/run.sh +4 -0
  53. biolib/_internal/templates/templates.py +25 -0
  54. biolib/_internal/tree_utils.py +106 -0
  55. biolib/_internal/utils/__init__.py +65 -0
  56. biolib/_internal/utils/auth.py +46 -0
  57. biolib/_internal/utils/job_url.py +33 -0
  58. biolib/_internal/utils/multinode.py +263 -0
  59. biolib/_runtime/runtime.py +157 -0
  60. biolib/_session/session.py +44 -0
  61. biolib/_shared/__init__.py +0 -0
  62. biolib/_shared/types/__init__.py +74 -0
  63. biolib/_shared/types/account.py +12 -0
  64. biolib/_shared/types/account_member.py +8 -0
  65. biolib/_shared/types/app.py +9 -0
  66. biolib/_shared/types/data_record.py +40 -0
  67. biolib/_shared/types/experiment.py +32 -0
  68. biolib/_shared/types/file_node.py +17 -0
  69. biolib/_shared/types/push.py +6 -0
  70. biolib/_shared/types/resource.py +37 -0
  71. biolib/_shared/types/resource_deploy_key.py +11 -0
  72. biolib/_shared/types/resource_permission.py +14 -0
  73. biolib/_shared/types/resource_version.py +19 -0
  74. biolib/_shared/types/result.py +14 -0
  75. biolib/_shared/types/typing.py +10 -0
  76. biolib/_shared/types/user.py +19 -0
  77. biolib/_shared/utils/__init__.py +7 -0
  78. biolib/_shared/utils/resource_uri.py +75 -0
  79. biolib/api/__init__.py +6 -0
  80. biolib/api/client.py +168 -0
  81. biolib/app/app.py +252 -49
  82. biolib/app/search_apps.py +45 -0
  83. biolib/biolib_api_client/api_client.py +126 -31
  84. biolib/biolib_api_client/app_types.py +24 -4
  85. biolib/biolib_api_client/auth.py +31 -8
  86. biolib/biolib_api_client/biolib_app_api.py +147 -52
  87. biolib/biolib_api_client/biolib_job_api.py +161 -141
  88. biolib/biolib_api_client/job_types.py +21 -5
  89. biolib/biolib_api_client/lfs_types.py +7 -23
  90. biolib/biolib_api_client/user_state.py +56 -0
  91. biolib/biolib_binary_format/__init__.py +1 -4
  92. biolib/biolib_binary_format/file_in_container.py +105 -0
  93. biolib/biolib_binary_format/module_input.py +24 -7
  94. biolib/biolib_binary_format/module_output_v2.py +149 -0
  95. biolib/biolib_binary_format/remote_endpoints.py +34 -0
  96. biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
  97. biolib/biolib_binary_format/saved_job.py +3 -2
  98. biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
  99. biolib/biolib_binary_format/system_status_update.py +3 -2
  100. biolib/biolib_binary_format/utils.py +175 -0
  101. biolib/biolib_docker_client/__init__.py +11 -2
  102. biolib/biolib_errors.py +36 -0
  103. biolib/biolib_logging.py +27 -10
  104. biolib/cli/__init__.py +38 -0
  105. biolib/cli/auth.py +46 -0
  106. biolib/cli/data_record.py +164 -0
  107. biolib/cli/index.py +32 -0
  108. biolib/cli/init.py +421 -0
  109. biolib/cli/lfs.py +101 -0
  110. biolib/cli/push.py +50 -0
  111. biolib/cli/run.py +63 -0
  112. biolib/cli/runtime.py +14 -0
  113. biolib/cli/sdk.py +16 -0
  114. biolib/cli/start.py +56 -0
  115. biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
  116. biolib/compute_node/job_worker/cache_state.py +66 -88
  117. biolib/compute_node/job_worker/cache_types.py +1 -6
  118. biolib/compute_node/job_worker/docker_image_cache.py +112 -37
  119. biolib/compute_node/job_worker/executors/__init__.py +0 -3
  120. biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
  121. biolib/compute_node/job_worker/executors/docker_types.py +9 -1
  122. biolib/compute_node/job_worker/executors/types.py +19 -9
  123. biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
  124. biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
  125. biolib/compute_node/job_worker/job_storage.py +108 -0
  126. biolib/compute_node/job_worker/job_worker.py +397 -212
  127. biolib/compute_node/job_worker/large_file_system.py +87 -38
  128. biolib/compute_node/job_worker/network_alloc.py +99 -0
  129. biolib/compute_node/job_worker/network_buffer.py +240 -0
  130. biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
  131. biolib/compute_node/job_worker/utils.py +9 -24
  132. biolib/compute_node/remote_host_proxy.py +400 -98
  133. biolib/compute_node/utils.py +31 -9
  134. biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
  135. biolib/compute_node/webserver/proxy_utils.py +28 -0
  136. biolib/compute_node/webserver/webserver.py +130 -44
  137. biolib/compute_node/webserver/webserver_types.py +2 -6
  138. biolib/compute_node/webserver/webserver_utils.py +77 -12
  139. biolib/compute_node/webserver/worker_thread.py +183 -42
  140. biolib/experiments/__init__.py +0 -0
  141. biolib/experiments/experiment.py +356 -0
  142. biolib/jobs/__init__.py +1 -0
  143. biolib/jobs/job.py +741 -0
  144. biolib/jobs/job_result.py +185 -0
  145. biolib/jobs/types.py +50 -0
  146. biolib/py.typed +0 -0
  147. biolib/runtime/__init__.py +14 -0
  148. biolib/sdk/__init__.py +91 -0
  149. biolib/tables.py +34 -0
  150. biolib/typing_utils.py +2 -7
  151. biolib/user/__init__.py +1 -0
  152. biolib/user/sign_in.py +54 -0
  153. biolib/utils/__init__.py +162 -0
  154. biolib/utils/cache_state.py +94 -0
  155. biolib/utils/multipart_uploader.py +194 -0
  156. biolib/utils/seq_util.py +150 -0
  157. biolib/utils/zip/remote_zip.py +640 -0
  158. pybiolib-1.2.1890.dist-info/METADATA +41 -0
  159. pybiolib-1.2.1890.dist-info/RECORD +177 -0
  160. {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
  161. pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
  162. README.md +0 -17
  163. biolib/app/app_result.py +0 -68
  164. biolib/app/utils.py +0 -62
  165. biolib/biolib-js/0-biolib.worker.js +0 -1
  166. biolib/biolib-js/1-biolib.worker.js +0 -1
  167. biolib/biolib-js/2-biolib.worker.js +0 -1
  168. biolib/biolib-js/3-biolib.worker.js +0 -1
  169. biolib/biolib-js/4-biolib.worker.js +0 -1
  170. biolib/biolib-js/5-biolib.worker.js +0 -1
  171. biolib/biolib-js/6-biolib.worker.js +0 -1
  172. biolib/biolib-js/index.html +0 -10
  173. biolib/biolib-js/main-biolib.js +0 -1
  174. biolib/biolib_api_client/biolib_account_api.py +0 -21
  175. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
  176. biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
  177. biolib/biolib_binary_format/module_output.py +0 -58
  178. biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
  179. biolib/biolib_push.py +0 -114
  180. biolib/cli.py +0 -203
  181. biolib/cli_utils.py +0 -273
  182. biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
  183. biolib/compute_node/enclave/__init__.py +0 -2
  184. biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
  185. biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
  186. biolib/compute_node/job_worker/executors/base_executor.py +0 -18
  187. biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
  188. biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
  189. biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
  190. biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
  191. biolib/lfs.py +0 -196
  192. biolib/pyppeteer/.circleci/config.yml +0 -100
  193. biolib/pyppeteer/.coveragerc +0 -3
  194. biolib/pyppeteer/.gitignore +0 -89
  195. biolib/pyppeteer/.pre-commit-config.yaml +0 -28
  196. biolib/pyppeteer/CHANGES.md +0 -253
  197. biolib/pyppeteer/CONTRIBUTING.md +0 -26
  198. biolib/pyppeteer/LICENSE +0 -12
  199. biolib/pyppeteer/README.md +0 -137
  200. biolib/pyppeteer/docs/Makefile +0 -177
  201. biolib/pyppeteer/docs/_static/custom.css +0 -28
  202. biolib/pyppeteer/docs/_templates/layout.html +0 -10
  203. biolib/pyppeteer/docs/changes.md +0 -1
  204. biolib/pyppeteer/docs/conf.py +0 -299
  205. biolib/pyppeteer/docs/index.md +0 -21
  206. biolib/pyppeteer/docs/make.bat +0 -242
  207. biolib/pyppeteer/docs/reference.md +0 -211
  208. biolib/pyppeteer/docs/server.py +0 -60
  209. biolib/pyppeteer/poetry.lock +0 -1699
  210. biolib/pyppeteer/pyppeteer/__init__.py +0 -135
  211. biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
  212. biolib/pyppeteer/pyppeteer/browser.py +0 -401
  213. biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
  214. biolib/pyppeteer/pyppeteer/command.py +0 -22
  215. biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
  216. biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
  217. biolib/pyppeteer/pyppeteer/coverage.py +0 -346
  218. biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
  219. biolib/pyppeteer/pyppeteer/dialog.py +0 -79
  220. biolib/pyppeteer/pyppeteer/domworld.py +0 -597
  221. biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
  222. biolib/pyppeteer/pyppeteer/errors.py +0 -48
  223. biolib/pyppeteer/pyppeteer/events.py +0 -63
  224. biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
  225. biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
  226. biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
  227. biolib/pyppeteer/pyppeteer/helpers.py +0 -245
  228. biolib/pyppeteer/pyppeteer/input.py +0 -371
  229. biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
  230. biolib/pyppeteer/pyppeteer/launcher.py +0 -683
  231. biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
  232. biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
  233. biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
  234. biolib/pyppeteer/pyppeteer/multimap.py +0 -82
  235. biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
  236. biolib/pyppeteer/pyppeteer/options.py +0 -8
  237. biolib/pyppeteer/pyppeteer/page.py +0 -1728
  238. biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
  239. biolib/pyppeteer/pyppeteer/target.py +0 -147
  240. biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
  241. biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
  242. biolib/pyppeteer/pyppeteer/tracing.py +0 -93
  243. biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
  244. biolib/pyppeteer/pyppeteer/util.py +0 -18
  245. biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
  246. biolib/pyppeteer/pyppeteer/worker.py +0 -101
  247. biolib/pyppeteer/pyproject.toml +0 -97
  248. biolib/pyppeteer/spell.txt +0 -137
  249. biolib/pyppeteer/tox.ini +0 -72
  250. biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
  251. biolib/start_cli.py +0 -7
  252. biolib/utils.py +0 -47
  253. biolib/validators/validate_app_version.py +0 -183
  254. biolib/validators/validate_argument.py +0 -134
  255. biolib/validators/validate_module.py +0 -323
  256. biolib/validators/validate_zip_file.py +0 -40
  257. biolib/validators/validator_utils.py +0 -103
  258. pybiolib-0.2.951.dist-info/LICENSE +0 -21
  259. pybiolib-0.2.951.dist-info/METADATA +0 -61
  260. pybiolib-0.2.951.dist-info/RECORD +0 -153
  261. pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
  262. /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
@@ -1,678 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
-
4
- """Network Manager module."""
5
-
6
- import asyncio
7
- import base64
8
- import json
9
- import logging
10
- from http import HTTPStatus
11
- from typing import TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set, Union
12
-
13
- from pyee import AsyncIOEventEmitter
14
- from biolib.pyppeteer.pyppeteer.connection import CDPSession
15
- from biolib.pyppeteer.pyppeteer.errors import NetworkError
16
- from biolib.pyppeteer.pyppeteer.events import Events
17
-
18
- if TYPE_CHECKING:
19
- from pyppeteer.frame import Frame, FrameManager
20
-
21
- logger = logging.getLogger(__name__)
22
-
23
-
24
- class NetworkManager(AsyncIOEventEmitter):
25
- """NetworkManager class."""
26
-
27
- def __init__(self, client: CDPSession, ignoreHttpsErrors: bool, frameManager: 'FrameManager') -> None:
28
- """Make new NetworkManager."""
29
- super().__init__()
30
- self._client = client
31
- self._ignoreHTTPSErrors = ignoreHttpsErrors
32
- self._frameManager = frameManager
33
- self._requestIdToRequest: Dict[Optional[str], Request] = {}
34
- self._requestIdToRequestWillBeSent: Dict[Optional[str], Dict] = {}
35
- self._extraHTTPHeaders: Dict[str, str] = {}
36
- self._offline: bool = False
37
- self._credentials: Optional[Dict[str, str]] = None
38
- self._attemptedAuthentications: Set[Optional[str]] = set()
39
- self._userRequestInterceptionEnabled = False
40
- self._protocolRequestInterceptionEnabled = False
41
- self._userCacheDisabled = False
42
- self._requestIdToInterceptionId: Dict[str, str] = {}
43
-
44
- self._client.on('Fetch.requestPaused', self._onRequestPaused)
45
- self._client.on('Fetch.authRequired', self._onAuthRequired)
46
- self._client.on('Network.requestWillBeSent', self._onRequestWillBeSent)
47
- self._client.on('Network.requestServedFromCache', self._onRequestServedFromCache)
48
- self._client.on('Network.responseReceived', self._onResponseReceived)
49
- self._client.on('Network.loadingFinished', self._onLoadingFinished)
50
- self._client.on('Network.loadingFailed', self._onLoadingFailed)
51
-
52
- async def initialize(self) -> None:
53
- await self._client.send('Network.enable')
54
- if self._ignoreHTTPSErrors:
55
- await self._client.send('Security.setIgnoreCertificateErrors', {'ignore': True})
56
-
57
- async def authenticate(self, credentials: Dict[str, str]) -> None:
58
- """Provide credentials for http auth."""
59
- self._credentials = credentials
60
- await self._updateProtocolRequestInterception()
61
-
62
- async def setExtraHTTPHeaders(self, extraHTTPHeaders: Dict[str, str]) -> None:
63
- """Set extra http headers."""
64
- self._extraHTTPHeaders = {}
65
- for k, v in extraHTTPHeaders.items():
66
- if not isinstance(v, str):
67
- raise TypeError(f'Expected value of header "{k}" to be string, but {type(v)} is found.')
68
- self._extraHTTPHeaders[k.lower()] = v
69
- await self._client.send('Network.setExtraHTTPHeaders', {'headers': self._extraHTTPHeaders})
70
-
71
- def extraHTTPHeaders(self) -> Dict[str, str]:
72
- """Get extra http headers."""
73
- return dict(**self._extraHTTPHeaders)
74
-
75
- async def setOfflineMode(self, value: bool) -> None:
76
- """Change offline mode enable/disable."""
77
- if self._offline == value:
78
- return
79
- self._offline = value
80
- await self._client.send(
81
- 'Network.emulateNetworkConditions',
82
- {'offline': self._offline, 'latency': 0, 'downloadThroughput': -1, 'uploadThroughput': -1,},
83
- )
84
-
85
- async def setUserAgent(self, userAgent: str) -> None:
86
- """Set user agent."""
87
- await self._client.send('Network.setUserAgentOverride', {'userAgent': userAgent})
88
-
89
- async def setCacheEnabled(self, enabled: bool) -> None:
90
- self._userCacheDisabled = not enabled
91
- await self._updateProtocolCacheDisabled()
92
-
93
- async def setRequestInterception(self, value: bool) -> None:
94
- """Enable request interception."""
95
- self._userRequestInterceptionEnabled = value
96
- await self._updateProtocolRequestInterception()
97
-
98
- async def _updateProtocolRequestInterception(self) -> None:
99
- enabled = self._userRequestInterceptionEnabled or bool(self._credentials)
100
- if enabled == self._protocolRequestInterceptionEnabled:
101
- return
102
- self._protocolRequestInterceptionEnabled = enabled
103
- if enabled:
104
- await asyncio.gather(
105
- self._updateProtocolCacheDisabled(),
106
- self._client.send('Fetch.enable', {'handleAuthRequests': True, 'patterns': [{'urlPattern': '*'}]}),
107
- )
108
- else:
109
- await asyncio.gather(self._updateProtocolCacheDisabled(), self._client.send('Fetch.disable'))
110
-
111
- async def _updateProtocolCacheDisabled(self) -> None:
112
- await self._client.send(
113
- 'Network.setCacheDisabled',
114
- {'cacheDisabled': self._userCacheDisabled or self._protocolRequestInterceptionEnabled},
115
- )
116
-
117
- async def _onRequestWillBeSent(self, event: Dict) -> None:
118
- is_data_request = event.get('request', {}).get('url', '').startswith('data:')
119
- if self._protocolRequestInterceptionEnabled and not is_data_request:
120
- requestId = event['requestId']
121
- interceptionId = self._requestIdToInterceptionId.get(requestId)
122
- if interceptionId:
123
- self._onRequest(event, interceptionId)
124
- self._requestIdToInterceptionId.pop(requestId)
125
- else:
126
- self._requestIdToRequestWillBeSent[requestId] = event
127
- else:
128
- self._onRequest(event, None)
129
-
130
- async def _onAuthRequired(self, event: Dict) -> None:
131
- response = 'Default'
132
- requestId = event.get('requestId')
133
- if requestId in self._attemptedAuthentications:
134
- response = 'CancelAuth'
135
- elif self._credentials:
136
- response = 'ProvideCredentials'
137
- self._attemptedAuthentications.add(requestId)
138
- if self._credentials:
139
- username = self._credentials.get('username')
140
- password = self._credentials.get('password')
141
- else:
142
- username, password = None, None
143
- await self._client.send(
144
- 'Fetch.continueWithAuth',
145
- {
146
- 'requestId': requestId,
147
- 'authChallengeResponse': {"response": response, "username": username, "password": password,},
148
- },
149
- )
150
-
151
- async def _onRequestPaused(self, event: Dict) -> None:
152
- if not self._userRequestInterceptionEnabled and self._protocolRequestInterceptionEnabled:
153
- await self._client.send('Fetch.continueRequest', {'requestId': event.get('requestId')})
154
- requestId = event['networkId']
155
- interceptionId = event['requestId']
156
- if requestId in self._requestIdToRequestWillBeSent:
157
- requestWillBeSentEvent = self._requestIdToRequestWillBeSent.pop(requestId)
158
- self._onRequest(requestWillBeSentEvent, interceptionId)
159
- else:
160
- self._requestIdToInterceptionId[requestId] = interceptionId
161
-
162
- def _onRequest(self, event: Dict, interceptionId: Optional[str]) -> None:
163
- redirectChain: List[Request] = []
164
- if event.get('redirectResponse'):
165
- request = self._requestIdToRequest.get(event['requestId'])
166
- if request:
167
- self._handleRequestRedirect(request, event['redirectResponse'])
168
- redirectChain = request._redirectChain
169
-
170
- frame = self._frameManager.frame(event.get('frameId'))
171
- request = Request(
172
- client=self._client,
173
- frame=frame,
174
- interceptionId=interceptionId,
175
- allowInterception=self._userRequestInterceptionEnabled,
176
- event=event,
177
- redirectChain=redirectChain,
178
- )
179
- self._requestIdToRequest[event['requestId']] = request
180
- self.emit(Events.NetworkManager.Request, request)
181
-
182
- def _onRequestServedFromCache(self, event: Dict) -> None:
183
- request = self._requestIdToRequest.get(event.get('requestId'))
184
- if request:
185
- request._fromMemoryCache = True
186
-
187
- def _handleRequestRedirect(self, request: 'Request', responsePayload: Dict[str, Any]) -> None:
188
- response = Response(client=self._client, request=request, responsePayload=responsePayload)
189
- request._response = response
190
- request._redirectChain.append(request)
191
- response._bodyLoadedFutureFulFill(NetworkError('Response body is unavailable for redirect response'))
192
- self._requestIdToRequest.pop(request._requestId, None)
193
- self._attemptedAuthentications.discard(request._interceptionId)
194
- self.emit(Events.NetworkManager.Response, response)
195
- self.emit(Events.NetworkManager.RequestFinished, request)
196
-
197
- def _onResponseReceived(self, event: dict) -> None:
198
- request = self._requestIdToRequest.get(event['requestId'])
199
- # FileUpload sends a response without a matching request.
200
- if not request:
201
- return
202
- response = Response(self._client, request, event['response'])
203
- request._response = response
204
- self.emit(Events.NetworkManager.Response, response)
205
-
206
- def _onLoadingFinished(self, event: dict) -> None:
207
- request = self._requestIdToRequest.get(event['requestId'])
208
- # For certain requestIds we never receive requestWillBeSent event.
209
- # @see https://crbug.com/750469
210
- if not request:
211
- return
212
- response = request.response
213
- if response:
214
- response._bodyLoadedFutureFulFill(None)
215
- self._requestIdToRequest.pop(request._requestId, None)
216
- self._attemptedAuthentications.discard(request._interceptionId)
217
- self.emit(Events.NetworkManager.RequestFinished, request)
218
-
219
- def _onLoadingFailed(self, event: dict) -> None:
220
- request = self._requestIdToRequest.get(event['requestId'])
221
- # For certain requestIds we never receive requestWillBeSent event.
222
- # @see https://crbug.com/750469
223
- if not request:
224
- return
225
- request._failureText = event.get('errorText')
226
- response = request.response
227
- if response:
228
- response._bodyLoadedFutureFulFill(None)
229
- self._requestIdToRequest.pop(request._requestId, None)
230
- self._attemptedAuthentications.discard(request._interceptionId)
231
- self.emit(Events.NetworkManager.RequestFailed, request)
232
-
233
-
234
- class Request:
235
- """Request class.
236
-
237
- Whenever the page sends a request, such as for a network resource, the
238
- following events are emitted by pyppeteer's page:
239
-
240
- - ``'request'``: emitted when the request is issued by the page.
241
- - ``'response'``: emitted when/if the response is received for the request.
242
- - ``'requestfinished'``: emitted when the response body is downloaded and
243
- the request is complete.
244
-
245
- If request fails at some point, then instead of ``'requestfinished'`` event
246
- (and possibly instead of ``'response'`` event), the ``'requestfailed'``
247
- event is emitted.
248
-
249
- If request gets a ``'redirect'`` response, the request is successfully
250
- finished with the ``'requestfinished'`` event, and a new request is issued
251
- to a redirect url.
252
- """
253
-
254
- _errorReasons = {
255
- 'aborted': 'Aborted',
256
- 'accessdenied': 'AccessDenied',
257
- 'addressunreachable': 'AddressUnreachable',
258
- 'blockedbyclient': 'BlockedByClient',
259
- 'blockedbyresponse': 'BlockedByResponse',
260
- 'connectionaborted': 'ConnectionAborted',
261
- 'connectionclosed': 'ConnectionClosed',
262
- 'connectionfailed': 'ConnectionFailed',
263
- 'connectionrefused': 'ConnectionRefused',
264
- 'connectionreset': 'ConnectionReset',
265
- 'internetdisconnected': 'InternetDisconnected',
266
- 'namenotresolved': 'NameNotResolved',
267
- 'timedout': 'TimedOut',
268
- 'failed': 'Failed',
269
- }
270
-
271
- def __init__(
272
- self,
273
- client: CDPSession,
274
- frame: Optional['Frame'],
275
- interceptionId: Optional[str],
276
- allowInterception: bool,
277
- event: Dict[str, Any],
278
- redirectChain: List['Request'],
279
- ):
280
- self._client = client
281
- self._requestId = event.get('requestId')
282
- self._isNavigationRequest = self._requestId == event.get('loaderId') and event['type'] == 'Document'
283
- self._interceptionId = interceptionId
284
- self._allowInterception = allowInterception
285
- self._interceptionHandled = False
286
- self._response = None
287
- self._failureText = None
288
-
289
- request_event = event['request']
290
- self._url = request_event['url']
291
- self._resourceType = event.get('type').lower() if isinstance(event.get('type'), str) else None
292
- self._method = request_event['method']
293
- self._postData = request_event.get('postData')
294
- self._frame = frame
295
- self._redirectChain = redirectChain
296
- self._headers = {k.lower(): v for k, v in request_event.get('headers', {}).items()}
297
-
298
- self._fromMemoryCache = False
299
-
300
- @property
301
- def url(self) -> str:
302
- return self._url
303
-
304
- @property
305
- def resourceType(self) -> str:
306
- """Resource type of this request perceived by the rendering engine.
307
-
308
- ResourceType will be one of the following: ``document``,
309
- ``stylesheet``, ``image``, ``media``, ``font``, ``script``,
310
- ``texttrack``, ``xhr``, ``fetch``, ``eventsource``, ``websocket``,
311
- ``manifest``, ``other``.
312
- """
313
- return self._resourceType
314
-
315
- @property
316
- def method(self) -> Optional[str]:
317
- """Return this request's method (GET, POST, etc.)."""
318
- return self._method
319
-
320
- @property
321
- def postData(self) -> Optional[str]:
322
- """Return post body of this request."""
323
- return self._postData
324
-
325
- @property
326
- def headers(self) -> Dict[str, str]:
327
- """Return a dictionary of HTTP headers of this request.
328
-
329
- All header names are lower-case.
330
- """
331
- return self._headers
332
-
333
- @property
334
- def response(self) -> Optional['Response']:
335
- """Return matching :class:`Response` object, or ``None``.
336
-
337
- If the response has not been received, return ``None``.
338
- """
339
- return self._response
340
-
341
- @property
342
- def frame(self) -> Optional['Frame']:
343
- """Return a matching :class:`~pyppeteer.frame_manager.frame` object.
344
-
345
- Return ``None`` if navigating to error page.
346
- """
347
- return self._frame
348
-
349
- @property
350
- def isNavigationRequest(self) -> bool:
351
- """Whether this request is driving frame's navigation."""
352
- return self._isNavigationRequest
353
-
354
- @property
355
- def redirectChain(self) -> List['Request']:
356
- """Return chain of requests initiated to fetch a resource.
357
-
358
- * If there are no redirects and request was successful, the chain will
359
- be empty.
360
- * If a server responds with at least a single redirect, then the chain
361
- will contain all the requests that were redirected.
362
-
363
- ``redirectChain`` is shared between all the requests of the same chain.
364
- """
365
- return self._redirectChain.copy()
366
-
367
- @property
368
- def failure(self) -> Optional[Dict[str, str]]:
369
- """Return error text.
370
-
371
- Return ``None`` unless this request was failed, as reported by
372
- ``requestfailed`` event.
373
-
374
- When request failed, this method return dictionary which has a
375
- ``errorText`` field, which contains human-readable error message, e.g.
376
- ``'net::ERR_RAILED'``.
377
- """
378
- return None if not self._failureText else {'errorText': self._failureText}
379
-
380
- async def continue_(
381
- self, url: str = None, method: str = None, postData: str = None, headers: Dict[str, str] = None
382
- ) -> None:
383
- """Continue request with optional request overrides.
384
-
385
- To use this method, request interception should be enabled by
386
- :meth:`pyppeteer.page.Page.setRequestInterception`. If request
387
- interception is not enabled, raise ``NetworkError``.
388
-
389
- ``overrides`` can have the following fields:
390
-
391
- * ``url`` (str): If set, the request url will be changed.
392
- * ``method`` (str): If set, change the request method (e.g. ``GET``).
393
- * ``postData`` (str): If set, change the post data or request.
394
- * ``headers`` (dict): If set, change the request HTTP header.
395
- """
396
- if self._is_actionable_request:
397
- if headers:
398
- headers = headersArray(headers)
399
-
400
- self._interceptionHandled = True
401
- try:
402
- await self._client.send(
403
- 'Fetch.continueRequest',
404
- {
405
- 'requestId': self._interceptionId,
406
- 'url': url,
407
- 'method': method,
408
- 'postData': postData,
409
- 'headers': headers,
410
- },
411
- )
412
- except Exception as e:
413
- # In certain cases, protocol will return error if the request was already canceled
414
- # or the page was closed. We should tolerate these errors.
415
- # (logger.exception will pick up the stack trace for us)
416
- logger.exception(f'An exception occurred while trying to continue the request')
417
-
418
- async def respond(
419
- self, status: int = 200, headers: Dict[str, str] = None, contentType: str = None, body: Union[bytes, str] = None
420
- ) -> None:
421
- """Fulfills request with given response.
422
-
423
- To use this, request interception should by enabled by
424
- :meth:`pyppeteer.page.Page.setRequestInterception`. Request
425
- interception is not enabled, raise ``NetworkError``.
426
-
427
- * ``status`` (int): Response status code, defaults to 200.
428
- * ``headers`` (dict): Optional response headers.
429
- * ``contentType`` (str): If set, equals to setting ``Content-Type``
430
- response header.
431
- * ``body`` (str|bytes): Optional response body.
432
- """
433
- if self._is_actionable_request:
434
- self._interceptionHandled = True
435
-
436
- if isinstance(body, str):
437
- responseBody = body.encode('utf-8')
438
- else:
439
- responseBody = body
440
-
441
- responseHeaders = {k.lower(): v for k, v in (headers or {}).items()}
442
- if contentType:
443
- responseHeaders['content-type'] = contentType
444
- if responseBody and 'content-length' not in responseHeaders:
445
- responseHeaders['content-length'] = str(len(responseBody))
446
- try:
447
- await self._client.send(
448
- 'Fetch.fulfillRequest',
449
- {
450
- 'requestId': self._interceptionId,
451
- 'responseCode': status,
452
- 'responsePhrase': STATUS_TEXTS[status],
453
- 'responseHeaders': headersArray(responseHeaders),
454
- 'body': base64.b64encode(responseBody).decode('ascii') if responseBody else None,
455
- },
456
- )
457
- except Exception as e:
458
- # In certain cases, protocol will return error if the request was already canceled
459
- # or the page was closed. We should tolerate these errors.
460
- logger.error(f'An exception occurred fulfilling a response: {e}')
461
-
462
- async def abort(self, errorCode: str = 'failed') -> None:
463
- """Abort request.
464
-
465
- To use this, request interception should be enabled by
466
- :meth:`pyppeteer.page.Page.setRequestInterception`.
467
- If request interception is not enabled, raise ``NetworkError``.
468
-
469
- ``errorCode`` is an optional error code string. Defaults to ``failed``,
470
- could be one of the following:
471
-
472
- - ``aborted``: An operation was aborted (due to user action).
473
- - ``accessdenied``: Permission to access a resource, other than the
474
- network, was denied.
475
- - ``addressunreachable``: The IP address is unreachable. This usually
476
- means that there is no route to the specified host or network.
477
- - ``blockedbyclient``: The client chose to block the request.
478
- - ``blockedbyresponse``: The request failed because the request was
479
- delivered along with requirements which are not met
480
- ('X-Frame-Options' and 'Content-Security-Policy' ancestor check,
481
- for instance).
482
- - ``connectionaborted``: A connection timeout as a result of not
483
- receiving an ACK for data sent.
484
- - ``connectionclosed``: A connection was closed (corresponding to a TCP
485
- FIN).
486
- - ``connectionfailed``: A connection attempt failed.
487
- - ``connectionrefused``: A connection attempt was refused.
488
- - ``connectionreset``: A connection was reset (corresponding to a TCP
489
- RST).
490
- - ``internetdisconnected``: The Internet connection has been lost.
491
- - ``namenotresolved``: The host name could not be resolved.
492
- - ``timedout``: An operation timed out.
493
- - ``failed``: A generic failure occurred.
494
- """
495
- if self._is_actionable_request:
496
- errorReason = self._errorReasons.get(errorCode)
497
- if not errorReason:
498
- raise NetworkError(f'Unknown error code: {errorCode}')
499
-
500
- self._interceptionHandled = True
501
- try:
502
- await self._client.send(
503
- 'Fetch.failRequest', {'requestId': self._interceptionId, 'errorReason': errorReason}
504
- )
505
- except Exception as e:
506
- # In certain cases, protocol will return error if the request was already canceled
507
- # or the page was closed. We should tolerate these errors.
508
- logger.error(f'An exception occurred: {e}')
509
-
510
- @property
511
- def _is_actionable_request(self) -> bool:
512
- """
513
- Checks if we can abort/continue/respond to request.
514
- :return: True if we can, False if we can't
515
- """
516
- # Mocking responses for dataURL requests is not currently supported.
517
- if self._url.startswith('data:'):
518
- return False
519
- if not self._allowInterception:
520
- raise ValueError('Request Interception is not enabled!')
521
- if self._interceptionHandled:
522
- raise ValueError('Request is already handled')
523
- return True
524
-
525
-
526
- class Response:
527
- """Response class represents responses which are received by ``Page``."""
528
-
529
- def __init__(self, client: CDPSession, request: 'Request', responsePayload):
530
- self._client = client
531
- self._request = request
532
- self._contentFuture = None
533
-
534
- self._bodyLoadedFuture = client.loop.create_future()
535
- self._bodyLoadedFutureFulFill = lambda x: self._bodyLoadedFuture.set_result(x)
536
-
537
- self._remoteAddress = {
538
- 'ip': responsePayload.get('remoteIPAddress'),
539
- 'port': responsePayload.get('remotePort'),
540
- }
541
- self._status = responsePayload.get('status')
542
- self._statusText = responsePayload.get('statusText')
543
- self._url = request.url
544
- self._fromDiskCache = bool(responsePayload.get('fromDiskCache'))
545
- self._fromServiceWorker = bool(responsePayload.get('fromServiceWorker'))
546
- self._headers = {k.lower(): v for k, v in responsePayload.get('headers', {}).items()}
547
- if responsePayload.get('securityDetails'):
548
- self._securityDetails = SecurityDetails(**responsePayload.get('securityDetails'))
549
- else:
550
- self._securityDetails = None
551
-
552
- @property
553
- def remoteAddress(self) -> Dict[str, str]:
554
- return self._remoteAddress
555
-
556
- @property
557
- def url(self) -> str:
558
- return self._url
559
-
560
- @property
561
- def ok(self) -> bool:
562
- return self._status == 0 or (200 <= self._status < 300)
563
-
564
- @property
565
- def status(self) -> int:
566
- return self._status
567
-
568
- @property
569
- def statusText(self) -> str:
570
- return self._statusText
571
-
572
- @property
573
- def headers(self) -> Dict[str, str]:
574
- """
575
- Return dictionary of HTTP headers of this response.
576
- All header names are lower-case.
577
- """
578
- return self._headers
579
-
580
- @property
581
- def securityDetails(self) -> Optional['SecurityDetails']:
582
- """Return security details associated with this response.
583
-
584
- Security details if the response was received over the secure
585
- connection, or `None` otherwise.
586
- """
587
- return self._securityDetails
588
-
589
- def buffer(self) -> Awaitable[bytes]:
590
- """Return awaitable which resolves to bytes with response body."""
591
- if self._contentFuture is None:
592
-
593
- async def buffer_read():
594
- await self._bodyLoadedFuture
595
- response = await self._client.send('Network.getResponseBody', {'requestId': self._request._requestId})
596
- body = response.get('body', '')
597
- if response.get('base64Encoded'):
598
- return base64.b64decode(body)
599
- # b64decode gives us bytes, we encode this so that this fn always returns a bytes
600
- return body.encode('utf-8')
601
-
602
- self._contentFuture = self._client.loop.create_task(buffer_read())
603
- return self._contentFuture
604
-
605
- @property
606
- async def text(self) -> str:
607
- """Text representation of response body."""
608
- return (await self.buffer()).decode('utf-8', errors='replace')
609
-
610
- @property
611
- async def json(self) -> Dict[str, Any]:
612
- """JSON representation of response body."""
613
- return json.loads(await self.text)
614
-
615
- @property
616
- def request(self) -> 'Request':
617
- """matching :class:`Request` object."""
618
- return self._request
619
-
620
- @property
621
- def fromCache(self) -> bool:
622
- """
623
- Return ``True`` if the response was served from cache.
624
- Here `cache` is either the browser's disk cache or memory cache.
625
- """
626
- return self._fromDiskCache or self._request._fromMemoryCache
627
-
628
- @property
629
- def fromServiceWorker(self) -> bool:
630
- """Return ``True`` if the response was served by a service worker."""
631
- return self._fromServiceWorker
632
-
633
- @property
634
- def frame(self) -> Optional['Frame']:
635
- return self._request.frame
636
-
637
-
638
- class SecurityDetails:
639
- """Class represents responses which are received by page."""
640
-
641
- _recorded_attributes = {'subjectName', 'issuer', 'validFrom', 'validTo', 'protocol'}
642
-
643
- def __init__(self, **kwargs):
644
- for k, v in kwargs.items():
645
- if k in self._recorded_attributes:
646
- self.__setattr__(f'_{k}', v)
647
-
648
- @property
649
- def subjectName(self) -> str:
650
- """Return the subject to which the certificate was issued to."""
651
- return self._subjectName
652
-
653
- @property
654
- def issuer(self) -> str:
655
- """Return a string with the name of issuer of the certificate."""
656
- return self._issuer
657
-
658
- @property
659
- def validFrom(self) -> int:
660
- """Return UnixTime of the start of validity of the certificate."""
661
- return self._validFrom
662
-
663
- @property
664
- def validTo(self) -> int:
665
- """Return UnixTime of the end of validity of the certificate."""
666
- return self._validTo
667
-
668
- @property
669
- def protocol(self) -> str:
670
- """Return string of with the security protocol, e.g. "TLS1.2"."""
671
- return self._protocol
672
-
673
-
674
- def headersArray(headers: Dict[str, str]) -> List[Dict[str, str]]:
675
- return [{'name': k, 'value': str(v)} for k, v in headers.items() if v is not None]
676
-
677
-
678
- STATUS_TEXTS = {num.value: code for code, num in vars(HTTPStatus).items() if isinstance(num, HTTPStatus)}
@@ -1,8 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
-
4
- """Options module."""
5
-
6
- from argparse import Namespace
7
-
8
- config = Namespace()