pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. biolib/__init__.py +357 -11
  2. biolib/_data_record/data_record.py +380 -0
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +55 -0
  5. biolib/_index/query_result.py +103 -0
  6. biolib/_internal/__init__.py +0 -0
  7. biolib/_internal/add_copilot_prompts.py +58 -0
  8. biolib/_internal/add_gui_files.py +81 -0
  9. biolib/_internal/data_record/__init__.py +1 -0
  10. biolib/_internal/data_record/data_record.py +85 -0
  11. biolib/_internal/data_record/push_data.py +116 -0
  12. biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
  13. biolib/_internal/errors.py +5 -0
  14. biolib/_internal/file_utils.py +125 -0
  15. biolib/_internal/fuse_mount/__init__.py +1 -0
  16. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  17. biolib/_internal/http_client.py +159 -0
  18. biolib/_internal/lfs/__init__.py +1 -0
  19. biolib/_internal/lfs/cache.py +51 -0
  20. biolib/_internal/libs/__init__.py +1 -0
  21. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  22. biolib/_internal/push_application.py +488 -0
  23. biolib/_internal/runtime.py +22 -0
  24. biolib/_internal/string_utils.py +13 -0
  25. biolib/_internal/templates/__init__.py +1 -0
  26. biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
  27. biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
  28. biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
  29. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  30. biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
  31. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  32. biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
  33. biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
  34. biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
  35. biolib/_internal/templates/gitignore_template/.gitignore +10 -0
  36. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  37. biolib/_internal/templates/gui_template/App.tsx +53 -0
  38. biolib/_internal/templates/gui_template/Dockerfile +27 -0
  39. biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
  40. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  41. biolib/_internal/templates/gui_template/index.css +5 -0
  42. biolib/_internal/templates/gui_template/index.html +13 -0
  43. biolib/_internal/templates/gui_template/index.tsx +10 -0
  44. biolib/_internal/templates/gui_template/package.json +27 -0
  45. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  46. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
  47. biolib/_internal/templates/gui_template/vite.config.mts +10 -0
  48. biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
  49. biolib/_internal/templates/init_template/Dockerfile +14 -0
  50. biolib/_internal/templates/init_template/requirements.txt +1 -0
  51. biolib/_internal/templates/init_template/run.py +12 -0
  52. biolib/_internal/templates/init_template/run.sh +4 -0
  53. biolib/_internal/templates/templates.py +25 -0
  54. biolib/_internal/tree_utils.py +106 -0
  55. biolib/_internal/utils/__init__.py +65 -0
  56. biolib/_internal/utils/auth.py +46 -0
  57. biolib/_internal/utils/job_url.py +33 -0
  58. biolib/_internal/utils/multinode.py +263 -0
  59. biolib/_runtime/runtime.py +157 -0
  60. biolib/_session/session.py +44 -0
  61. biolib/_shared/__init__.py +0 -0
  62. biolib/_shared/types/__init__.py +74 -0
  63. biolib/_shared/types/account.py +12 -0
  64. biolib/_shared/types/account_member.py +8 -0
  65. biolib/_shared/types/app.py +9 -0
  66. biolib/_shared/types/data_record.py +40 -0
  67. biolib/_shared/types/experiment.py +32 -0
  68. biolib/_shared/types/file_node.py +17 -0
  69. biolib/_shared/types/push.py +6 -0
  70. biolib/_shared/types/resource.py +37 -0
  71. biolib/_shared/types/resource_deploy_key.py +11 -0
  72. biolib/_shared/types/resource_permission.py +14 -0
  73. biolib/_shared/types/resource_version.py +19 -0
  74. biolib/_shared/types/result.py +14 -0
  75. biolib/_shared/types/typing.py +10 -0
  76. biolib/_shared/types/user.py +19 -0
  77. biolib/_shared/utils/__init__.py +7 -0
  78. biolib/_shared/utils/resource_uri.py +75 -0
  79. biolib/api/__init__.py +6 -0
  80. biolib/api/client.py +168 -0
  81. biolib/app/app.py +252 -49
  82. biolib/app/search_apps.py +45 -0
  83. biolib/biolib_api_client/api_client.py +126 -31
  84. biolib/biolib_api_client/app_types.py +24 -4
  85. biolib/biolib_api_client/auth.py +31 -8
  86. biolib/biolib_api_client/biolib_app_api.py +147 -52
  87. biolib/biolib_api_client/biolib_job_api.py +161 -141
  88. biolib/biolib_api_client/job_types.py +21 -5
  89. biolib/biolib_api_client/lfs_types.py +7 -23
  90. biolib/biolib_api_client/user_state.py +56 -0
  91. biolib/biolib_binary_format/__init__.py +1 -4
  92. biolib/biolib_binary_format/file_in_container.py +105 -0
  93. biolib/biolib_binary_format/module_input.py +24 -7
  94. biolib/biolib_binary_format/module_output_v2.py +149 -0
  95. biolib/biolib_binary_format/remote_endpoints.py +34 -0
  96. biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
  97. biolib/biolib_binary_format/saved_job.py +3 -2
  98. biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
  99. biolib/biolib_binary_format/system_status_update.py +3 -2
  100. biolib/biolib_binary_format/utils.py +175 -0
  101. biolib/biolib_docker_client/__init__.py +11 -2
  102. biolib/biolib_errors.py +36 -0
  103. biolib/biolib_logging.py +27 -10
  104. biolib/cli/__init__.py +38 -0
  105. biolib/cli/auth.py +46 -0
  106. biolib/cli/data_record.py +164 -0
  107. biolib/cli/index.py +32 -0
  108. biolib/cli/init.py +421 -0
  109. biolib/cli/lfs.py +101 -0
  110. biolib/cli/push.py +50 -0
  111. biolib/cli/run.py +63 -0
  112. biolib/cli/runtime.py +14 -0
  113. biolib/cli/sdk.py +16 -0
  114. biolib/cli/start.py +56 -0
  115. biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
  116. biolib/compute_node/job_worker/cache_state.py +66 -88
  117. biolib/compute_node/job_worker/cache_types.py +1 -6
  118. biolib/compute_node/job_worker/docker_image_cache.py +112 -37
  119. biolib/compute_node/job_worker/executors/__init__.py +0 -3
  120. biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
  121. biolib/compute_node/job_worker/executors/docker_types.py +9 -1
  122. biolib/compute_node/job_worker/executors/types.py +19 -9
  123. biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
  124. biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
  125. biolib/compute_node/job_worker/job_storage.py +108 -0
  126. biolib/compute_node/job_worker/job_worker.py +397 -212
  127. biolib/compute_node/job_worker/large_file_system.py +87 -38
  128. biolib/compute_node/job_worker/network_alloc.py +99 -0
  129. biolib/compute_node/job_worker/network_buffer.py +240 -0
  130. biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
  131. biolib/compute_node/job_worker/utils.py +9 -24
  132. biolib/compute_node/remote_host_proxy.py +400 -98
  133. biolib/compute_node/utils.py +31 -9
  134. biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
  135. biolib/compute_node/webserver/proxy_utils.py +28 -0
  136. biolib/compute_node/webserver/webserver.py +130 -44
  137. biolib/compute_node/webserver/webserver_types.py +2 -6
  138. biolib/compute_node/webserver/webserver_utils.py +77 -12
  139. biolib/compute_node/webserver/worker_thread.py +183 -42
  140. biolib/experiments/__init__.py +0 -0
  141. biolib/experiments/experiment.py +356 -0
  142. biolib/jobs/__init__.py +1 -0
  143. biolib/jobs/job.py +741 -0
  144. biolib/jobs/job_result.py +185 -0
  145. biolib/jobs/types.py +50 -0
  146. biolib/py.typed +0 -0
  147. biolib/runtime/__init__.py +14 -0
  148. biolib/sdk/__init__.py +91 -0
  149. biolib/tables.py +34 -0
  150. biolib/typing_utils.py +2 -7
  151. biolib/user/__init__.py +1 -0
  152. biolib/user/sign_in.py +54 -0
  153. biolib/utils/__init__.py +162 -0
  154. biolib/utils/cache_state.py +94 -0
  155. biolib/utils/multipart_uploader.py +194 -0
  156. biolib/utils/seq_util.py +150 -0
  157. biolib/utils/zip/remote_zip.py +640 -0
  158. pybiolib-1.2.1890.dist-info/METADATA +41 -0
  159. pybiolib-1.2.1890.dist-info/RECORD +177 -0
  160. {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
  161. pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
  162. README.md +0 -17
  163. biolib/app/app_result.py +0 -68
  164. biolib/app/utils.py +0 -62
  165. biolib/biolib-js/0-biolib.worker.js +0 -1
  166. biolib/biolib-js/1-biolib.worker.js +0 -1
  167. biolib/biolib-js/2-biolib.worker.js +0 -1
  168. biolib/biolib-js/3-biolib.worker.js +0 -1
  169. biolib/biolib-js/4-biolib.worker.js +0 -1
  170. biolib/biolib-js/5-biolib.worker.js +0 -1
  171. biolib/biolib-js/6-biolib.worker.js +0 -1
  172. biolib/biolib-js/index.html +0 -10
  173. biolib/biolib-js/main-biolib.js +0 -1
  174. biolib/biolib_api_client/biolib_account_api.py +0 -21
  175. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
  176. biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
  177. biolib/biolib_binary_format/module_output.py +0 -58
  178. biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
  179. biolib/biolib_push.py +0 -114
  180. biolib/cli.py +0 -203
  181. biolib/cli_utils.py +0 -273
  182. biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
  183. biolib/compute_node/enclave/__init__.py +0 -2
  184. biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
  185. biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
  186. biolib/compute_node/job_worker/executors/base_executor.py +0 -18
  187. biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
  188. biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
  189. biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
  190. biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
  191. biolib/lfs.py +0 -196
  192. biolib/pyppeteer/.circleci/config.yml +0 -100
  193. biolib/pyppeteer/.coveragerc +0 -3
  194. biolib/pyppeteer/.gitignore +0 -89
  195. biolib/pyppeteer/.pre-commit-config.yaml +0 -28
  196. biolib/pyppeteer/CHANGES.md +0 -253
  197. biolib/pyppeteer/CONTRIBUTING.md +0 -26
  198. biolib/pyppeteer/LICENSE +0 -12
  199. biolib/pyppeteer/README.md +0 -137
  200. biolib/pyppeteer/docs/Makefile +0 -177
  201. biolib/pyppeteer/docs/_static/custom.css +0 -28
  202. biolib/pyppeteer/docs/_templates/layout.html +0 -10
  203. biolib/pyppeteer/docs/changes.md +0 -1
  204. biolib/pyppeteer/docs/conf.py +0 -299
  205. biolib/pyppeteer/docs/index.md +0 -21
  206. biolib/pyppeteer/docs/make.bat +0 -242
  207. biolib/pyppeteer/docs/reference.md +0 -211
  208. biolib/pyppeteer/docs/server.py +0 -60
  209. biolib/pyppeteer/poetry.lock +0 -1699
  210. biolib/pyppeteer/pyppeteer/__init__.py +0 -135
  211. biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
  212. biolib/pyppeteer/pyppeteer/browser.py +0 -401
  213. biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
  214. biolib/pyppeteer/pyppeteer/command.py +0 -22
  215. biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
  216. biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
  217. biolib/pyppeteer/pyppeteer/coverage.py +0 -346
  218. biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
  219. biolib/pyppeteer/pyppeteer/dialog.py +0 -79
  220. biolib/pyppeteer/pyppeteer/domworld.py +0 -597
  221. biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
  222. biolib/pyppeteer/pyppeteer/errors.py +0 -48
  223. biolib/pyppeteer/pyppeteer/events.py +0 -63
  224. biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
  225. biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
  226. biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
  227. biolib/pyppeteer/pyppeteer/helpers.py +0 -245
  228. biolib/pyppeteer/pyppeteer/input.py +0 -371
  229. biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
  230. biolib/pyppeteer/pyppeteer/launcher.py +0 -683
  231. biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
  232. biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
  233. biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
  234. biolib/pyppeteer/pyppeteer/multimap.py +0 -82
  235. biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
  236. biolib/pyppeteer/pyppeteer/options.py +0 -8
  237. biolib/pyppeteer/pyppeteer/page.py +0 -1728
  238. biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
  239. biolib/pyppeteer/pyppeteer/target.py +0 -147
  240. biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
  241. biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
  242. biolib/pyppeteer/pyppeteer/tracing.py +0 -93
  243. biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
  244. biolib/pyppeteer/pyppeteer/util.py +0 -18
  245. biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
  246. biolib/pyppeteer/pyppeteer/worker.py +0 -101
  247. biolib/pyppeteer/pyproject.toml +0 -97
  248. biolib/pyppeteer/spell.txt +0 -137
  249. biolib/pyppeteer/tox.ini +0 -72
  250. biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
  251. biolib/start_cli.py +0 -7
  252. biolib/utils.py +0 -47
  253. biolib/validators/validate_app_version.py +0 -183
  254. biolib/validators/validate_argument.py +0 -134
  255. biolib/validators/validate_module.py +0 -323
  256. biolib/validators/validate_zip_file.py +0 -40
  257. biolib/validators/validator_utils.py +0 -103
  258. pybiolib-0.2.951.dist-info/LICENSE +0 -21
  259. pybiolib-0.2.951.dist-info/METADATA +0 -61
  260. pybiolib-0.2.951.dist-info/RECORD +0 -153
  261. pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
  262. /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
@@ -1,242 +0,0 @@
1
- import asyncio
2
- import json
3
- import logging
4
- import sys
5
- from typing import Any, Awaitable, Dict
6
-
7
- import websockets
8
- from pyee import AsyncIOEventEmitter
9
-
10
- from biolib.pyppeteer.pyppeteer.errors import NetworkError
11
- from biolib.pyppeteer.pyppeteer.events import Events
12
- from biolib.pyppeteer.pyppeteer.websocket_transport import WebsocketTransport
13
-
14
- if sys.version_info < (3, 8):
15
- from typing_extensions import TypedDict
16
- else:
17
- from typing import TypedDict
18
-
19
- logger = logging.getLogger(__name__)
20
- logger.setLevel(logging.DEBUG)
21
- logger_connection = logging.getLogger(__name__ + '.Connection')
22
-
23
-
24
- class TargetInfo(TypedDict, total=False):
25
- type: str
26
-
27
-
28
- class MessageParams(TypedDict, total=False):
29
- targetInfo: TargetInfo
30
- sessionId: str
31
-
32
-
33
- class MessageError(TypedDict, total=False):
34
- message: str
35
- data: Any
36
-
37
-
38
- class Message(TypedDict, total=False):
39
- method: str
40
- id: int
41
- params: MessageParams
42
- error: MessageError
43
- result: Any
44
-
45
-
46
- class Connection(AsyncIOEventEmitter):
47
- """Connection management class."""
48
-
49
- def __init__(
50
- self, url: str, transport: WebsocketTransport, delay: float = 0, loop: asyncio.AbstractEventLoop = None,
51
- ) -> None:
52
- """Make connection.
53
-
54
- :arg str url: WebSocket url to connect devtool.
55
- :arg int delay: delay to wait before processing received messages.
56
- """
57
- super().__init__()
58
- self._url = url
59
- self._lastId = 0
60
- self._callbacks: Dict[int, asyncio.Future] = {}
61
- self._delay = delay / 1000
62
-
63
- self._transport = transport
64
-
65
- self.loop = loop or asyncio.get_event_loop()
66
- self._sessions: Dict[str, CDPSession] = {}
67
- self._connected = False
68
- self._closed = False
69
- self.loop.create_task(self._recv_loop())
70
-
71
- @staticmethod
72
- def fromSession(session: 'CDPSession') -> 'Connection':
73
- return session._connection
74
-
75
- def session(self, sessionId) -> 'CDPSession':
76
- return self._sessions.get(sessionId)
77
-
78
- @property
79
- def url(self) -> str:
80
- """Get connected WebSocket url."""
81
- return self._url
82
-
83
- async def _recv_loop(self) -> None:
84
- try:
85
- self._connected = True
86
- self.connection = self._transport
87
- self.connection.onmessage = lambda msg: self._onMessage(msg)
88
- self.connection.onclose = self._onClose
89
- while self._connected:
90
- try:
91
- await self.connection.recv()
92
- except (websockets.ConnectionClosed, ConnectionResetError) as excpt:
93
- logger.warning(f'Transport connection closed: {excpt}')
94
- break
95
- # wait 1 async loop frame, no other data will be accessible in between frames
96
- await asyncio.sleep(0)
97
- except Exception as excpt:
98
- await self.dispose(reason=str(excpt))
99
- raise excpt
100
- await self.dispose(reason=None)
101
-
102
- async def _async_send(self, msg: Message) -> None:
103
- while not self._connected:
104
- await asyncio.sleep(self._delay)
105
- try:
106
- remove_none_items_inplace(msg)
107
- msg_to_send = json.dumps(msg)
108
- await self.connection.send(msg_to_send)
109
- logger_connection.debug(f'SEND ▶ {msg_to_send}')
110
- except websockets.ConnectionClosed:
111
- logger.error('connection unexpectedly closed')
112
- callback = self._callbacks.get(msg['id'], None)
113
- if callback and not callback.done():
114
- callback.set_result(None)
115
- await self.dispose()
116
-
117
- def send(self, method: str, params: dict = None) -> Awaitable:
118
- """Send message via the connection."""
119
- # Detect connection availability from the second transmission
120
- if self._lastId and not self._connected:
121
- raise ConnectionError('Connection is closed')
122
- id_ = self._rawSend({'method': method, 'params': params or {}})
123
- callback = self.loop.create_future()
124
- callback.error: Exception = NetworkError() # type: ignore
125
- callback.method: str = method # type: ignore
126
- self._callbacks[id_] = callback
127
- return callback
128
-
129
- def _rawSend(self, message: Message) -> int:
130
- self._lastId += 1
131
- id_ = self._lastId
132
- message['id'] = id_
133
- self.loop.create_task(self._async_send(message))
134
- return id_
135
-
136
- async def _onMessage(self, msg: str) -> None:
137
- loaded_msg: Message = json.loads(msg)
138
- if self._delay:
139
- await asyncio.sleep(self._delay)
140
- logger_connection.debug(f'◀ RECV {loaded_msg}')
141
-
142
- # Handle Target attach/detach methods
143
- if loaded_msg.get('method') == 'Target.attachedToTarget':
144
- sessionId = loaded_msg['params']['sessionId']
145
- self._sessions[sessionId] = CDPSession(
146
- connection=self,
147
- targetType=loaded_msg['params']['targetInfo']['type'],
148
- sessionId=sessionId,
149
- loop=self.loop,
150
- )
151
- elif loaded_msg.get('method') == 'Target.detachedFromTarget':
152
- session = self._sessions.get(loaded_msg['params']['sessionId'])
153
- if session:
154
- session._onClosed()
155
- del self._sessions[loaded_msg['params']['sessionId']]
156
-
157
- if loaded_msg.get('sessionId'):
158
- session = self._sessions.get(loaded_msg['sessionId'])
159
- if session:
160
- session._onMessage(loaded_msg)
161
- elif loaded_msg.get('id'):
162
- # Callbacks could be all rejected if someone has called `.dispose()`
163
- callback = self._callbacks.get(loaded_msg['id'])
164
- if callback:
165
- if loaded_msg.get('error'):
166
- callback.set_exception(createProtocolError(callback.error, callback.method, loaded_msg))
167
- else:
168
- callback.set_result(loaded_msg.get('result'))
169
- del self._callbacks[loaded_msg['id']]
170
- else:
171
- self.emit(loaded_msg['method'], loaded_msg['params'])
172
-
173
- async def _onClose(self) -> None:
174
- if self._closed:
175
- return
176
- self._closed = True
177
- self._transport.onmessage = None
178
- self._transport.onclose = None
179
-
180
- for cb in self._callbacks.values():
181
- cb.set_exception(
182
- rewriteError(
183
- cb.error, # type: ignore
184
- f'Protocol error {cb.method}: Target closed.', # type: ignore
185
- )
186
- )
187
- self._callbacks.clear()
188
-
189
- for session in self._sessions.values():
190
- session._onClosed()
191
- self._sessions.clear()
192
-
193
- # close connection
194
- if hasattr(self, 'connection'): # may not have connection
195
- await self.connection.close()
196
- self._sessions.clear()
197
- self.emit(Events.Connection.Disconnected)
198
-
199
- async def dispose(self, code: int = 1000, reason: str = None) -> None:
200
- """Close all connection."""
201
- self._connected = False
202
- await self._onClose()
203
- await self._transport.close(code=code, reason=str(reason))
204
-
205
- async def createSession(self, targetInfo: Dict) -> 'CDPSession':
206
- """Create new session."""
207
- resp = await self.send('Target.attachToTarget', {'targetId': targetInfo['targetId'], 'flatten': True})
208
- sessionId = resp.get('sessionId')
209
- return self._sessions[sessionId]
210
-
211
-
212
- def createProtocolError(error: Exception, method: str, obj: Dict) -> Exception:
213
- message = f'Protocol error ({method}): {obj["error"]["message"]}'
214
- if 'data' in obj['error']:
215
- message += f' {obj["error"]["data"]}'
216
- return rewriteError(error, message)
217
-
218
-
219
- def rewriteError(error: Exception, message: str) -> Exception:
220
- error.args = (message,)
221
- return error
222
-
223
-
224
- def remove_none_items_inplace(o: Dict[str, Any]) -> None:
225
- """
226
- Removes items that have a value of None. There are instances in puppeteer where a object (dict) is sent which has
227
- undefined values, which are then omitted from the resulting json. This function emulates such behaviour, removing
228
- all k:v pairs where v = None
229
- :param o:
230
- :return Dict[str, Any]: dict without any None values
231
- """
232
- none_keys = []
233
- for key, value in o.items():
234
- if isinstance(value, dict):
235
- remove_none_items_inplace(value)
236
- if value is None:
237
- none_keys.append(key)
238
- for key in none_keys:
239
- del o[key]
240
-
241
-
242
- from biolib.pyppeteer.pyppeteer.connection.cdpsession import CDPSession # isort:skip
@@ -1,101 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
-
4
- """Connection/Session management module."""
5
-
6
- import asyncio
7
- from typing import Awaitable, Dict, Union
8
-
9
- from pyee import AsyncIOEventEmitter
10
-
11
- from biolib.pyppeteer.pyppeteer.connection import Connection, Message, createProtocolError, rewriteError
12
- from biolib.pyppeteer.pyppeteer.errors import NetworkError
13
- from biolib.pyppeteer.pyppeteer.events import Events
14
-
15
-
16
- class CDPSession(AsyncIOEventEmitter):
17
- """Chrome Devtools Protocol Session.
18
-
19
- The :class:`CDPSession` instances are used to talk raw Chrome Devtools
20
- Protocol:
21
-
22
- * protocol methods can be called with :meth:`send` method.
23
- * protocol events can be subscribed to with :meth:`on` method.
24
-
25
- Documentation on DevTools Protocol can be found
26
- `here <https://chromedevtools.github.io/devtools-protocol/>`__.
27
- """
28
-
29
- def __init__(
30
- self,
31
- connection: Union[Connection, 'CDPSession'],
32
- targetType: str,
33
- sessionId: str,
34
- loop: asyncio.AbstractEventLoop,
35
- ) -> None:
36
- """Make new session."""
37
- super().__init__()
38
- self._callbacks: Dict[int, asyncio.Future] = {}
39
- self._connection = connection
40
- self._targetType = targetType
41
- self._sessionId = sessionId
42
- self.loop = loop
43
-
44
- def send(self, method: str, params: dict = None) -> Awaitable:
45
- """Send message to the connected session.
46
-
47
- :arg str method: Protocol method name.
48
- :arg dict params: Optional method parameters.
49
- """
50
- if not self._connection:
51
- raise NetworkError(
52
- f'Protocol Error ({method}): Session closed. Most likely the {self._targetType} has been closed.'
53
- )
54
- id_ = self._connection._rawSend({'sessionId': self._sessionId, 'method': method, 'params': params or {},})
55
- callback = self.loop.create_future()
56
- callback.method = method
57
- callback.error = NetworkError()
58
- self._callbacks[id_] = callback
59
- return callback
60
-
61
- def _onMessage(self, msg: Message) -> None:
62
- id_ = msg.get('id')
63
- callback = self._callbacks.get(id_)
64
- if id_ and id_ in self._callbacks:
65
- if msg.get('error'):
66
- callback.set_exception(
67
- createProtocolError(
68
- callback.error, # type: ignore
69
- callback.method, # type: ignore
70
- msg,
71
- )
72
- )
73
- else:
74
- callback.set_result(msg.get('result'))
75
- del self._callbacks[id_]
76
- else:
77
- if msg.get('id'):
78
- raise ConnectionError(f'Received unexpected message with no callback: {msg}')
79
- self.emit(msg.get('method'), msg.get('params'))
80
-
81
- async def detach(self) -> None:
82
- """Detach session from target.
83
-
84
- Once detached, session won't emit any events and can't be used to send
85
- messages.
86
- """
87
- if not self._connection:
88
- raise NetworkError('Session already detached. Most likelythe {self._targetType} has been closed')
89
- await self._connection.send('Target.detachFromTarget', {'sessionId': self._sessionId})
90
-
91
- def _onClosed(self) -> None:
92
- for cb in self._callbacks.values():
93
- cb.set_exception(
94
- rewriteError(
95
- cb.error, # type: ignore
96
- f'Protocol error {cb.method}: Target closed.', # type: ignore
97
- )
98
- )
99
- self._callbacks.clear()
100
- self._connection = None
101
- self.emit(Events.CDPSession.Disconnected)
@@ -1,346 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
-
4
- """Coverage module."""
5
- import asyncio
6
- import logging
7
- from functools import cmp_to_key
8
- from typing import Dict, List
9
-
10
- from biolib.pyppeteer.pyppeteer import helpers
11
- from biolib.pyppeteer.pyppeteer.connection import CDPSession
12
- from biolib.pyppeteer.pyppeteer.errors import PageError
13
- from biolib.pyppeteer.pyppeteer.execution_context import EVALUATION_SCRIPT_URL
14
- from biolib.pyppeteer.pyppeteer.models import CoverageResult, NestedRangeItemInput, NestedRangeItem, Protocol
15
-
16
-
17
- logger = logging.getLogger(__name__)
18
-
19
-
20
- class Coverage:
21
- """Coverage class.
22
-
23
- Coverage gathers information about parts of JavaScript and CSS that were
24
- used by the page.
25
-
26
- An example of using JavaScript and CSS coverage to get percentage of
27
- initially executed code::
28
-
29
- # Enable both JavaScript and CSS coverage
30
- await page.coverage.startJSCoverage()
31
- await page.coverage.startCSSCoverage()
32
-
33
- # Navigate to page
34
- await page.goto('https://example.com')
35
- # Disable JS and CSS coverage and get results
36
- jsCoverage = await page.coverage.stopJSCoverage()
37
- cssCoverage = await page.coverage.stopCSSCoverage()
38
- totalBytes = 0
39
- usedBytes = 0
40
- coverage = jsCoverage + cssCoverage
41
- for entry in coverage:
42
- totalBytes += len(entry['text'])
43
- for range in entry['ranges']:
44
- usedBytes += range['end'] - range['start'] - 1
45
-
46
- print('Bytes used: {}%'.format(usedBytes / totalBytes * 100))
47
- """
48
-
49
- def __init__(self, client: CDPSession) -> None:
50
- self._jsCoverage = JSCoverage(client)
51
- self._cssCoverage = CSSCoverage(client)
52
-
53
- async def startJSCoverage(self, resetOnNavigation: bool = True, reportAnonymousScripts: bool = False,) -> None:
54
- """Start JS coverage measurement.
55
-
56
- :param resetOnNavigation: Whether to reset coverage on every
57
- navigation.
58
- :param reportAnonymousScripts: Whether anonymous script generated
59
- by the page should be reported.
60
-
61
- .. note::
62
- Anonymous scripts are ones that don't have an associated url. These
63
- are scripts that are dynamically created on the page using ``eval``
64
- of ``new Function``. If ``reportAnonymousScript`` is set to
65
- ``True``, anonymous scripts will have
66
- ``__pyppeteer_evaluation_script__`` as their url.
67
- """
68
- await self._jsCoverage.start(
69
- resetOnNavigation=resetOnNavigation, reportAnonymousScripts=reportAnonymousScripts,
70
- )
71
-
72
- async def stopJSCoverage(self) -> List:
73
- """Stop JS coverage measurement and get result.
74
-
75
- Return list of coverage reports for all scripts. Each report includes:
76
-
77
- * ``url`` (str): Script url.
78
- * ``text`` (str): Script content.
79
- * ``ranges`` (List[Dict]): Script ranges that were executed. Ranges are
80
- sorted and non-overlapping.
81
-
82
- * ``start`` (int): A start offset in text, inclusive.
83
- * ``end`` (int): An end offset in text, exclusive.
84
-
85
- .. note::
86
- JavaScript coverage doesn't include anonymous scripts by default.
87
- However, scripts with sourceURLs are reported.
88
- """
89
- return await self._jsCoverage.stop()
90
-
91
- async def startCSSCoverage(self, resetOnNavigation: bool = True) -> None:
92
- """Start CSS coverage measurement.
93
-
94
-
95
- :param resetOnNavigation: Whether to reset coverage on every
96
- navigation.
97
- """
98
- await self._cssCoverage.start(resetOnNavigation=resetOnNavigation)
99
-
100
- async def stopCSSCoverage(self) -> List:
101
- """Stop CSS coverage measurement and get result.
102
-
103
- Return list of coverage reports for all non-anonymous scripts. Each
104
- report includes:
105
-
106
- * ``url`` (str): StyleSheet url.
107
- * ``text`` (str): StyleSheet content.
108
- * ``ranges`` (List[Dict]): StyleSheet ranges that were executed. Ranges
109
- are sorted and non-overlapping.
110
-
111
- * ``start`` (int): A start offset in text, inclusive.
112
- * ``end`` (int): An end offset in text, exclusive.
113
-
114
- .. note::
115
- CSS coverage doesn't include dynamically injected style tags without
116
- sourceURLs (but currently includes... to be fixed).
117
- """
118
- return await self._cssCoverage.stop()
119
-
120
-
121
- class JSCoverage:
122
- """JavaScript Coverage class."""
123
-
124
- def __init__(self, client: CDPSession) -> None:
125
- self._client = client
126
- self._enabled = False
127
- self._scriptURLs = {}
128
- self._scriptSources = {}
129
- self._eventListeners = []
130
- self._resetOnNavigation = False
131
-
132
- async def start(self, resetOnNavigation: bool = True, reportAnonymousScripts: bool = False,) -> None:
133
- """Start coverage measurement."""
134
- if self._enabled:
135
- raise PageError('JSCoverage is already enabled.')
136
- self._resetOnNavigation = resetOnNavigation
137
- self._reportAnonymousScript = reportAnonymousScripts
138
- self._enabled = True
139
- self._scriptURLs.clear()
140
- self._scriptSources.clear()
141
- self._eventListeners = [
142
- helpers.addEventListener(
143
- self._client, 'Debugger.scriptParsed', lambda e: self._onScriptParsed(e)
144
- ),
145
- helpers.addEventListener(
146
- self._client, 'Runtime.executionContextsCleared', self._onExecutionContextsCleared
147
- ),
148
- ]
149
- await asyncio.gather(
150
- self._client.send('Profiler.enable'),
151
- self._client.send('Profiler.startPreciseCoverage', {'callCount': False, 'detailed': True}),
152
- self._client.send('Debugger.enable'),
153
- self._client.send('Debugger.setSkipAllPauses', {'skip': True}),
154
- )
155
-
156
- def _onExecutionContextsCleared(self, event: Dict) -> None:
157
- if not self._resetOnNavigation:
158
- return
159
- self._scriptURLs.clear()
160
- self._scriptSources.clear()
161
-
162
- async def _onScriptParsed(self, event: Protocol.Debugger.scriptParsedPayload) -> None:
163
- # Ignore pyppeteer-injected scripts
164
- if event.get('url') == EVALUATION_SCRIPT_URL:
165
- return
166
- # Ignore other anonymous scripts unless the reportAnonymousScript
167
- # option is True
168
- if not event.get('url') and not self._reportAnonymousScript:
169
- return
170
-
171
- scriptId = event.get('scriptId')
172
- url = event.get('url')
173
- try:
174
- response = await self._client.send('Debugger.getScriptSource', {'scriptId': scriptId})
175
- self._scriptURLs[scriptId] = url
176
- self._scriptSources[scriptId] = response.get('scriptSource')
177
- except Exception as e:
178
- logger.error(f'An exception occurred during _onScriptParsed handling: {e}'
179
- f'\nThis might happen if the page has already navigated away.')
180
-
181
- async def stop(self) -> List:
182
- """Stop coverage measurement and return results."""
183
- if not self._enabled:
184
- raise PageError('JSCoverage is not enabled')
185
- self._enabled = False
186
-
187
- result, *_ = await asyncio.gather(
188
- self._client.send('Profiler.takePreciseCoverage'),
189
- self._client.send('Profiler.stopPreciseCoverage'),
190
- self._client.send('Profiler.disable'),
191
- self._client.send('Debugger.disable'),
192
- )
193
- helpers.removeEventListeners(self._eventListeners)
194
-
195
- coverage = []
196
- for entry in result.get('result', []):
197
- scriptId = entry.get('scriptId')
198
- url = self._scriptURLs.get(scriptId)
199
- if not url and self._reportAnonymousScript:
200
- url = f'debugger://VM{scriptId}'
201
- text = self._scriptSources.get(scriptId)
202
- if text is None or url is None:
203
- continue
204
- flattenRanges = []
205
- for func in entry.get('functions', []):
206
- flattenRanges.extend(func.get('ranges', []))
207
- ranges = convertToDisjointRanges(flattenRanges)
208
- coverage.append({'url': url, 'ranges': ranges, 'text': text})
209
- return coverage
210
-
211
-
212
- class CSSCoverage:
213
- """CSS Coverage class."""
214
-
215
- def __init__(self, client: CDPSession) -> None:
216
- self._client = client
217
- self._enabled = False
218
- self._stylesheetURLs: Dict = {}
219
- self._stylesheetSources: Dict = {}
220
- self._eventListeners: List = []
221
- self._resetOnNavigation = False
222
-
223
- async def start(self, resetOnNavigation: bool = True) -> None:
224
- """Start coverage measurement."""
225
- if self._enabled:
226
- raise PageError('CSSCoverage is already enabled.')
227
- self._resetOnNavigation = resetOnNavigation
228
- self._enabled = True
229
- self._stylesheetURLs.clear()
230
- self._stylesheetSources.clear()
231
- self._eventListeners = [
232
- helpers.addEventListener(
233
- self._client, 'CSS.styleSheetAdded', lambda e: self._client.loop.create_task(self._onStyleSheet(e))
234
- ),
235
- helpers.addEventListener(
236
- self._client, 'Runtime.executionContextsCleared', self._onExecutionContextsCleared
237
- ),
238
- ]
239
- await asyncio.gather(
240
- self._client.send('DOM.enable'),
241
- self._client.send('CSS.enable'),
242
- self._client.send('CSS.startRuleUsageTracking'),
243
- )
244
-
245
- def _onExecutionContextsCleared(self, event: Dict) -> None:
246
- if not self._resetOnNavigation:
247
- return
248
- self._stylesheetURLs.clear()
249
- self._stylesheetSources.clear()
250
-
251
- async def _onStyleSheet(self, event: Dict) -> None:
252
- header = event.get('header', {})
253
- # Ignore anonymous scripts
254
- if not header.get('sourceURL'):
255
- return
256
- try:
257
- response = await self._client.send('CSS.getStyleSheetText', {'styleSheetId': header['styleSheetId']})
258
- self._stylesheetURLs[header['styleSheetId']] = header['sourceURL']
259
- self._stylesheetSources[header['styleSheetId']] = response['text']
260
- except Exception as e:
261
- # This might happen if the page has already navigated away.
262
- logger.error(f'An exception occurred: {e}')
263
-
264
- async def stop(self) -> List[CoverageResult]:
265
- """Stop coverage measurement and return results."""
266
- if not self._enabled:
267
- raise PageError('CSSCoverage is not enabled.')
268
- self._enabled = False
269
- ruleTrackingResponse = await self._client.send('CSS.stopRuleUsageTracking')
270
- await asyncio.gather(self._client.send('CSS.disable'), self._client.send('DOM.disable'))
271
- helpers.removeEventListeners(self._eventListeners)
272
-
273
- # aggregate by styleSheetId
274
- styleSheetIdToCoverage: Dict[str, List[Dict[str, str]]] = {}
275
- for entry in ruleTrackingResponse['ruleUsage']:
276
- ranges = styleSheetIdToCoverage.get(entry['styleSheetId'])
277
- if not ranges:
278
- ranges = []
279
- styleSheetIdToCoverage[entry['styleSheetId']] = ranges
280
- ranges.append(
281
- {
282
- 'startOffset': entry['startOffset'],
283
- 'endOffset': entry['endOffset'],
284
- 'count': 1 if entry['used'] else 0,
285
- }
286
- )
287
-
288
- coverage = []
289
- for styleSheetId in self._stylesheetURLs:
290
- url = self._stylesheetURLs.get(styleSheetId)
291
- text = self._stylesheetSources.get(styleSheetId)
292
- ranges = convertToDisjointRanges(styleSheetIdToCoverage.get(styleSheetId, []))
293
- coverage.append({'url': url, 'ranges': ranges, 'text': text})
294
-
295
- return coverage
296
-
297
-
298
- def convertToDisjointRanges(nestedRanges: List[NestedRangeItemInput]) -> List[NestedRangeItem]:
299
- """
300
- Convert ranges.
301
- NestedRange members support keys:
302
- * startOffset: float
303
- * endOffset: float
304
- * count: int
305
- """
306
- points: List = []
307
- for nested_range in nestedRanges:
308
- points.append({'offset': nested_range['startOffset'], 'type': 0, 'range': nested_range})
309
- points.append({'offset': nested_range['endOffset'], 'type': 1, 'range': nested_range})
310
-
311
- # Sort points to form a valid parenthesis sequence.
312
- def _sort_func(a: Dict, b: Dict) -> int:
313
- # Sort with increasing offsets.
314
- if a['offset'] != b['offset']:
315
- return a['offset'] - b['offset']
316
- # All "end" points should go before "start" points.
317
- if a['type'] != b['type']:
318
- return b['type'] - a['type']
319
- aLength = a['range']['endOffset'] - a['range']['startOffset']
320
- bLength = b['range']['endOffset'] - b['range']['startOffset']
321
- # For two "start" points, the one with longer range goes first.
322
- if a['type'] == 0:
323
- return bLength - aLength
324
- # For two "end" points, the one with shorter range goes first.
325
- return aLength - bLength
326
-
327
- points.sort(key=cmp_to_key(_sort_func))
328
-
329
- hitCountStack = []
330
- results = []
331
- lastOffset = 0
332
- # Run scanning line to intersect all ranges.
333
- for point in points:
334
- if hitCountStack and lastOffset < point['offset'] and hitCountStack[len(hitCountStack) - 1] > 0:
335
- lastResult = results[-1] if results else None
336
- if lastResult and lastResult['end'] == lastOffset:
337
- lastResult['end'] = point['offset']
338
- else:
339
- results.append({'start': lastOffset, 'end': point['offset']})
340
- lastOffset = point['offset']
341
- if point['type'] == 0:
342
- hitCountStack.append(point['range']['count'])
343
- else:
344
- hitCountStack.pop()
345
- # Filter out empty ranges.
346
- return [range for range in results if range['end'] - range['start'] > 1]