pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. biolib/__init__.py +357 -11
  2. biolib/_data_record/data_record.py +380 -0
  3. biolib/_index/__init__.py +0 -0
  4. biolib/_index/index.py +55 -0
  5. biolib/_index/query_result.py +103 -0
  6. biolib/_internal/__init__.py +0 -0
  7. biolib/_internal/add_copilot_prompts.py +58 -0
  8. biolib/_internal/add_gui_files.py +81 -0
  9. biolib/_internal/data_record/__init__.py +1 -0
  10. biolib/_internal/data_record/data_record.py +85 -0
  11. biolib/_internal/data_record/push_data.py +116 -0
  12. biolib/_internal/data_record/remote_storage_endpoint.py +43 -0
  13. biolib/_internal/errors.py +5 -0
  14. biolib/_internal/file_utils.py +125 -0
  15. biolib/_internal/fuse_mount/__init__.py +1 -0
  16. biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
  17. biolib/_internal/http_client.py +159 -0
  18. biolib/_internal/lfs/__init__.py +1 -0
  19. biolib/_internal/lfs/cache.py +51 -0
  20. biolib/_internal/libs/__init__.py +1 -0
  21. biolib/_internal/libs/fusepy/__init__.py +1257 -0
  22. biolib/_internal/push_application.py +488 -0
  23. biolib/_internal/runtime.py +22 -0
  24. biolib/_internal/string_utils.py +13 -0
  25. biolib/_internal/templates/__init__.py +1 -0
  26. biolib/_internal/templates/copilot_template/.github/instructions/general-app-knowledge.instructions.md +10 -0
  27. biolib/_internal/templates/copilot_template/.github/instructions/style-general.instructions.md +20 -0
  28. biolib/_internal/templates/copilot_template/.github/instructions/style-python.instructions.md +16 -0
  29. biolib/_internal/templates/copilot_template/.github/instructions/style-react-ts.instructions.md +47 -0
  30. biolib/_internal/templates/copilot_template/.github/prompts/biolib_app_inputs.prompt.md +11 -0
  31. biolib/_internal/templates/copilot_template/.github/prompts/biolib_onboard_repo.prompt.md +19 -0
  32. biolib/_internal/templates/copilot_template/.github/prompts/biolib_run_apps.prompt.md +12 -0
  33. biolib/_internal/templates/dashboard_template/.biolib/config.yml +5 -0
  34. biolib/_internal/templates/github_workflow_template/.github/workflows/biolib.yml +21 -0
  35. biolib/_internal/templates/gitignore_template/.gitignore +10 -0
  36. biolib/_internal/templates/gui_template/.yarnrc.yml +1 -0
  37. biolib/_internal/templates/gui_template/App.tsx +53 -0
  38. biolib/_internal/templates/gui_template/Dockerfile +27 -0
  39. biolib/_internal/templates/gui_template/biolib-sdk.ts +82 -0
  40. biolib/_internal/templates/gui_template/dev-data/output.json +7 -0
  41. biolib/_internal/templates/gui_template/index.css +5 -0
  42. biolib/_internal/templates/gui_template/index.html +13 -0
  43. biolib/_internal/templates/gui_template/index.tsx +10 -0
  44. biolib/_internal/templates/gui_template/package.json +27 -0
  45. biolib/_internal/templates/gui_template/tsconfig.json +24 -0
  46. biolib/_internal/templates/gui_template/vite-plugin-dev-data.ts +50 -0
  47. biolib/_internal/templates/gui_template/vite.config.mts +10 -0
  48. biolib/_internal/templates/init_template/.biolib/config.yml +19 -0
  49. biolib/_internal/templates/init_template/Dockerfile +14 -0
  50. biolib/_internal/templates/init_template/requirements.txt +1 -0
  51. biolib/_internal/templates/init_template/run.py +12 -0
  52. biolib/_internal/templates/init_template/run.sh +4 -0
  53. biolib/_internal/templates/templates.py +25 -0
  54. biolib/_internal/tree_utils.py +106 -0
  55. biolib/_internal/utils/__init__.py +65 -0
  56. biolib/_internal/utils/auth.py +46 -0
  57. biolib/_internal/utils/job_url.py +33 -0
  58. biolib/_internal/utils/multinode.py +263 -0
  59. biolib/_runtime/runtime.py +157 -0
  60. biolib/_session/session.py +44 -0
  61. biolib/_shared/__init__.py +0 -0
  62. biolib/_shared/types/__init__.py +74 -0
  63. biolib/_shared/types/account.py +12 -0
  64. biolib/_shared/types/account_member.py +8 -0
  65. biolib/_shared/types/app.py +9 -0
  66. biolib/_shared/types/data_record.py +40 -0
  67. biolib/_shared/types/experiment.py +32 -0
  68. biolib/_shared/types/file_node.py +17 -0
  69. biolib/_shared/types/push.py +6 -0
  70. biolib/_shared/types/resource.py +37 -0
  71. biolib/_shared/types/resource_deploy_key.py +11 -0
  72. biolib/_shared/types/resource_permission.py +14 -0
  73. biolib/_shared/types/resource_version.py +19 -0
  74. biolib/_shared/types/result.py +14 -0
  75. biolib/_shared/types/typing.py +10 -0
  76. biolib/_shared/types/user.py +19 -0
  77. biolib/_shared/utils/__init__.py +7 -0
  78. biolib/_shared/utils/resource_uri.py +75 -0
  79. biolib/api/__init__.py +6 -0
  80. biolib/api/client.py +168 -0
  81. biolib/app/app.py +252 -49
  82. biolib/app/search_apps.py +45 -0
  83. biolib/biolib_api_client/api_client.py +126 -31
  84. biolib/biolib_api_client/app_types.py +24 -4
  85. biolib/biolib_api_client/auth.py +31 -8
  86. biolib/biolib_api_client/biolib_app_api.py +147 -52
  87. biolib/biolib_api_client/biolib_job_api.py +161 -141
  88. biolib/biolib_api_client/job_types.py +21 -5
  89. biolib/biolib_api_client/lfs_types.py +7 -23
  90. biolib/biolib_api_client/user_state.py +56 -0
  91. biolib/biolib_binary_format/__init__.py +1 -4
  92. biolib/biolib_binary_format/file_in_container.py +105 -0
  93. biolib/biolib_binary_format/module_input.py +24 -7
  94. biolib/biolib_binary_format/module_output_v2.py +149 -0
  95. biolib/biolib_binary_format/remote_endpoints.py +34 -0
  96. biolib/biolib_binary_format/remote_stream_seeker.py +59 -0
  97. biolib/biolib_binary_format/saved_job.py +3 -2
  98. biolib/biolib_binary_format/{attestation_document.py → stdout_and_stderr.py} +8 -8
  99. biolib/biolib_binary_format/system_status_update.py +3 -2
  100. biolib/biolib_binary_format/utils.py +175 -0
  101. biolib/biolib_docker_client/__init__.py +11 -2
  102. biolib/biolib_errors.py +36 -0
  103. biolib/biolib_logging.py +27 -10
  104. biolib/cli/__init__.py +38 -0
  105. biolib/cli/auth.py +46 -0
  106. biolib/cli/data_record.py +164 -0
  107. biolib/cli/index.py +32 -0
  108. biolib/cli/init.py +421 -0
  109. biolib/cli/lfs.py +101 -0
  110. biolib/cli/push.py +50 -0
  111. biolib/cli/run.py +63 -0
  112. biolib/cli/runtime.py +14 -0
  113. biolib/cli/sdk.py +16 -0
  114. biolib/cli/start.py +56 -0
  115. biolib/compute_node/cloud_utils/cloud_utils.py +110 -161
  116. biolib/compute_node/job_worker/cache_state.py +66 -88
  117. biolib/compute_node/job_worker/cache_types.py +1 -6
  118. biolib/compute_node/job_worker/docker_image_cache.py +112 -37
  119. biolib/compute_node/job_worker/executors/__init__.py +0 -3
  120. biolib/compute_node/job_worker/executors/docker_executor.py +532 -199
  121. biolib/compute_node/job_worker/executors/docker_types.py +9 -1
  122. biolib/compute_node/job_worker/executors/types.py +19 -9
  123. biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py +30 -0
  124. biolib/compute_node/job_worker/job_max_runtime_timer_thread.py +3 -5
  125. biolib/compute_node/job_worker/job_storage.py +108 -0
  126. biolib/compute_node/job_worker/job_worker.py +397 -212
  127. biolib/compute_node/job_worker/large_file_system.py +87 -38
  128. biolib/compute_node/job_worker/network_alloc.py +99 -0
  129. biolib/compute_node/job_worker/network_buffer.py +240 -0
  130. biolib/compute_node/job_worker/utilization_reporter_thread.py +197 -0
  131. biolib/compute_node/job_worker/utils.py +9 -24
  132. biolib/compute_node/remote_host_proxy.py +400 -98
  133. biolib/compute_node/utils.py +31 -9
  134. biolib/compute_node/webserver/compute_node_results_proxy.py +189 -0
  135. biolib/compute_node/webserver/proxy_utils.py +28 -0
  136. biolib/compute_node/webserver/webserver.py +130 -44
  137. biolib/compute_node/webserver/webserver_types.py +2 -6
  138. biolib/compute_node/webserver/webserver_utils.py +77 -12
  139. biolib/compute_node/webserver/worker_thread.py +183 -42
  140. biolib/experiments/__init__.py +0 -0
  141. biolib/experiments/experiment.py +356 -0
  142. biolib/jobs/__init__.py +1 -0
  143. biolib/jobs/job.py +741 -0
  144. biolib/jobs/job_result.py +185 -0
  145. biolib/jobs/types.py +50 -0
  146. biolib/py.typed +0 -0
  147. biolib/runtime/__init__.py +14 -0
  148. biolib/sdk/__init__.py +91 -0
  149. biolib/tables.py +34 -0
  150. biolib/typing_utils.py +2 -7
  151. biolib/user/__init__.py +1 -0
  152. biolib/user/sign_in.py +54 -0
  153. biolib/utils/__init__.py +162 -0
  154. biolib/utils/cache_state.py +94 -0
  155. biolib/utils/multipart_uploader.py +194 -0
  156. biolib/utils/seq_util.py +150 -0
  157. biolib/utils/zip/remote_zip.py +640 -0
  158. pybiolib-1.2.1890.dist-info/METADATA +41 -0
  159. pybiolib-1.2.1890.dist-info/RECORD +177 -0
  160. {pybiolib-0.2.951.dist-info → pybiolib-1.2.1890.dist-info}/WHEEL +1 -1
  161. pybiolib-1.2.1890.dist-info/entry_points.txt +2 -0
  162. README.md +0 -17
  163. biolib/app/app_result.py +0 -68
  164. biolib/app/utils.py +0 -62
  165. biolib/biolib-js/0-biolib.worker.js +0 -1
  166. biolib/biolib-js/1-biolib.worker.js +0 -1
  167. biolib/biolib-js/2-biolib.worker.js +0 -1
  168. biolib/biolib-js/3-biolib.worker.js +0 -1
  169. biolib/biolib-js/4-biolib.worker.js +0 -1
  170. biolib/biolib-js/5-biolib.worker.js +0 -1
  171. biolib/biolib-js/6-biolib.worker.js +0 -1
  172. biolib/biolib-js/index.html +0 -10
  173. biolib/biolib-js/main-biolib.js +0 -1
  174. biolib/biolib_api_client/biolib_account_api.py +0 -21
  175. biolib/biolib_api_client/biolib_large_file_system_api.py +0 -108
  176. biolib/biolib_binary_format/aes_encrypted_package.py +0 -42
  177. biolib/biolib_binary_format/module_output.py +0 -58
  178. biolib/biolib_binary_format/rsa_encrypted_aes_package.py +0 -57
  179. biolib/biolib_push.py +0 -114
  180. biolib/cli.py +0 -203
  181. biolib/cli_utils.py +0 -273
  182. biolib/compute_node/cloud_utils/enclave_parent_types.py +0 -7
  183. biolib/compute_node/enclave/__init__.py +0 -2
  184. biolib/compute_node/enclave/enclave_remote_hosts.py +0 -53
  185. biolib/compute_node/enclave/nitro_secure_module_utils.py +0 -64
  186. biolib/compute_node/job_worker/executors/base_executor.py +0 -18
  187. biolib/compute_node/job_worker/executors/pyppeteer_executor.py +0 -173
  188. biolib/compute_node/job_worker/executors/remote/__init__.py +0 -1
  189. biolib/compute_node/job_worker/executors/remote/nitro_enclave_utils.py +0 -81
  190. biolib/compute_node/job_worker/executors/remote/remote_executor.py +0 -51
  191. biolib/lfs.py +0 -196
  192. biolib/pyppeteer/.circleci/config.yml +0 -100
  193. biolib/pyppeteer/.coveragerc +0 -3
  194. biolib/pyppeteer/.gitignore +0 -89
  195. biolib/pyppeteer/.pre-commit-config.yaml +0 -28
  196. biolib/pyppeteer/CHANGES.md +0 -253
  197. biolib/pyppeteer/CONTRIBUTING.md +0 -26
  198. biolib/pyppeteer/LICENSE +0 -12
  199. biolib/pyppeteer/README.md +0 -137
  200. biolib/pyppeteer/docs/Makefile +0 -177
  201. biolib/pyppeteer/docs/_static/custom.css +0 -28
  202. biolib/pyppeteer/docs/_templates/layout.html +0 -10
  203. biolib/pyppeteer/docs/changes.md +0 -1
  204. biolib/pyppeteer/docs/conf.py +0 -299
  205. biolib/pyppeteer/docs/index.md +0 -21
  206. biolib/pyppeteer/docs/make.bat +0 -242
  207. biolib/pyppeteer/docs/reference.md +0 -211
  208. biolib/pyppeteer/docs/server.py +0 -60
  209. biolib/pyppeteer/poetry.lock +0 -1699
  210. biolib/pyppeteer/pyppeteer/__init__.py +0 -135
  211. biolib/pyppeteer/pyppeteer/accessibility.py +0 -286
  212. biolib/pyppeteer/pyppeteer/browser.py +0 -401
  213. biolib/pyppeteer/pyppeteer/browser_fetcher.py +0 -194
  214. biolib/pyppeteer/pyppeteer/command.py +0 -22
  215. biolib/pyppeteer/pyppeteer/connection/__init__.py +0 -242
  216. biolib/pyppeteer/pyppeteer/connection/cdpsession.py +0 -101
  217. biolib/pyppeteer/pyppeteer/coverage.py +0 -346
  218. biolib/pyppeteer/pyppeteer/device_descriptors.py +0 -787
  219. biolib/pyppeteer/pyppeteer/dialog.py +0 -79
  220. biolib/pyppeteer/pyppeteer/domworld.py +0 -597
  221. biolib/pyppeteer/pyppeteer/emulation_manager.py +0 -53
  222. biolib/pyppeteer/pyppeteer/errors.py +0 -48
  223. biolib/pyppeteer/pyppeteer/events.py +0 -63
  224. biolib/pyppeteer/pyppeteer/execution_context.py +0 -156
  225. biolib/pyppeteer/pyppeteer/frame/__init__.py +0 -299
  226. biolib/pyppeteer/pyppeteer/frame/frame_manager.py +0 -306
  227. biolib/pyppeteer/pyppeteer/helpers.py +0 -245
  228. biolib/pyppeteer/pyppeteer/input.py +0 -371
  229. biolib/pyppeteer/pyppeteer/jshandle.py +0 -598
  230. biolib/pyppeteer/pyppeteer/launcher.py +0 -683
  231. biolib/pyppeteer/pyppeteer/lifecycle_watcher.py +0 -169
  232. biolib/pyppeteer/pyppeteer/models/__init__.py +0 -103
  233. biolib/pyppeteer/pyppeteer/models/_protocol.py +0 -12460
  234. biolib/pyppeteer/pyppeteer/multimap.py +0 -82
  235. biolib/pyppeteer/pyppeteer/network_manager.py +0 -678
  236. biolib/pyppeteer/pyppeteer/options.py +0 -8
  237. biolib/pyppeteer/pyppeteer/page.py +0 -1728
  238. biolib/pyppeteer/pyppeteer/pipe_transport.py +0 -59
  239. biolib/pyppeteer/pyppeteer/target.py +0 -147
  240. biolib/pyppeteer/pyppeteer/task_queue.py +0 -24
  241. biolib/pyppeteer/pyppeteer/timeout_settings.py +0 -36
  242. biolib/pyppeteer/pyppeteer/tracing.py +0 -93
  243. biolib/pyppeteer/pyppeteer/us_keyboard_layout.py +0 -305
  244. biolib/pyppeteer/pyppeteer/util.py +0 -18
  245. biolib/pyppeteer/pyppeteer/websocket_transport.py +0 -47
  246. biolib/pyppeteer/pyppeteer/worker.py +0 -101
  247. biolib/pyppeteer/pyproject.toml +0 -97
  248. biolib/pyppeteer/spell.txt +0 -137
  249. biolib/pyppeteer/tox.ini +0 -72
  250. biolib/pyppeteer/utils/generate_protocol_types.py +0 -603
  251. biolib/start_cli.py +0 -7
  252. biolib/utils.py +0 -47
  253. biolib/validators/validate_app_version.py +0 -183
  254. biolib/validators/validate_argument.py +0 -134
  255. biolib/validators/validate_module.py +0 -323
  256. biolib/validators/validate_zip_file.py +0 -40
  257. biolib/validators/validator_utils.py +0 -103
  258. pybiolib-0.2.951.dist-info/LICENSE +0 -21
  259. pybiolib-0.2.951.dist-info/METADATA +0 -61
  260. pybiolib-0.2.951.dist-info/RECORD +0 -153
  261. pybiolib-0.2.951.dist-info/entry_points.txt +0 -3
  262. /LICENSE → /pybiolib-1.2.1890.dist-info/licenses/LICENSE +0 -0
@@ -1,603 +0,0 @@
1
- """
2
- CLI script to generate Protocol types
3
- """
4
- import asyncio
5
- import json
6
- import logging
7
- import re
8
- import time
9
- from argparse import ArgumentParser
10
- from datetime import datetime
11
- from functools import partial
12
- from pathlib import Path
13
- from textwrap import dedent
14
- from typing import Any, Dict, Hashable, List, Literal, Match, Tuple, Union
15
-
16
- import networkx as nx
17
-
18
- from pyppeteer import launch
19
-
20
- handler = logging.StreamHandler()
21
- handler.setFormatter(logging.Formatter('[{levelname}] {name}: {message}', style='{'))
22
- logging.getLogger('pyppeteer').addHandler(handler)
23
-
24
- logger = logging.getLogger('CLI')
25
- logger.addHandler(handler)
26
- logger.setLevel(logging.INFO)
27
- handler.setLevel(logging.INFO)
28
-
29
-
30
- class ProtocolTypesGenerator:
31
- """
32
- Class for generating a black formatter-semi-compliant code relating relating to the CDP specification.
33
- Attributes:
34
- td_references: set of relations of one TypedDict to another. Needed to detect/resolve recursive references
35
- domains: List containing all the domains of the CDP protocol
36
- all_known_types: Dict containing all known types. Needed to resolve forward references.
37
- typed_dicts: Dict of typed_dict_name to TypedDictGenerator(). We need a dict so we can access arbitrary elements
38
- and update them if a recursive reference is found
39
- code_gen = instance of TypingCodeGenerator for recording code lines
40
- _extern_code_gen = instance of TypingCodeGenerator for actually recording code the will appear outside the if
41
- TYPE_CHECKING guard
42
- """
43
-
44
- _forward_ref_re = r'\'Protocol\.(\w+\.\w+)\''
45
- js_to_py_types = {
46
- 'any': 'Any',
47
- 'string': 'str',
48
- 'object': 'Dict[str, str]',
49
- 'boolean': 'bool',
50
- 'number': 'float',
51
- 'integer': 'int',
52
- 'binary': 'bytes',
53
- }
54
-
55
- def __init__(self):
56
- self.td_references = set()
57
- self.domains = []
58
- self.all_known_types = {}
59
- self.typed_dicts = {}
60
- self.code_gen = TypingCodeGenerator()
61
- self._extern_code_gen = TypingCodeGenerator()
62
-
63
- def _resolve_forward_ref_re_sub_repl(self, match: Match, fw_ref: bool) -> str:
64
- domain_, ref = match.group(1).split('.')
65
- resolved_fwref = self.all_known_types[domain_][ref]
66
-
67
- # resolve nested forward references
68
- if re.search(self._forward_ref_re, resolved_fwref):
69
- resolved_fwref = self.resolve_forward_ref_on_line(resolved_fwref)
70
- if (
71
- fw_ref
72
- and resolved_fwref not in self.js_to_py_types.values()
73
- and not resolved_fwref.startswith('Literal')
74
- and not resolved_fwref.startswith('List')
75
- ):
76
- # forward ref to a typed dict, not sure that it will be defined
77
- resolved_fwref = f'\'{resolved_fwref}\''
78
-
79
- return resolved_fwref
80
-
81
- def resolve_forward_ref_on_line(self, line: str, fw_ref: bool = True) -> str:
82
- """
83
- Replaces a forward reference in the form 'Protocol.domain.ref' to the actual value of Protocol.domain.ref
84
- Args:
85
- line: line in which protocol forward reference occurs.
86
- fw_ref: whether or not to forward reference the resolved reference
87
- Returns:
88
- str: line with resolved forward reference
89
- """
90
- # PyCharm can't handle partial
91
- # noinspection PyTypeChecker
92
- return re.sub(self._forward_ref_re, partial(self._resolve_forward_ref_re_sub_repl, fw_ref=fw_ref), line)
93
-
94
- async def _retrieve_top_level_domain(self):
95
- browser = await launch(args=['--no-sandbox', '--disable-setuid-sandbox'])
96
- base_endpoint = re.search(r'ws://([0-9A-Za-z:.]*)/', browser.wsEndpoint).group(1)
97
- page = await browser.newPage()
98
-
99
- logger.info(f'Loading raw protocol specification')
100
- t_start = time.perf_counter()
101
-
102
- await page.goto(f'http://{base_endpoint}/json/protocol')
103
- page_content = await page.evaluate('document.documentElement.innerText')
104
- try:
105
- await browser.close()
106
- except Exception as e:
107
- logger.warning(f'Exception on browser close: {e}')
108
-
109
- logger.info(f'Loading raw protocol specification in {time.perf_counter()-t_start:.2f}s')
110
- self.domains = json.loads(page_content)['domains']
111
-
112
- def retrieve_top_level_domain(self):
113
- """
114
- Fetches and data, parses it, and sets the class variable 'domains' to it for later use.
115
- Returns: None
116
- """
117
- asyncio.get_event_loop().run_until_complete(self._retrieve_top_level_domain())
118
-
119
- def gen_spec(self):
120
- """
121
- Generate the Protocol class file lines within self.code_gen attribute. Uses an IndentManager context manager to
122
- keep track of the current indentation level. Resolves all forward references. Expands recursive types to an
123
- approximation of the cyclic type referenced. Wraps protocol class in a if TYPE_CHECKING guard and provides a
124
- dummy protocol class for runtime for performance.
125
-
126
- Returns: None
127
- """
128
-
129
- self.generate_header_doc_string()
130
-
131
- logger.info(f'Generating protocol spec')
132
- t_start = time.perf_counter()
133
-
134
- self.code_gen.add_code('class Protocol:')
135
- with self.code_gen.indent_manager:
136
- for domain in self.domains:
137
- domain_name = domain['domain']
138
- self.code_gen.add_code(f'class {domain_name}:')
139
- self.all_known_types[domain_name] = {}
140
- with self.code_gen.indent_manager:
141
- self.code_gen.add_comment_from_info(domain)
142
- for type_info in domain.get('types', []):
143
- self.add_type_item(type_info, 'id', 'properties', domain_name)
144
-
145
- for payload in domain.get('events', []):
146
- payload["name"] = payload["name"] + 'Payload'
147
- self.add_type_item(payload, 'name', 'parameters', domain_name, type_conversion_fallback='None')
148
-
149
- for command_info in domain.get('commands', []):
150
- for td_key, suffix in (('properties', 'Parameters'), ('returns', 'ReturnValues')):
151
- command_info['name'] = re.subn(r'(Parameters$|$)', suffix, command_info['name'], 1)[0]
152
- self.add_type_item(
153
- command_info, 'name', td_key, domain_name, type_conversion_fallback='None'
154
- )
155
- self.code_gen.add_newlines(num=2)
156
-
157
- self.generate_overview()
158
- self.resolve_all_fw_refs()
159
-
160
- self.expand_recursive_references()
161
- self.typed_dicts = {k: v for k, v in sorted(self.typed_dicts.items(), key=lambda x: x[0])}
162
- # all typed dicts are inserted prior to the main Protocol class
163
- for index, td in enumerate(self.typed_dicts.values()):
164
- td.add_newlines(num=1)
165
- self.code_gen.add_code(lines=td.code_lines, lines_classification='inserted')
166
-
167
- self.code_gen.add_code(lines=self._extern_code_gen.code_lines)
168
- logger.info(f'Generated protocol spec in {time.perf_counter() - t_start:.2f}s')
169
-
170
- def generate_header_doc_string(self):
171
- """
172
- Generates a headers doc string for the top of the file.
173
- Returns: None
174
- """
175
- self.code_gen.add_code('"""', lines_classification='inserted')
176
- self.code_gen.add_code(
177
- f'''\
178
- Automatically generated by ./{Path(__file__).relative_to(Path(__file__).parents[1]).as_posix()}
179
- Attention! This file should *not* be modified directly! Instead, use the script to update it.
180
-
181
- Last regeneration: {datetime.utcnow()}''',
182
- lines_classification='inserted',
183
- )
184
- self.code_gen.add_code('"""', lines_classification='inserted')
185
-
186
- def resolve_all_fw_refs(self) -> None:
187
- """
188
- Resolves all forward reference to the root of said references eg 'Protocol.Animation.thingParams' -> thingParams
189
- Returns: None
190
- """
191
- # no need for copying list as we aren't adding/removing elements
192
- # resolve forward refs in main protocol class
193
- for index, line in enumerate(self.code_gen.code_lines):
194
- # skip empty lines or lines positively without forward reference
195
- if not line.strip() or 'Protocol' not in line:
196
- continue
197
- self.code_gen.code_lines[index] = self.resolve_forward_ref_on_line(line, fw_ref=False)
198
- # resolve forward refs in typed dicts, and store instances where s TypedDict references another
199
- for td_name, td in self.typed_dicts.items():
200
- for index, line in enumerate(td.code_lines):
201
- resolved_fw_ref = self.resolve_forward_ref_on_line(line)
202
- resolved_fw_ref_splits = resolved_fw_ref.split(': ')
203
- if len(resolved_fw_ref_splits) == 2: # only pay attention to actual resolve fw refs
204
- ref = resolved_fw_ref_splits[1]
205
- td_ref_re = r'^(?:List\[)?\'(\w+)\'\]?'
206
- if re.search(td_ref_re, ref):
207
- self.td_references.add((td_name, re.search(td_ref_re, ref).group(1)))
208
-
209
- self.typed_dicts[td_name].code_lines[index] = resolved_fw_ref
210
-
211
- def generate_overview(self) -> None:
212
- """
213
- Generate several convenience overview classes, listed in overview_info
214
- Returns: None
215
- """
216
- overview_info = {
217
- 'Events': ('events', '\'{domain}.{item_name}\'', True, False),
218
- 'CommandParameters': ('commands', '\'Protocol.{domain}.{item_name}\'', False, False),
219
- 'CommandReturnValues': ('commands', '\'Protocol.{domain}.{item_name}\'', False, False),
220
- 'CommandNames': ('commands', '\'{domain}.{item_name}\'', True, True),
221
- }
222
-
223
- last_overview_class_name = [*overview_info.keys()][-1]
224
- for overview_class_name, (domain_key, item_fmt, gen_externally, no_suffix) in overview_info.items():
225
- overview_code_gen = TypingCodeGenerator(init_imports=False)
226
- overview_code_gen.add_code(f'class {overview_class_name}:')
227
- with overview_code_gen.indent_manager:
228
- for domain in self.domains:
229
- if domain_key in domain:
230
- overview_code_gen.add_code(f'class {domain["domain"]}:')
231
- with overview_code_gen.indent_manager:
232
- for item in domain[domain_key]:
233
- if no_suffix:
234
- item['name'] = re.sub('(ReturnValues|Parameters)$', '', item['name'])
235
- formatted_name = item_fmt.format(domain=domain['domain'], item_name=item['name'])
236
- overview_code_gen.add_code(f'{item["name"]} = {formatted_name}')
237
- overview_code_gen.add_newlines(num=1)
238
- if overview_class_name != last_overview_class_name:
239
- # don't add newlines to EOF
240
- overview_code_gen.add_newlines(num=1)
241
- if gen_externally:
242
- self._extern_code_gen.add_code(lines=overview_code_gen.code_lines)
243
- else:
244
- self.code_gen.add_code(lines=overview_code_gen.code_lines)
245
-
246
- def add_type_item(
247
- self,
248
- type_info,
249
- type_name_key,
250
- td_key,
251
- domain_name,
252
- type_conversion_fallback: Union[str, Literal[False]] = False,
253
- ) -> None:
254
- """
255
- Adds a class attr based on type_info, type_name_key, td_key, domain_name, and type_conversion_fallback
256
-
257
- Args:
258
- type_info: Dict containing info pertaining to the type
259
- type_name_key: Key to the name of the type
260
- td_key: Key to the item which contains TypedDict info
261
- domain_name: Name of domain
262
- type_conversion_fallback: If not false, used when the type cannot be determined from type_info
263
-
264
- Returns: None
265
- """
266
- self.code_gen.add_comment_from_info(type_info)
267
- item_name = type_info[type_name_key]
268
- if td_key in type_info:
269
- td = self.generate_typed_dict(type_info, domain_name)
270
- self.typed_dicts.update(td)
271
- type_ = [*td.keys()][0]
272
- typed_dict = True
273
- else:
274
- try:
275
- type_ = self.convert_js_to_py_type(type_info, domain_name)
276
- except KeyError:
277
- if type_conversion_fallback is not False:
278
- type_ = type_conversion_fallback
279
- else:
280
- raise
281
- typed_dict = False
282
-
283
- self.all_known_types[domain_name][item_name] = type_
284
- if typed_dict:
285
- # https://github.com/python/mypy/issues/7866
286
- type_ = f'Union[{type_}]'
287
- self.code_gen.add_code(f'{item_name} = {type_}')
288
-
289
- def expand_recursive_references(self) -> None:
290
- """
291
- Expands recursive references to TypedDicts with Dict[str, Union[Dict[str, Any], str, bool, int, float, List]],
292
- and adds a comment with the actual type reference.
293
- Returns: None
294
- """
295
- expansion = 'Dict[str, Union[Dict[str, Any], str, bool, int, float, List]]'
296
- # todo: networkx will soon support sets: https://github.com/networkx/networkx/pull/3907
297
- for recursive_refs in nx.simple_cycles(nx.DiGraph([*self.td_references])):
298
- any_recursive_ref = "|".join(recursive_refs)
299
- for recursing_itm in recursive_refs:
300
- self.typed_dicts[recursing_itm].filter_lines(
301
- (rf'(\s+)(\w+): [\w\[]*?\'({any_recursive_ref})\'\]?', rf'\1# actual: \3\n\1\2: {expansion}'),
302
- )
303
-
304
- def write_generated_code(self, path: Path) -> None:
305
- """
306
- Write generated code lines to the specified path. Writes to a temporary file and checks that file with mypy to
307
- 'resolve' any cyclic references.
308
-
309
- Args:
310
- path: path to write type code to.
311
-
312
- Returns: None
313
- """
314
- if path.is_dir():
315
- path /= '_protocol.py'
316
- logger.info(f'Writing generated protocol code to {path}')
317
- # PyCharm can't handle this path type properly
318
- # noinspection PyTypeChecker
319
- with open(path, 'w') as p:
320
- p.write(str(self.code_gen))
321
-
322
- def generate_typed_dict(self, type_info: Dict[str, Any], domain_name: str) -> Dict[str, 'TypedDictGenerator']:
323
- """
324
- Generates TypedDicts based on type_info.
325
-
326
- Args:
327
- type_info: Dict containing the info for the TypedDict
328
- domain_name: path to resolve relative forward references in type_info against
329
-
330
- Returns: TypedDict corresponding to type information found in type_info
331
- """
332
- items = self._multi_fallback_get(type_info, 'returns', 'parameters', 'properties')
333
- td_name = self._multi_fallback_get(type_info, 'id', 'name')
334
- is_total = any(1 for x in items if x.get('optional'))
335
- td = TypedDictGenerator(td_name, is_total)
336
- doc_string = TypingCodeGenerator(init_imports=False)
337
- with td.indent_manager, doc_string.indent_manager:
338
- needs_closing_triple_q = False
339
- if 'description' in type_info or any('description' in item for item in items):
340
- doc_string.add_code('"""')
341
- needs_closing_triple_q = True
342
- if 'description' in type_info:
343
- doc_string.add_code(type_info['description'])
344
- doc_string.add_newlines(num=1)
345
- if any('description' in item for item in items):
346
- doc_string.add_code('Attributes:')
347
-
348
- for item in items:
349
- type_ = self.convert_js_to_py_type(item, domain_name)
350
- if 'description' in item:
351
- lines = item['description'].split('\n')
352
- with doc_string.indent_manager:
353
- doc_string.add_code(f'{item["name"]}: {lines[0]}')
354
- if len(lines) > 1:
355
- with doc_string.indent_manager:
356
- doc_string.add_code(lines=lines[1:])
357
- td.add_code(f'{item["name"]}: {type_}')
358
- if needs_closing_triple_q:
359
- doc_string.add_code('"""')
360
- td.insert_code(lines=doc_string.code_lines)
361
-
362
- return {td_name: td}
363
-
364
- @staticmethod
365
- def _multi_fallback_get(d: Dict[Hashable, Any], *k: Hashable) -> Any:
366
- """
367
- Convenience method to retrieve item from dict with multiple keys as fallbacks for failed accesses
368
-
369
- Args:
370
- d: Dict to retrieve values from
371
- k: keys of Dict to retrieve values from
372
- Returns: Any
373
- first found value where key in k
374
- """
375
- for key in k:
376
- if key in d:
377
- return d[key]
378
-
379
- raise KeyError(f'{", ".join([str(s) for s in k])} all not found in {d}')
380
-
381
- def convert_js_to_py_type(self, item_info: Union[Dict[str, Any], str], domain_name) -> str:
382
- """
383
- Generates a valid python type from the JS type. In the case of type_info being a str, we simply return the
384
- matching python type from self.js_to_py_types. Otherwise, in the case of type_info being a Dict, we know that
385
- it will contain vital information about the type we are trying to convert.
386
-
387
- The domain_name is used to qualify relative forward reference in type_info. For example, if
388
- type_info['$ref'] == 'foo', domain_name would be used produce an absolute forward reference, ie domain_name.foo
389
-
390
- Args:
391
- item_info: Dict or str containing type_info
392
- domain_name: path to resolve relative forward references in type_info against
393
- Returns: str
394
- valid python type, either in the form of an absolute forward reference (eg Protocol.bar.foo) or
395
- primitive type (eg int, float, str, etc)
396
- """
397
- if isinstance(item_info, str):
398
- type_ = self.js_to_py_types[item_info]
399
- elif 'items' in item_info:
400
- assert item_info['type'] == 'array'
401
- if '$ref' in item_info['items']:
402
- ref = item_info['items']['$ref']
403
- type_ = f'List[{self.get_forward_ref(ref, domain_name)}]'
404
- else:
405
- type_ = f'List[{self.convert_js_to_py_type(item_info["items"]["type"], domain_name)}]'
406
- else:
407
- if '$ref' in item_info:
408
- type_ = self.get_forward_ref(item_info['$ref'], domain_name)
409
- else:
410
- if 'enum' in item_info:
411
- _enum_vals = ', '.join([f'\'{x}\'' for x in item_info['enum']])
412
- type_ = f'Literal[{_enum_vals}]'
413
- else:
414
- type_ = self.js_to_py_types[item_info['type']]
415
-
416
- return type_
417
-
418
- @staticmethod
419
- def get_forward_ref(relative_ref: str, potential_domain_context: str) -> str:
420
- """
421
- Generates a forward absolute forward reference to Protocol class attr. If the reference is relative
422
- to a nested class, the full path is resolved against potential_domain_context. In the case of
423
- the reference being relative to the Protocol class, the path is simple resolved against the Protocol class
424
-
425
- Args:
426
- relative_ref: reference to another class, in the form of foo or foo.bar
427
- potential_domain_context: context to resolve class against if relative_ref is relative to it
428
-
429
- Returns: str
430
- absolute forward reference to nested class attr
431
- """
432
- if len(relative_ref.split('.')) == 2:
433
- non_fw_ref = f'Protocol.{relative_ref}'
434
- else:
435
- non_fw_ref = f'Protocol.{potential_domain_context}.{relative_ref}'
436
- return f'\'{non_fw_ref}\''
437
-
438
-
439
- class TypingCodeGenerator:
440
- """
441
- Class to facilitate the generation of typing related related code
442
-
443
- Attributes:
444
- indent_manager: instance of IndentManager. Used to manage the current indentation level
445
- import_lines: List containing lines of import code
446
- inserted_lines: List containing lines of code before the main body
447
- code_lines: List containing lines of code
448
- lines_classification: Classification of the lines, the value should be one such that self.<value>_lines is
449
- defined. The default is 'code'
450
- """
451
-
452
- def __init__(self, init_imports: bool = True):
453
- """
454
- Args:
455
- init_imports: Whether or not to write importing code to self.import_lines, defaults to True
456
- """
457
- self.indent_manager = IndentManager()
458
- self.import_lines = []
459
- self.inserted_lines = []
460
- self.code_lines = []
461
- self.lines_classification = 'code'
462
- if init_imports:
463
- self.init_imports()
464
-
465
- def init_imports(self) -> None:
466
- """
467
- Writes import code relating to typing to self.import_lines
468
- Returns: None
469
- """
470
- self.lines_classification = 'import'
471
- self.add_code('import sys')
472
- self.add_newlines(num=1)
473
- self.add_code('from typing import Any, Dict, List, TYPE_CHECKING, Union')
474
- self.add_newlines(num=1)
475
- self.add_code('if sys.version_info < (3, 8):')
476
- with self.indent_manager:
477
- self.add_code('from typing_extensions import Literal, TypedDict')
478
- self.add_code('else:')
479
- with self.indent_manager:
480
- self.add_code('from typing import Literal, TypedDict')
481
- self.add_newlines(num=1)
482
- self.lines_classification = 'code'
483
-
484
- def add_newlines(self, num: int = 1, lines_classification: str = None) -> None:
485
- """
486
- Adds num newlines
487
- Args:
488
- num: number of newlines to add
489
- lines_classification: str of lines_classification, defaults to self.lines_classification
490
-
491
- Returns: None
492
- """
493
- self.add_code('\n' * num, lines_classification)
494
-
495
- def add_comment_from_info(self, info: Dict[str, Any]) -> None:
496
- """
497
- Adds a comment to code_lines if a description is defined in info
498
- Args:
499
- info: Dict possible containing the key info
500
-
501
- Returns: None
502
- """
503
- if 'description' in info:
504
- newline = '\n'
505
- self.add_code(f'# {info["description"].replace(newline, " ")}')
506
-
507
- def add_code(self, code: str = None, lines: List[str] = None, lines_classification: str = None) -> None:
508
- """
509
- Adds code from a string or code from lines. If code is not None, lines is assigned to the str dedented and
510
- split by newlines. Each line in lines is applied the current indent level before being added to the
511
- <lines_classification>_lines list
512
- Args:
513
- code: str containing code
514
- lines: List[str] containing code
515
- lines_classification: str of lines_classification, defaults to self.lines_classification
516
-
517
- Returns: None
518
- """
519
- assert code is not None or lines is not None, 'One of code or lines must be specified'
520
- lines_classification = lines_classification or self.lines_classification
521
- if code is not None:
522
- if all([char == '\n' for char in code]):
523
- # if we are adding a newline, '\n'.split('\n') == ['', ''], which will expand to 2 newlines instead of
524
- lines = ['' for _ in range(code.count('\n'))]
525
- else:
526
- lines = dedent(code).split('\n')
527
- # don't indent empty lines
528
- lines = [f'{self.indent_manager if li else ""}{li}' for li in lines]
529
- self.__getattribute__(f'{lines_classification}_lines').extend(lines)
530
-
531
- def __str__(self):
532
- return '\n'.join([*self.import_lines, *self.inserted_lines, *self.code_lines])
533
-
534
-
535
- class TypedDictGenerator(TypingCodeGenerator):
536
- """
537
- Class to manage creation of TypedDicts specifically. Does not initialize imports.
538
- """
539
-
540
- def __init__(self, name: str, total: bool):
541
- super().__init__(init_imports=False)
542
- self.name = name
543
- self.total = total
544
- total_spec = ', total=False' if total else ''
545
- self.add_code(f'class {name}(TypedDict{total_spec}):')
546
-
547
- def filter_lines(self, *sub_pattern_replacements: Tuple[str, Any]) -> None:
548
- """
549
- Performs re.sub on all code_lines (except for the first one) according to sub_pattern_replacements. Each
550
- sub_sub_pattern_replacements should be a re pattern and regex replacement function/regex replacement string
551
- Args:
552
- sub_pattern_replacements: tuple containing a regex pattern and
553
- regex replacement function or regex replacement string
554
-
555
- Returns: None
556
- """
557
- # temp_code_lines = self.code_lines[:]
558
- # self.code_lines = [self.code_lines[0]]
559
- filtered_lines = []
560
- for index, line in enumerate(self.code_lines[1:]):
561
- for sub_p, sub_r in sub_pattern_replacements:
562
- line = re.sub(sub_p, sub_r, line)
563
- filtered_lines.extend(line.split('\n'))
564
- self.code_lines = self.code_lines[:1] + filtered_lines
565
-
566
- def insert_code(self, code: str = None, lines: List[str] = None, lines_classification: str = None) -> None:
567
- old_lines = self.code_lines[1:]
568
- self.code_lines = self.code_lines[:1]
569
- self.add_code(code, lines, lines_classification)
570
- self.add_code(lines=old_lines)
571
-
572
-
573
- class IndentManager:
574
- """
575
- Simple class which can be used with a with statement to increment/decrement the current indent level/
576
- """
577
-
578
- def __init__(self):
579
- self.indent = ''
580
-
581
- def __enter__(self) -> 'IndentManager':
582
- self.indent += ' '
583
- return self
584
-
585
- def __exit__(self, exc_type, exc_val, exc_tb) -> None:
586
- self.indent = self.indent[:-4]
587
-
588
- def __str__(self) -> str:
589
- return self.indent
590
-
591
-
592
- if __name__ == '__main__':
593
- parser = ArgumentParser()
594
- parser.add_argument(
595
- '--output',
596
- '-o',
597
- help='dir or file to output to',
598
- default=Path(__file__).parents[1] / 'pyppeteer' / 'models' / '_protocol.py',
599
- )
600
- generator = ProtocolTypesGenerator()
601
- generator.retrieve_top_level_domain()
602
- generator.gen_spec()
603
- generator.write_generated_code(path=parser.parse_args().output)
biolib/start_cli.py DELETED
@@ -1,7 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- import re
3
- import sys
4
- from biolib import call_cli
5
- if __name__ == '__main__':
6
- sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
7
- sys.exit(call_cli()) # type: ignore
biolib/utils.py DELETED
@@ -1,47 +0,0 @@
1
- import os
2
- from urllib.parse import urlparse
3
-
4
- from importlib_metadata import version, PackageNotFoundError
5
-
6
- # try fetching version, if it fails (usually when in dev), add default
7
- from biolib.biolib_logging import logger
8
-
9
- try:
10
- BIOLIB_PACKAGE_VERSION = version('pybiolib')
11
- except PackageNotFoundError:
12
- BIOLIB_PACKAGE_VERSION = '0.0.0'
13
-
14
- IS_DEV = os.getenv('BIOLIB_DEV', '').upper() == 'TRUE'
15
-
16
- BIOLIB_PACKAGE_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
17
-
18
- BIOLIB_CLOUD_ENVIRONMENT = os.getenv('BIOLIB_CLOUD_ENVIRONMENT', '').lower()
19
-
20
- BIOLIB_IS_RUNNING_IN_ENCLAVE = BIOLIB_CLOUD_ENVIRONMENT == 'enclave'
21
-
22
- IS_RUNNING_IN_CLOUD = BIOLIB_CLOUD_ENVIRONMENT in ('enclave', 'non-enclave')
23
-
24
- if BIOLIB_CLOUD_ENVIRONMENT and not IS_RUNNING_IN_CLOUD:
25
- logger.warning((
26
- 'BIOLIB_CLOUD_ENVIRONMENT defined but does not specify the cloud environment correctly. ',
27
- 'The compute node will not act as a cloud compute node'
28
- ))
29
-
30
- BIOLIB_CLOUD_SKIP_PCR_VERIFICATION = os.getenv('BIOLIB_CLOUD_SKIP_PCR_VERIFICATION', '').upper() == 'TRUE'
31
-
32
- RUN_DEV_JOB_ID = 'run-dev-mocked-job-id'
33
-
34
-
35
- def get_absolute_container_image_uri(base_url: str, relative_image_uri: str):
36
- if base_url == 'https://biolib.com':
37
- container_registry_hostname = 'containers.biolib.com'
38
- elif base_url in ('https://staging-elb.biolib.com', 'https://staging.biolib.com'):
39
- container_registry_hostname = 'containers.staging.biolib.com'
40
- else:
41
- # Expect registry to be accessible on the hostname of base_url if not running on biolib.com
42
- base_hostname = urlparse(base_url).hostname
43
- if not base_hostname:
44
- raise Exception("Could not get hostname from base_url. Tried to get ecr_proxy_uri for image pulling.")
45
- container_registry_hostname = base_hostname
46
-
47
- return f'{container_registry_hostname}/{relative_image_uri}'