minknow-api 6.10.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. minknow_api/__init__.py +955 -0
  2. minknow_api/_support.py +92 -0
  3. minknow_api/_version.py +1 -0
  4. minknow_api/acquisition_pb2.py +726 -0
  5. minknow_api/acquisition_pb2_grpc.py +699 -0
  6. minknow_api/acquisition_service.py +888 -0
  7. minknow_api/analysis_configuration_pb2.py +1439 -0
  8. minknow_api/analysis_configuration_pb2_grpc.py +905 -0
  9. minknow_api/analysis_configuration_service.py +1274 -0
  10. minknow_api/analysis_workflows_pb2.py +66 -0
  11. minknow_api/analysis_workflows_pb2_grpc.py +101 -0
  12. minknow_api/analysis_workflows_service.py +110 -0
  13. minknow_api/basecaller_pb2.py +577 -0
  14. minknow_api/basecaller_pb2_grpc.py +752 -0
  15. minknow_api/basecaller_service.py +1004 -0
  16. minknow_api/data.py +246 -0
  17. minknow_api/data_pb2.py +700 -0
  18. minknow_api/data_pb2_grpc.py +715 -0
  19. minknow_api/data_service.py +900 -0
  20. minknow_api/debug_pb2.py +106 -0
  21. minknow_api/debug_pb2_grpc.py +289 -0
  22. minknow_api/debug_service.py +325 -0
  23. minknow_api/device.py +76 -0
  24. minknow_api/device_pb2.py +856 -0
  25. minknow_api/device_pb2_grpc.py +1522 -0
  26. minknow_api/device_service.py +1688 -0
  27. minknow_api/examples/README.md +155 -0
  28. minknow_api/examples/__init__.py +0 -0
  29. minknow_api/examples/create_client_certificates.py +279 -0
  30. minknow_api/examples/export_to_csv.py +196 -0
  31. minknow_api/examples/extract_run_statistics.py +227 -0
  32. minknow_api/examples/generate_run_report.py +196 -0
  33. minknow_api/examples/list_flow_cell_check.py +122 -0
  34. minknow_api/examples/list_sequencing_positions.py +40 -0
  35. minknow_api/examples/load_sample_sheet.py +368 -0
  36. minknow_api/examples/manage_simulated_devices.py +260 -0
  37. minknow_api/examples/run_after_protocol.py +196 -0
  38. minknow_api/examples/start_flow_cell_check.py +168 -0
  39. minknow_api/examples/start_protocol.py +1080 -0
  40. minknow_api/hardware_check_pb2.py +201 -0
  41. minknow_api/hardware_check_pb2_grpc.py +281 -0
  42. minknow_api/hardware_check_service.py +324 -0
  43. minknow_api/instance_pb2.py +248 -0
  44. minknow_api/instance_pb2_grpc.py +498 -0
  45. minknow_api/instance_service.py +484 -0
  46. minknow_api/keystore_pb2.py +154 -0
  47. minknow_api/keystore_pb2_grpc.py +326 -0
  48. minknow_api/keystore_service.py +360 -0
  49. minknow_api/log_pb2.py +127 -0
  50. minknow_api/log_pb2_grpc.py +252 -0
  51. minknow_api/log_service.py +298 -0
  52. minknow_api/manager.py +813 -0
  53. minknow_api/manager_pb2.py +1097 -0
  54. minknow_api/manager_pb2_grpc.py +1640 -0
  55. minknow_api/manager_service.py +1856 -0
  56. minknow_api/minion_device_pb2.py +291 -0
  57. minknow_api/minion_device_pb2_grpc.py +199 -0
  58. minknow_api/minion_device_service.py +486 -0
  59. minknow_api/notifications_pb2.py +102 -0
  60. minknow_api/notifications_pb2_grpc.py +191 -0
  61. minknow_api/notifications_service.py +179 -0
  62. minknow_api/pebble_device_pb2.py +308 -0
  63. minknow_api/pebble_device_pb2_grpc.py +285 -0
  64. minknow_api/pebble_device_service.py +429 -0
  65. minknow_api/post_processing_protocol_connection.py +97 -0
  66. minknow_api/promethion_device_pb2.py +420 -0
  67. minknow_api/promethion_device_pb2_grpc.py +371 -0
  68. minknow_api/promethion_device_service.py +439 -0
  69. minknow_api/protocol_pb2.py +1003 -0
  70. minknow_api/protocol_pb2_grpc.py +1415 -0
  71. minknow_api/protocol_service.py +1602 -0
  72. minknow_api/protocol_settings_pb2.py +66 -0
  73. minknow_api/protocol_settings_pb2_grpc.py +29 -0
  74. minknow_api/protocol_settings_service.py +39 -0
  75. minknow_api/read_end_reason_pb2.py +27 -0
  76. minknow_api/read_end_reason_pb2_grpc.py +29 -0
  77. minknow_api/read_end_reason_service.py +50 -0
  78. minknow_api/report_data_pb2.py +157 -0
  79. minknow_api/report_data_pb2_grpc.py +29 -0
  80. minknow_api/report_data_service.py +53 -0
  81. minknow_api/rpc_options_pb2.py +26 -0
  82. minknow_api/run_until_pb2.py +197 -0
  83. minknow_api/run_until_pb2_grpc.py +699 -0
  84. minknow_api/run_until_service.py +433 -0
  85. minknow_api/statistics_pb2.py +785 -0
  86. minknow_api/statistics_pb2_grpc.py +673 -0
  87. minknow_api/statistics_service.py +995 -0
  88. minknow_api/testutils.py +228 -0
  89. minknow_api/tools/__init__.py +0 -0
  90. minknow_api/tools/any_helpers.py +130 -0
  91. minknow_api/tools/compatibility_helpers.py +9 -0
  92. minknow_api/tools/protocols.py +573 -0
  93. minknow_api/ui/__init__.py +0 -0
  94. minknow_api/ui/sequencing_run/__init__.py +0 -0
  95. minknow_api/ui/sequencing_run/presets_pb2.py +97 -0
  96. minknow_api/ui/sequencing_run/presets_pb2_grpc.py +332 -0
  97. minknow_api/ui/sequencing_run/presets_service.py +338 -0
  98. minknow_api/v2/__init__.py +0 -0
  99. minknow_api/v2/protocols_pb2.py +82 -0
  100. minknow_api/v2/protocols_pb2_grpc.py +507 -0
  101. minknow_api/v2/protocols_service.py +494 -0
  102. minknow_api-6.10.1.dist-info/METADATA +100 -0
  103. minknow_api-6.10.1.dist-info/RECORD +104 -0
  104. minknow_api-6.10.1.dist-info/WHEEL +5 -0
@@ -0,0 +1,955 @@
1
+ """
2
+ MinKNOW RPC Access
3
+ ==================
4
+
5
+ Provides access to MinKNOW via RPC.
6
+
7
+ This RPC system is gRPC-based. You might want to look at the `gRPC documentation
8
+ <http://www.grpc.io/grpc/python/>`_ for more information, but most of the detail is hidden by the
9
+ code in this module.
10
+
11
+ For external systems accessing MinKNOW, start with `minknow_api.manager.Manager` - this will allow
12
+ you to access high-level information, and enumerate and access each flow cell position.
13
+
14
+ The central class for accessing a flow cell position is `Connection`.
15
+
16
+ The functionality available on a flow cell position is divided into services, covering related units
17
+ like controlling the device or managing data acquisition. Each service is available as a property on
18
+ a `Connection` object. For each service, the related Protobuf messages are available from
19
+ ``minknow_api.<service>_service``, or as ``connection.<service>._pb`` (if ``connection`` is a
20
+ ``Connection`` object).
21
+
22
+ .. _rpc-services:
23
+
24
+ Services
25
+ --------
26
+
27
+ The available services are:
28
+
29
+ acquisition
30
+ Control data acquisition. See `acquisition_service.AcquisitionService` for a description of the
31
+ available methods.
32
+ analysis_configuration
33
+ Configure data acquisition. See `analysis_configuration_service.AnalysisConfigurationService`
34
+ for a description of the available methods.
35
+ analysis_workflows
36
+ Provides a way to manage analysis workflows that are ran after (or potentially during)
37
+ a protocol run. See `analysis_workflows_service.AnalysisWorkflowsService` for a description of
38
+ the available methods.
39
+ data
40
+ Stream acquisition data. Note that this is for data directly produced during acquisition, rather
41
+ than statistics about acquired data. See `data_service.DataService` for a description of the
42
+ available methods.
43
+ debug
44
+ Debugging data. Note that this is for data that is useful for debugging. See
45
+ `debug_service.DebugService` for a description of the available methods.
46
+ device
47
+ Get information about and control the attached device. This useful presents information and
48
+ settings in a device-independent way, so it can be used on PromethIONs as easily as on MinIONs.
49
+ See `device_service.DeviceService` for a description of the available methods.
50
+ keystore
51
+ A service for storing and retrieving arbitrary data on the instance. This can be used to
52
+ communicate with other users of the API. See `keystore_service.DeviceService` for a description
53
+ of the available methods.
54
+ hardware_check
55
+ A service for starting hardware checks and retrieving the data.
56
+ See `hardware_check_service.HardwareCheckService` for a description of the available methods.
57
+ instance
58
+ Get information about the instance of MinKNOW you are connected to (eg: software version). See
59
+ `instance_service.InstanceService` for a description of the available methods.
60
+ log
61
+ Get or produce general informational messages. See `log_service.LogService` for a description of
62
+ the available methods.
63
+ manager_service
64
+ Central service for managing individual sequencing positions. Can inspect information about
65
+ the host itself or information about sequencing positions. See `manager_service.ManagerService`
66
+ for a description of the available methods
67
+ minion_device
68
+ MinION-specific device interface. This exposes low-level settings for MinIONs and similar
69
+ devices (eg: GridIONs). See `minion_device_service.MinionDeviceService` for a
70
+ description of the available methods.
71
+ pebble_device
72
+ Pebble-specific device interface. This exposes low-level settings for Pebble.
73
+ See `pebble_device_service.PebbleDeviceService` for a
74
+ description of the available methods.
75
+ protocol
76
+ Control protocol scripts. See `protocol_service.ProtocolService` for a description of the
77
+ available methods.
78
+ promethion_device
79
+ PromethION-specific device interface. This exposes low-level settings for PromethIONs. See
80
+ `promethion_device_service.PromethionDeviceService` for a description of the available methods.
81
+ run_until
82
+ Get and set criteria, values and updates which relate to stopping an acquisition when certain
83
+ run-until criteria are met. See run_until_service.RunUntilService for a description of the
84
+ available methods.
85
+ statistics
86
+ Get statistics about an acquisition period. Statistics can be streamed live during acquisition,
87
+ or retrieved afterwards. See `statistics_service.StatisticsService` for a description of the
88
+ available methods.
89
+
90
+ Helpers
91
+ -------
92
+
93
+ The `minknow_api.data`, `minknow_api.device` and `minknow_api.manager` modules contain helpers for
94
+ working with the services of the same name. See the documentation for those modules for more
95
+ information.
96
+
97
+ """
98
+
99
+ from dataclasses import dataclass
100
+ import importlib
101
+ import json
102
+ import logging
103
+ import os
104
+ from pathlib import Path
105
+ import platform
106
+ import sys
107
+ import threading
108
+ import warnings
109
+ import datetime
110
+ from enum import Enum
111
+ from typing import Any, Dict, Optional, Tuple, Union, List
112
+
113
+ import grpc
114
+ import pyrfc3339
115
+
116
+ from . import data
117
+
118
+ # Try and import from minknow_api_production package
119
+ try:
120
+ from minknow_api_production import (
121
+ production_pb2,
122
+ production_pb2_grpc,
123
+ production_service,
124
+ )
125
+
126
+ sys.modules["minknow_api.production_service"] = production_service
127
+ sys.modules["minknow_api.production_pb2"] = production_pb2
128
+ sys.modules["minknow_api.production_pb2_grpc"] = production_pb2_grpc
129
+ except ImportError:
130
+ pass
131
+
132
+
133
+ #
134
+ # Services
135
+ #
136
+ @dataclass
137
+ class Service:
138
+ services: List[str]
139
+ anchor: str = ""
140
+ """If this service is defined under a sub-directory,
141
+ where is it? Should be in python module form - replace '/' with '.'"""
142
+
143
+
144
+ _services: Dict[str, Service] = {
145
+ "acquisition": Service(["AcquisitionService"]),
146
+ "analysis_configuration": Service(["AnalysisConfigurationService"]),
147
+ "analysis_workflows": Service(["AnalysisWorkflowsService"]),
148
+ "data": Service(["DataService"]),
149
+ "debug": Service(["DebugService"]),
150
+ "device": Service(["DeviceService"]),
151
+ "hardware_check": Service(["HardwareCheckService"]),
152
+ "instance": Service(["InstanceService"]),
153
+ "keystore": Service(["KeyStoreService"]),
154
+ "log": Service(["LogService"]),
155
+ "manager": Service(["ManagerService"]),
156
+ "minion_device": Service(["MinionDeviceService"]),
157
+ "pebble_device": Service(["PebbleDeviceService"]),
158
+ "production": Service(["ProductionService"]),
159
+ "promethion_device": Service(["PromethionDeviceService"]),
160
+ "protocol": Service(["ProtocolService"]),
161
+ "protocols": Service(["ProtocolsService"], ".v2"),
162
+ "presets": Service(["PresetsService"], ".ui.sequencing_run"),
163
+ "run_until": Service(["RunUntilService"]),
164
+ "statistics": Service(["StatisticsService"]),
165
+ }
166
+
167
+ _optional_services = ["production"]
168
+
169
+
170
+ #
171
+ # Module meta-information
172
+ #
173
+
174
+ __all__ = [name + "_service" for name in _services.keys()] + [
175
+ "Connection",
176
+ "LocalAuthTokenCredentials",
177
+ "data",
178
+ "device",
179
+ "load_grpc_credentials",
180
+ "get_local_authentication_token_file",
181
+ "grpc_credentials",
182
+ "read_ssl_certificate",
183
+ "manager",
184
+ "post_processing_protocol_connection",
185
+ ]
186
+
187
+ try:
188
+ from ._version import __version__
189
+ except ImportError:
190
+ __version__ = None
191
+
192
+
193
+ #
194
+ # Submodule imports
195
+ #
196
+ class SubmoduleType(Enum):
197
+ PB2 = 1
198
+ PB2_GRPC = 2
199
+ SERVICE = 3
200
+
201
+
202
+ # Import a submodule.
203
+ # name corresponds to a name as one of the keys in _services
204
+ # submodule_type corresponds to if its service, pb2 or pb2_grpc file
205
+ def _import_submodule(svc_name: str, svc: Service, submodule_type: SubmoduleType):
206
+ _submodule_type_str = {
207
+ SubmoduleType.PB2: "pb2",
208
+ SubmoduleType.PB2_GRPC: "pb2_grpc",
209
+ SubmoduleType.SERVICE: "service",
210
+ }
211
+ try:
212
+ module_type_str = _submodule_type_str[submodule_type]
213
+ except KeyError:
214
+ raise
215
+
216
+ # effectively does `import .{name}_service as {name}_service`
217
+ return importlib.import_module(
218
+ f".{svc_name}_{module_type_str}", __name__ + svc.anchor
219
+ )
220
+
221
+
222
+ for svc_name, svc in _services.items():
223
+ try:
224
+ globals()[f"{svc_name}_service"] = _import_submodule(
225
+ svc_name, svc, SubmoduleType.SERVICE
226
+ )
227
+ except ImportError:
228
+ if svc_name not in _optional_services:
229
+ raise
230
+
231
+ logger = logging.getLogger(__name__)
232
+
233
+ #
234
+ # Connection helpers
235
+ #
236
+
237
+ GRPC_CHANNEL_OPTIONS = [
238
+ ("grpc.max_send_message_length", 16 * 1024 * 1024),
239
+ ("grpc.max_receive_message_length", 16 * 1024 * 1024),
240
+ ("grpc.http2.min_time_between_pings_ms", 1000),
241
+ (
242
+ "grpc.ssl_target_name_override",
243
+ "localhost",
244
+ ), # that's what our cert's CN is
245
+ ]
246
+
247
+
248
+ class LocalAuthTokenCredentials(grpc.AuthMetadataPlugin):
249
+ """Token based auth that gets the token from a known
250
+ location on the local filesystem. So only clients that
251
+ have access to the local filesystem can read the token.
252
+
253
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
254
+ that function.
255
+ """
256
+
257
+ def __init__(self, local_auth_path):
258
+ super().__init__()
259
+ self.local_auth_path = local_auth_path
260
+ self.lock = threading.Lock()
261
+ self._refresh_local_token()
262
+
263
+ def _refresh_local_token(self) -> None:
264
+ with self.lock:
265
+ try:
266
+ with open(self.local_auth_path, "r") as f:
267
+ token_json = json.load(f)
268
+ self.token = token_json["token"]
269
+ # Remove 120 secs from expiry just to ensure a token refresh before the token
270
+ # actually expires
271
+ self.expires_at = pyrfc3339.parse(
272
+ token_json["expires"]
273
+ ) - datetime.timedelta(seconds=120)
274
+ logger.debug(
275
+ "Found local auth token %s[...], expires at %s",
276
+ self.token[:8],
277
+ token_json["expires"],
278
+ )
279
+ except Exception as e:
280
+ logger.warning("Local auth token unable to be refreshed: %s", e)
281
+ self.token = None
282
+ self.expires_at = None
283
+
284
+ def __call__(self, context, callback):
285
+ metadata = None
286
+ if self.token:
287
+ if self.expires_at:
288
+ now = datetime.datetime.now(datetime.timezone.utc)
289
+ if now >= self.expires_at:
290
+ self._refresh_local_token()
291
+
292
+ metadata = (("local-auth", self.token),)
293
+
294
+ callback(metadata, None)
295
+
296
+
297
+ class DeveloperApiTokenCredentials(grpc.AuthMetadataPlugin):
298
+ """Presents a developer API token to MinKNOW that was obtained from the MinKNOW UI.
299
+
300
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
301
+ that function.
302
+ """
303
+
304
+ def __init__(self, token):
305
+ self.token = token
306
+
307
+ def __call__(self, context, callback):
308
+ # it's not really a "local auth" token, but all the tokens share the same
309
+ # namespace so it will work anyway
310
+ metadata = (("local-auth", self.token),)
311
+ callback(metadata, None)
312
+
313
+
314
+ class ProtocolTokenCredentials(grpc.AuthMetadataPlugin):
315
+ """Simple token based auth that assumes that the token
316
+ will last long enough for the duration of the experiment.
317
+
318
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
319
+ that function.
320
+ """
321
+
322
+ def __init__(self, token):
323
+ self.token = token
324
+
325
+ def __call__(self, context, callback):
326
+ metadata = (("protocol-auth", self.token),)
327
+ callback(metadata, None)
328
+
329
+
330
+ def get_local_authentication_token_file(
331
+ host: str = "127.0.0.1", port: Optional[int] = None, ca_cert: Optional[bytes] = None
332
+ ) -> Optional[str]:
333
+ """Starts an isolated manager instance to retrieve the path
334
+ of the local authentication token file, which can then
335
+ be read to extract the local authentication token"""
336
+ if not port:
337
+ port = 9502
338
+
339
+ try:
340
+ if not ca_cert:
341
+ ca_cert = read_ssl_certificate()
342
+
343
+ ssl_creds = grpc.ssl_channel_credentials(ca_cert)
344
+
345
+ channel = grpc.secure_channel(
346
+ host + ":" + str(port),
347
+ ssl_creds,
348
+ # we need the ssl target name override
349
+ options=GRPC_CHANNEL_OPTIONS,
350
+ )
351
+
352
+ # Disable F821 for pyflakes as i believe it can't recognise our use of import_module
353
+ service = manager_service.ManagerService(channel) # noqa: F821
354
+ return service.local_authentication_token_path().path
355
+ except grpc.RpcError:
356
+ logger.debug(
357
+ f"Unable to connect to manager on port '{port}' to retrieve local auth token path",
358
+ exc_info=True,
359
+ )
360
+ return None
361
+
362
+
363
+ def get_protocol_token_credentials(
364
+ environ: Union[Dict[str, str], os._Environ] = os.environ
365
+ ) -> Optional[grpc.ChannelCredentials]:
366
+ """If running as a protocol in MinKNOW, get the protocol token used to authenticate
367
+ to MinKNOW.
368
+
369
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
370
+ that function.
371
+ """
372
+ token = environ.get("PROTOCOL_TOKEN")
373
+
374
+ if token:
375
+ return grpc.metadata_call_credentials(ProtocolTokenCredentials(token))
376
+ else:
377
+ logger.debug("No protocol token found")
378
+ return None
379
+
380
+
381
+ def _try_to_make_helpful_warning_about_lat_permissions(token_path: Path) -> bool:
382
+ if platform.system() != "Linux":
383
+ # any reasonable user should already have access on macOS and Windows
384
+ return False
385
+
386
+ try:
387
+ token_folder = token_path.parent
388
+ stat_info = token_folder.stat()
389
+
390
+ import grp
391
+
392
+ group = grp.getgrgid(stat_info.st_gid)[0]
393
+
394
+ logger.warning(
395
+ 'Could not access local authentication token at "%s", '
396
+ 'maybe you need to be added to the "%s" group?',
397
+ token_path,
398
+ group,
399
+ )
400
+ return True
401
+ except Exception:
402
+ return False
403
+
404
+
405
+ def get_local_auth_token_credentials(
406
+ manager_port: Optional[int], ca_cert: Optional[bytes] = None
407
+ ) -> Optional[grpc.ChannelCredentials]:
408
+ """Attempt to get the local authentication token.
409
+
410
+ This will return None if local guest mode is not enabled or if MinKNOW is not
411
+ running on the local machine.
412
+
413
+ Note that if this returns a token, it can *only* be used to connect to MinKNOW
414
+ running on localhost at the port given by `manager_port` (the default ports of 9501
415
+ or 9502 if `manager_port` is None).
416
+
417
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
418
+ that function.
419
+ """
420
+ local_auth_path = get_local_authentication_token_file(
421
+ port=manager_port, ca_cert=ca_cert
422
+ )
423
+ logger.debug("Retrieving local token from file: '%s'", local_auth_path)
424
+ if local_auth_path:
425
+ local_auth_path = Path(local_auth_path)
426
+ try:
427
+ if local_auth_path.exists():
428
+ return grpc.metadata_call_credentials(
429
+ LocalAuthTokenCredentials(local_auth_path)
430
+ )
431
+ else:
432
+ logger.warning(
433
+ 'Local authentication token should be at "%s", '
434
+ "but that file does not exist",
435
+ local_auth_path,
436
+ )
437
+ except OSError:
438
+ if not _try_to_make_helpful_warning_about_lat_permissions(local_auth_path):
439
+ logger.warning(
440
+ 'Could not access local authentication token at "%s", '
441
+ "check file permissions",
442
+ local_auth_path,
443
+ )
444
+
445
+ return None
446
+
447
+
448
+ def get_developer_api_token_credentials(
449
+ developer_api_token: str,
450
+ ) -> grpc.ChannelCredentials:
451
+ """Create a channel credentials object for the given developer API token.
452
+
453
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
454
+ that function.
455
+ """
456
+ if not isinstance(developer_api_token, str):
457
+ raise Exception(
458
+ "Invalid developer api token, expected str, got %s"
459
+ % developer_api_token.__class__
460
+ )
461
+ return grpc.metadata_call_credentials(
462
+ DeveloperApiTokenCredentials(developer_api_token)
463
+ )
464
+
465
+
466
+ def read_ssl_certificate(
467
+ environ: Union[Dict[str, str], os._Environ] = os.environ
468
+ ) -> bytes:
469
+ """Get the CA certificate that should be used to verify a TLS connection to MinKNOW.
470
+
471
+ If the ``MINKNOW_TRUSTED_CA`` environment variable is set to the path to an
472
+ existing file, its contents will be used.
473
+
474
+ If environment variable not set then make best guess for the platform/OS where
475
+ the location of the ca.crt will be fixed after installation.
476
+
477
+ You probably want to use `grpc_credentials()` instead. This is mostly a helper for
478
+ that function.
479
+ """
480
+ try:
481
+ with open(environ["MINKNOW_TRUSTED_CA"], "rb") as f:
482
+ return f.read()
483
+ except KeyError:
484
+ pass
485
+ except OSError:
486
+ logger.warning(
487
+ "$MINKNOW_TRUSTED_CA set but failed to read '%s'",
488
+ environ.get("MINKNOW_TRUSTED_CA"),
489
+ )
490
+
491
+ auto_cert_paths = []
492
+ os_name = platform.system()
493
+
494
+ # Provide best guess locations based on platform
495
+ if os_name == "Windows":
496
+ auto_cert_paths.append(Path(r"C:\data\rpc-certs\minknow\ca.crt"))
497
+ elif os_name == "Linux":
498
+ auto_cert_paths.append(Path("/data/rpc-certs/minknow/ca.crt"))
499
+ auto_cert_paths.append(Path("/var/lib/minknow/data/rpc-certs/minknow/ca.crt"))
500
+ elif os_name == "Darwin":
501
+ auto_cert_paths.append(Path("/Library/MinKNOW/data/rpc-certs/minknow/ca.crt"))
502
+ else:
503
+ pass
504
+
505
+ for cert in auto_cert_paths:
506
+ try:
507
+ with open(cert, "rb") as f:
508
+ return f.read()
509
+ except FileNotFoundError:
510
+ pass
511
+
512
+ msg = "Could not locate ca.crt file"
513
+ logger.error(msg)
514
+ raise Exception(msg)
515
+
516
+
517
+ def _is_localhost(host: str) -> bool:
518
+ # common cases:
519
+ if host == "localhost" or host == "localhost.localdomain":
520
+ return True
521
+ if host == "127.0.0.1" or host == "::1":
522
+ return True
523
+
524
+ # It is an IP address?
525
+ # IPv6 addresses in particular have multiple equivalent representations.
526
+ try:
527
+ import ipaddress
528
+
529
+ address = ipaddress.ip_address(host)
530
+ except ValueError:
531
+ # it wasn't an ip address
532
+ pass
533
+ else:
534
+ # it was an ip address: is it it the loopback address?
535
+ return address == ipaddress.IPv6Address(
536
+ "::1"
537
+ ) or address == ipaddress.IPv4Address("127.0.0.1")
538
+
539
+ # Is it just our own hostname?
540
+ try:
541
+ lowercase_host = host.lower()
542
+ import socket
543
+
544
+ if (
545
+ lowercase_host == socket.gethostname().lower()
546
+ or lowercase_host == socket.getfqdn().lower()
547
+ ):
548
+ return True
549
+ except OSError:
550
+ logger.exception(
551
+ 'Failed to get hostname (cannot check whether "%s" is localhost)', host
552
+ )
553
+
554
+ return False
555
+
556
+
557
+ def _try_client_cert_from_env_vars(
558
+ environ: Union[Dict[str, str], os._Environ] = os.environ
559
+ ) -> Tuple[Optional[bytes], Optional[bytes]]:
560
+ cert_chain_path = environ.get("MINKNOW_API_CLIENT_CERTIFICATE_CHAIN")
561
+ cert_key_path = environ.get("MINKNOW_API_CLIENT_KEY")
562
+ if cert_chain_path is None or cert_key_path is None:
563
+ if cert_chain_path is not None:
564
+ logger.warn(
565
+ "MINKNOW_API_CLIENT_CERTIFICATE_CHAIN environment variable set "
566
+ "but MINKNOW_API_CLIENT_KEY is not set"
567
+ )
568
+ elif cert_key_path is not None:
569
+ logger.warn(
570
+ "MINKNOW_API_CLIENT_KEY environment variable set "
571
+ "but MINKNOW_API_CLIENT_CERTIFICATE_CHAIN is not set"
572
+ )
573
+ return (None, None)
574
+
575
+ try:
576
+ with open(cert_chain_path, "rb") as f:
577
+ cert_chain = f.read()
578
+ except IOError as e:
579
+ logger.warn(
580
+ "Cannot load %s (from MINKNOW_API_CLIENT_CERTIFICATE_CHAIN): %s",
581
+ cert_chain_path,
582
+ e,
583
+ )
584
+ return (None, None)
585
+ try:
586
+ with open(cert_key_path, "rb") as f:
587
+ cert_key = f.read()
588
+ except IOError as e:
589
+ logger.warn(
590
+ "Cannot load %s (from MINKNOW_API_CLIENT_KEY): %s", cert_key_path, e
591
+ )
592
+ return (None, None)
593
+
594
+ return (cert_chain, cert_key)
595
+
596
+
597
+ def load_grpc_credentials(
598
+ manager_port: Optional[int] = None,
599
+ developer_api_token: Optional[str] = None,
600
+ host: Optional[str] = None,
601
+ client_certificate_chain: Optional[bytes] = None,
602
+ client_private_key: Optional[bytes] = None,
603
+ ca_certificate: Optional[bytes] = None,
604
+ _warning_stacklevel: int = 0,
605
+ environ: Union[Dict[str, str], os._Environ] = os.environ,
606
+ ) -> grpc.ChannelCredentials:
607
+ """Load gRPC credentials.
608
+
609
+ Args:
610
+ manager_port: The port the Manager is running on. This is used to find the
611
+ location of the local authentication token.
612
+ developer_api_token: A developer API token obtained from the UI for
613
+ authentication. This is deprecated, and it is recommended to use
614
+ `client_cert` and `client_private_key` instead.
615
+ host: The name of the host being connected to. This is used to decide whether to
616
+ try to use the local authentication token.
617
+ client_cert_chain: The (PEM-encoded) certificate chain linking
618
+ `client_private_key` to a certificate in ``conf/rpc-client-certs`` in the
619
+ MinKNOW installation directory.
620
+ client_private_key: The (PEM-encoded) private key for the first certificate in
621
+ `client_cert_chain`.
622
+ ca_certificate: The (PEM-encoded) root CA certificate.
623
+ environ: Optional dictionary containing global environment variables, if not provided
624
+ then os.environ is used.
625
+
626
+ This is the same as `grpc_credentials`, but does not cache the result. You should
627
+ probably be using that function instead.
628
+ """
629
+
630
+ if developer_api_token is not None:
631
+ warnings.warn(
632
+ "`developer_api_token` is deprecated",
633
+ DeprecationWarning,
634
+ stacklevel=_warning_stacklevel,
635
+ )
636
+
637
+ logger.debug("Reading ssl certificate")
638
+ if (client_certificate_chain is None) != (client_private_key is None):
639
+ raise TypeError(
640
+ "client_cert_chain and client_private_key must either both "
641
+ "be provided, or both be omitted"
642
+ )
643
+ if client_certificate_chain is None:
644
+ client_certificate_chain, client_private_key = _try_client_cert_from_env_vars(
645
+ environ
646
+ )
647
+ if ca_certificate is None:
648
+ ca_certificate = read_ssl_certificate(environ)
649
+
650
+ ssl_creds = grpc.ssl_channel_credentials(
651
+ root_certificates=ca_certificate,
652
+ private_key=client_private_key,
653
+ certificate_chain=client_certificate_chain,
654
+ )
655
+
656
+ # First use developer api token, if supplied:
657
+ call_creds = None
658
+ if developer_api_token is not None:
659
+ logger.debug("Using developer api token")
660
+ call_creds = get_developer_api_token_credentials(developer_api_token)
661
+
662
+ # Next try and get a token from the local filesystem if available:
663
+ if not call_creds:
664
+ local_token_override = environ.get("MINKNOW_API_USE_LOCAL_TOKEN")
665
+ if local_token_override is None:
666
+ # (allow host==None for backwards compatibility)
667
+ try_local_token = host is None or _is_localhost(host)
668
+ elif local_token_override.lower() in ("", "0", "no"):
669
+ try_local_token = False
670
+ else:
671
+ try_local_token = True
672
+ if try_local_token:
673
+ logger.debug("Getting local token")
674
+ call_creds = get_local_auth_token_credentials(manager_port, ca_certificate)
675
+
676
+ if not call_creds:
677
+ # No local token, so check if there is a token provided from starting as a
678
+ # protocol.
679
+ # NB: We try this *after* the local token because Bream currently needs to talk
680
+ # to the manager (where the protocol token doesn't work) - see CORE-3605 - but
681
+ # this may change in the future.
682
+ logger.debug("Getting protocol token")
683
+ call_creds = get_protocol_token_credentials(environ)
684
+
685
+ if call_creds:
686
+ grpc_creds = grpc.composite_channel_credentials(ssl_creds, call_creds)
687
+ else:
688
+ grpc_creds = ssl_creds
689
+
690
+ return grpc_creds
691
+
692
+
693
+ _grpc_credentials_cache: Dict[Any, grpc.ChannelCredentials] = dict()
694
+
695
+
696
+ def grpc_credentials(
697
+ manager_port: Optional[int] = None,
698
+ developer_api_token: Optional[str] = None,
699
+ host: Optional[str] = None,
700
+ client_certificate_chain: Optional[bytes] = None,
701
+ client_private_key: Optional[bytes] = None,
702
+ _warning_stacklevel: int = 0,
703
+ ca_certificate: Optional[bytes] = None,
704
+ environ: Union[Dict[str, str], os._Environ] = os.environ,
705
+ ) -> grpc.ChannelCredentials:
706
+ """Get a grpc.ChannelCredentials object for connecting to secure versions of MinKNOW"s gRPC
707
+ services.
708
+
709
+ Args:
710
+ manager_port: The port the Manager is running on. This is used for caching and
711
+ for finding the location of the local authentication token.
712
+ developer_api_token: A developer API token obtained from the UI for
713
+ authentication. This is deprecated, and it is recommended to use
714
+ `client_cert` and `client_private_key` instead.
715
+ host: The name of the host being connected to. This is used for caching and also
716
+ to decide whether to try to use the local authentication token.
717
+ client_certificate_chain: The (PEM-encoded) certificate chain linking
718
+ `client_private_key` to a certificate in ``conf/rpc-client-certs`` in the
719
+ MinKNOW installation directory.
720
+ client_private_key: The (PEM-encoded) private key for the first certificate in
721
+ `client_cert_chain`.
722
+ ca_certificate: The (PEM-encoded) root CA certificate.
723
+ environ: Optional dictionary containing global environment variables, if not provided
724
+ then os.environ is used.
725
+
726
+ Use like:
727
+
728
+ >>> import grpc
729
+ >>> channel = grpc.secure_channel("localhost:9502", grpc_credentials())
730
+
731
+ If run from the Python embedded in MinKNOW, this will find the correct certificate
732
+ automatically. Otherwise, you may need to set the ``MINKNOW_TRUSTED_CA`` to point to the CA
733
+ certificate used by MinKNOW (by default found under ``<data_dir>/rpc-certs/minknow/ca.crt``).
734
+ """
735
+ cache_key = (
736
+ manager_port,
737
+ developer_api_token,
738
+ host,
739
+ client_certificate_chain,
740
+ client_private_key,
741
+ ca_certificate,
742
+ )
743
+
744
+ global _grpc_credentials_cache
745
+
746
+ try:
747
+ creds = _grpc_credentials_cache[cache_key]
748
+ logger.debug("Using grpc credentials with cache key: (%s)", cache_key)
749
+ return creds
750
+ except KeyError:
751
+ pass
752
+
753
+ creds = load_grpc_credentials(
754
+ manager_port,
755
+ developer_api_token,
756
+ host,
757
+ client_certificate_chain,
758
+ client_private_key,
759
+ ca_certificate,
760
+ _warning_stacklevel=_warning_stacklevel + 1,
761
+ environ=environ,
762
+ )
763
+ _grpc_credentials_cache[cache_key] = creds
764
+ return creds
765
+
766
+
767
+ def clear_credentials_cache() -> None:
768
+ """
769
+ Clears the gRPC credentials cache.
770
+
771
+ This can be useful in tests, such as when the manager is being restarted.
772
+ """
773
+ global _grpc_credentials_cache
774
+ _grpc_credentials_cache = dict()
775
+
776
+
777
+ #
778
+ # Connection class
779
+ #
780
+
781
+
782
+ class Connection(object):
783
+ """A connection to a MinKNOW flow cell sequencing position via RPC.
784
+
785
+ The port for a given flow cell position is provided by the manager service. See
786
+ `minknow_api.manager.Manager`.
787
+
788
+ Args:
789
+ port: The port to connect to.
790
+ host: The host MinKNOW is running on (defaults to localhost).
791
+ credentials: gRPC credentials to use. If None, then
792
+ minknow_api.grpc_credentials() is called (using the other arguments) to
793
+ obtain credentials.
794
+ developer_api_token: A token obtained from the MinKNOW UI. Will be used to
795
+ authorise to MinKNOW if provided. This is deprecated, and it is recommended
796
+ to use `client_certificate_chain` and `client_private_key` instead. Note: if
797
+ `credentials` is provided, this parameter is ignored.
798
+ client_certificate_chain: The (PEM-encoded) certificate chain linking
799
+ `client_private_key` to a certificate in ``conf/rpc-client-certs`` in
800
+ the MinKNOW installation directory. Note: if `credentials` is provided,
801
+ this parameter is ignored.
802
+ client_private_key: The (PEM-encoded) private key for the first certificate
803
+ in `client_cert_chain`. Note: if `credentials` is provided, this
804
+ parameter is ignored.
805
+ ca_certificate: The (PEM-encoded) root CA certificate. Note: if `credentials` is
806
+ provided, this parameter is ignored.
807
+ environ: Optional dictionary containing global environment variables, if not provided
808
+ then os.environ is used.
809
+
810
+ If no port is provided, the MINKNOW_RPC_PORT environment variable will be used
811
+ (MinKNOW sets this when running protocol scripts, for example). If this environment
812
+ variable does not exist (or is not a number), an exception will be raised.
813
+
814
+ Each service is available as a property of the same name on the Connection
815
+ object. See :ref:`rpc-services` for a list.
816
+
817
+ Given a connection object ``connection``, for each service,
818
+ ``connection.<service>`` is a "service object". This exposes the RPC methods
819
+ for that service in a more convenient form than gRPC's own Python bindings
820
+ do.
821
+
822
+ For example, when calling ``start_protocol`` on the ``protocol`` service,
823
+ instead of doing
824
+
825
+ >>> protocol_service.start_protocol(
826
+ >>> protocol_service._pb.StartProtocolMessage(path="my_script"))
827
+
828
+ you can do
829
+
830
+ >>> connection.protocol.start_protocol(path="my_script")
831
+
832
+ Note that you must use keyword arguments - no positional parameters are
833
+ available.
834
+
835
+ This "unwrapping" of request messages only happens at one level, however. If
836
+ you want to change the target temperature settings on a MinION, you need to do
837
+ something like
838
+
839
+ >>> temp_settings = connection.minion_device._pb.TemperatureRange(min=37.0, max=37.0)
840
+ >>> connection.minion_device.change_settings(
841
+ >>> temperature_target=temp_settings)
842
+ """
843
+
844
+ def __init__(
845
+ self,
846
+ port: Optional[int] = None,
847
+ host: str = "127.0.0.1",
848
+ credentials: Optional[grpc.ChannelCredentials] = None,
849
+ developer_api_token: Optional[str] = None,
850
+ client_certificate_chain: Optional[bytes] = None,
851
+ client_private_key: Optional[bytes] = None,
852
+ ca_certificate: Optional[bytes] = None,
853
+ environ: Union[Dict[str, str], os._Environ] = os.environ,
854
+ ):
855
+ import time
856
+ import grpc
857
+
858
+ self.environ = environ
859
+
860
+ self.host = host
861
+ if port is None:
862
+ port = int(self.environ["MINKNOW_RPC_PORT_SECURE"])
863
+ self.port = port
864
+
865
+ if credentials is not None:
866
+ if developer_api_token is not None:
867
+ warnings.warn(
868
+ "`developer_api_token` ignored as `credentials` was provided"
869
+ )
870
+ if client_certificate_chain is not None or client_private_key is not None:
871
+ warnings.warn(
872
+ "`client_certificate_chain` and `client_private_key` ignored as `credentials` was provided"
873
+ )
874
+ if ca_certificate is not None:
875
+ warnings.warn("`ca_certificate` ignored as `credentials` was provided")
876
+
877
+ error = None
878
+ retry_count = 5
879
+ for i in range(retry_count):
880
+ if not credentials:
881
+ try:
882
+ manager_port = int(self.environ["MINKNOW_MANAGER_TEST_PORT"])
883
+ except KeyError:
884
+ manager_port = None
885
+ credentials = grpc_credentials(
886
+ manager_port=manager_port,
887
+ developer_api_token=developer_api_token,
888
+ host=host,
889
+ client_certificate_chain=client_certificate_chain,
890
+ client_private_key=client_private_key,
891
+ ca_certificate=ca_certificate,
892
+ _warning_stacklevel=1,
893
+ environ=self.environ,
894
+ )
895
+
896
+ self.channel = grpc.secure_channel(
897
+ f"{host}:{port}",
898
+ credentials=credentials,
899
+ options=GRPC_CHANNEL_OPTIONS,
900
+ )
901
+
902
+ # One entry for each service
903
+ for name, svc in _services.items():
904
+ for svc_class_name in svc.services:
905
+ try:
906
+ # effectively does `self.{name} = {name}_service.{svc_class_name}(self.channel)`
907
+ setattr(
908
+ self,
909
+ name,
910
+ getattr(globals()[f"{name}_service"], svc_class_name)(
911
+ self.channel
912
+ ),
913
+ )
914
+ except KeyError:
915
+ if name not in _optional_services:
916
+ raise
917
+
918
+ # Ensure channel is ready for communication
919
+ try:
920
+ logger.debug("Calling get_version_info to test connection")
921
+ self.instance.get_version_info()
922
+ error = None
923
+ break
924
+ except grpc.RpcError as e:
925
+ logger.info("Error received from rpc")
926
+ if (
927
+ e.code() == grpc.StatusCode.INTERNAL
928
+ and e.details() == "GOAWAY received"
929
+ ):
930
+ logger.warning(
931
+ "Failed to connect to minknow instance (retry %s/%s): %s",
932
+ i + 1,
933
+ retry_count,
934
+ e.details(),
935
+ )
936
+ elif e.code() == grpc.StatusCode.UNAVAILABLE:
937
+ logger.warning(
938
+ "Failed to connect to minknow instance (retry %s/%s): %s",
939
+ i + 1,
940
+ retry_count,
941
+ e.details(),
942
+ )
943
+ else:
944
+ raise
945
+ error = e
946
+ time.sleep(0.5)
947
+
948
+ if error:
949
+ raise error
950
+
951
+ def __enter__(self):
952
+ return self
953
+
954
+ def __exit__(self, exc_type, exc_value, traceback):
955
+ self.channel.close()