pyetp 0.0.46__py3-none-any.whl → 0.0.47__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- energistics/__init__.py +0 -0
- energistics/etp/__init__.py +0 -0
- energistics/etp/v12/__init__.py +0 -0
- energistics/etp/v12/datatypes/__init__.py +25 -0
- energistics/etp/v12/datatypes/data_array_types/__init__.py +27 -0
- energistics/etp/v12/datatypes/object/__init__.py +22 -0
- energistics/etp/v12/protocol/__init__.py +0 -0
- energistics/etp/v12/protocol/core/__init__.py +19 -0
- energistics/etp/v12/protocol/data_array/__init__.py +51 -0
- energistics/etp/v12/protocol/dataspace/__init__.py +23 -0
- energistics/etp/v12/protocol/discovery/__init__.py +21 -0
- energistics/etp/v12/protocol/store/__init__.py +27 -0
- energistics/etp/v12/protocol/transaction/__init__.py +27 -0
- pyetp/_version.py +2 -2
- pyetp/client.py +218 -172
- pyetp/errors.py +39 -0
- {pyetp-0.0.46.dist-info → pyetp-0.0.47.dist-info}/METADATA +1 -1
- pyetp-0.0.47.dist-info/RECORD +39 -0
- {pyetp-0.0.46.dist-info → pyetp-0.0.47.dist-info}/WHEEL +1 -1
- {pyetp-0.0.46.dist-info → pyetp-0.0.47.dist-info}/top_level.txt +2 -0
- rddms_io/__init__.py +0 -0
- rddms_io/client.py +1234 -0
- rddms_io/data_types.py +11 -0
- resqml_objects/v201/generated.py +44 -2
- resqml_objects/v201/utils.py +38 -0
- pyetp-0.0.46.dist-info/RECORD +0 -22
- {pyetp-0.0.46.dist-info → pyetp-0.0.47.dist-info}/licenses/LICENSE.md +0 -0
pyetp/client.py
CHANGED
|
@@ -6,7 +6,7 @@ import typing as T
|
|
|
6
6
|
import uuid
|
|
7
7
|
import warnings
|
|
8
8
|
from collections import defaultdict
|
|
9
|
-
from collections.abc import AsyncGenerator
|
|
9
|
+
from collections.abc import AsyncGenerator, Generator
|
|
10
10
|
from types import TracebackType
|
|
11
11
|
|
|
12
12
|
import numpy as np
|
|
@@ -16,138 +16,89 @@ import websockets.client
|
|
|
16
16
|
from etpproto.connection import CommunicationProtocol, ConnectionType, ETPConnection
|
|
17
17
|
from etpproto.messages import Message, MessageFlags
|
|
18
18
|
from etptypes import ETPModel
|
|
19
|
-
from
|
|
20
|
-
from
|
|
19
|
+
from pydantic import SecretStr
|
|
20
|
+
from xtgeo import RegularSurface
|
|
21
|
+
|
|
22
|
+
import resqml_objects.v201 as ro
|
|
23
|
+
from energistics.etp.v12.datatypes import (
|
|
24
|
+
AnyArrayType,
|
|
21
25
|
AnyLogicalArrayType,
|
|
26
|
+
ArrayOfString,
|
|
27
|
+
DataValue,
|
|
28
|
+
ErrorInfo,
|
|
29
|
+
SupportedDataObject,
|
|
30
|
+
SupportedProtocol,
|
|
31
|
+
Uuid,
|
|
32
|
+
Version,
|
|
22
33
|
)
|
|
23
|
-
from
|
|
24
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.data_array_identifier import (
|
|
34
|
+
from energistics.etp.v12.datatypes.data_array_types import (
|
|
25
35
|
DataArrayIdentifier,
|
|
26
|
-
)
|
|
27
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.data_array_metadata import (
|
|
28
36
|
DataArrayMetadata,
|
|
29
|
-
)
|
|
30
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.get_data_subarrays_type import (
|
|
31
37
|
GetDataSubarraysType,
|
|
32
|
-
)
|
|
33
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_arrays_type import (
|
|
34
38
|
PutDataArraysType,
|
|
35
|
-
)
|
|
36
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_subarrays_type import (
|
|
37
39
|
PutDataSubarraysType,
|
|
38
|
-
)
|
|
39
|
-
from etptypes.energistics.etp.v12.datatypes.data_array_types.put_uninitialized_data_array_type import (
|
|
40
40
|
PutUninitializedDataArrayType,
|
|
41
41
|
)
|
|
42
|
-
from
|
|
43
|
-
|
|
44
|
-
from etptypes.energistics.etp.v12.datatypes.object.context_info import ContextInfo
|
|
45
|
-
from etptypes.energistics.etp.v12.datatypes.object.context_scope_kind import (
|
|
42
|
+
from energistics.etp.v12.datatypes.object import (
|
|
43
|
+
ContextInfo,
|
|
46
44
|
ContextScopeKind,
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
from etptypes.energistics.etp.v12.datatypes.object.dataspace import Dataspace
|
|
50
|
-
from etptypes.energistics.etp.v12.datatypes.object.relationship_kind import (
|
|
45
|
+
DataObject,
|
|
46
|
+
Dataspace,
|
|
51
47
|
RelationshipKind,
|
|
48
|
+
Resource,
|
|
52
49
|
)
|
|
53
|
-
from
|
|
54
|
-
|
|
55
|
-
SupportedDataObject,
|
|
56
|
-
)
|
|
57
|
-
from etptypes.energistics.etp.v12.datatypes.supported_protocol import SupportedProtocol
|
|
58
|
-
from etptypes.energistics.etp.v12.datatypes.uuid import Uuid
|
|
59
|
-
from etptypes.energistics.etp.v12.datatypes.version import Version
|
|
60
|
-
from etptypes.energistics.etp.v12.protocol.core.authorize import Authorize
|
|
61
|
-
from etptypes.energistics.etp.v12.protocol.core.authorize_response import (
|
|
50
|
+
from energistics.etp.v12.protocol.core import (
|
|
51
|
+
Authorize,
|
|
62
52
|
AuthorizeResponse,
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
from etptypes.energistics.etp.v12.protocol.core.open_session import OpenSession
|
|
66
|
-
from etptypes.energistics.etp.v12.protocol.core.protocol_exception import (
|
|
53
|
+
CloseSession,
|
|
54
|
+
OpenSession,
|
|
67
55
|
ProtocolException,
|
|
56
|
+
RequestSession,
|
|
68
57
|
)
|
|
69
|
-
from
|
|
70
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata import (
|
|
58
|
+
from energistics.etp.v12.protocol.data_array import (
|
|
71
59
|
GetDataArrayMetadata,
|
|
72
|
-
)
|
|
73
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata_response import (
|
|
74
60
|
GetDataArrayMetadataResponse,
|
|
75
|
-
)
|
|
76
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays import (
|
|
77
61
|
GetDataArrays,
|
|
78
|
-
)
|
|
79
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays_response import (
|
|
80
62
|
GetDataArraysResponse,
|
|
81
|
-
)
|
|
82
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays import (
|
|
83
63
|
GetDataSubarrays,
|
|
84
|
-
)
|
|
85
|
-
from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays_response import (
|
|
86
64
|
GetDataSubarraysResponse,
|
|
87
|
-
)
|
|
88
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays import (
|
|
89
65
|
PutDataArrays,
|
|
90
|
-
)
|
|
91
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays_response import (
|
|
92
66
|
PutDataArraysResponse,
|
|
93
|
-
)
|
|
94
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays import (
|
|
95
67
|
PutDataSubarrays,
|
|
96
|
-
)
|
|
97
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays_response import (
|
|
98
68
|
PutDataSubarraysResponse,
|
|
99
|
-
)
|
|
100
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays import (
|
|
101
69
|
PutUninitializedDataArrays,
|
|
102
|
-
)
|
|
103
|
-
from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays_response import (
|
|
104
70
|
PutUninitializedDataArraysResponse,
|
|
105
71
|
)
|
|
106
|
-
from
|
|
72
|
+
from energistics.etp.v12.protocol.dataspace import (
|
|
107
73
|
DeleteDataspaces,
|
|
108
|
-
)
|
|
109
|
-
from etptypes.energistics.etp.v12.protocol.dataspace.delete_dataspaces_response import (
|
|
110
74
|
DeleteDataspacesResponse,
|
|
111
|
-
|
|
112
|
-
from etptypes.energistics.etp.v12.protocol.dataspace.get_dataspaces import GetDataspaces
|
|
113
|
-
from etptypes.energistics.etp.v12.protocol.dataspace.get_dataspaces_response import (
|
|
75
|
+
GetDataspaces,
|
|
114
76
|
GetDataspacesResponse,
|
|
115
|
-
|
|
116
|
-
from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces import PutDataspaces
|
|
117
|
-
from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces_response import (
|
|
77
|
+
PutDataspaces,
|
|
118
78
|
PutDataspacesResponse,
|
|
119
79
|
)
|
|
120
|
-
from
|
|
121
|
-
|
|
122
|
-
|
|
80
|
+
from energistics.etp.v12.protocol.discovery import (
|
|
81
|
+
GetResources,
|
|
82
|
+
GetResourcesResponse,
|
|
123
83
|
)
|
|
124
|
-
from
|
|
84
|
+
from energistics.etp.v12.protocol.store import (
|
|
85
|
+
DeleteDataObjects,
|
|
125
86
|
DeleteDataObjectsResponse,
|
|
126
|
-
|
|
127
|
-
from etptypes.energistics.etp.v12.protocol.store.get_data_objects import GetDataObjects
|
|
128
|
-
from etptypes.energistics.etp.v12.protocol.store.get_data_objects_response import (
|
|
87
|
+
GetDataObjects,
|
|
129
88
|
GetDataObjectsResponse,
|
|
130
|
-
|
|
131
|
-
from etptypes.energistics.etp.v12.protocol.store.put_data_objects import PutDataObjects
|
|
132
|
-
from etptypes.energistics.etp.v12.protocol.store.put_data_objects_response import (
|
|
89
|
+
PutDataObjects,
|
|
133
90
|
PutDataObjectsResponse,
|
|
134
91
|
)
|
|
135
|
-
from
|
|
92
|
+
from energistics.etp.v12.protocol.transaction import (
|
|
136
93
|
CommitTransaction,
|
|
137
|
-
)
|
|
138
|
-
from etptypes.energistics.etp.v12.protocol.transaction.rollback_transaction import (
|
|
139
94
|
RollbackTransaction,
|
|
140
|
-
)
|
|
141
|
-
from etptypes.energistics.etp.v12.protocol.transaction.start_transaction import (
|
|
142
95
|
StartTransaction,
|
|
96
|
+
StartTransactionResponse,
|
|
143
97
|
)
|
|
144
|
-
from pydantic import SecretStr
|
|
145
|
-
from xtgeo import RegularSurface
|
|
146
|
-
|
|
147
|
-
import resqml_objects.v201 as ro
|
|
148
98
|
from pyetp import utils_arrays, utils_xml
|
|
149
99
|
from pyetp._version import version
|
|
150
100
|
from pyetp.config import SETTINGS
|
|
101
|
+
from pyetp.errors import ETPTransactionFailure
|
|
151
102
|
from pyetp.uri import DataObjectURI, DataspaceURI
|
|
152
103
|
from resqml_objects import parse_resqml_v201_object, serialize_resqml_v201_object
|
|
153
104
|
|
|
@@ -223,11 +174,11 @@ class ETPClient(ETPConnection):
|
|
|
223
174
|
# client
|
|
224
175
|
#
|
|
225
176
|
|
|
226
|
-
async def send(self, body: ETPModel):
|
|
177
|
+
async def send(self, body: ETPModel) -> list[ETPModel]:
|
|
227
178
|
correlation_id = await self._send(body)
|
|
228
179
|
return await self._recv(correlation_id)
|
|
229
180
|
|
|
230
|
-
async def _send(self, body: ETPModel):
|
|
181
|
+
async def _send(self, body: ETPModel) -> int:
|
|
231
182
|
msg = Message.get_object_message(body, message_flags=MessageFlags.FINALPART)
|
|
232
183
|
if msg is None:
|
|
233
184
|
raise TypeError(f"{type(body)} not valid etp protocol")
|
|
@@ -246,37 +197,11 @@ class ETPClient(ETPConnection):
|
|
|
246
197
|
|
|
247
198
|
return msg.header.message_id
|
|
248
199
|
|
|
249
|
-
async def _recv(self, correlation_id: int) -> ETPModel:
|
|
200
|
+
async def _recv(self, correlation_id: int) -> list[ETPModel]:
|
|
250
201
|
assert correlation_id in self._recv_events, (
|
|
251
202
|
"Trying to receive a response on non-existing message"
|
|
252
203
|
)
|
|
253
204
|
|
|
254
|
-
def timeout_intervals(etp_timeout):
|
|
255
|
-
# Local function generating progressively longer timeout intervals.
|
|
256
|
-
|
|
257
|
-
# Use the timeout-interval generator from the Python websockets
|
|
258
|
-
# library.
|
|
259
|
-
backoff_generator = websockets.client.backoff(
|
|
260
|
-
initial_delay=5.0, min_delay=5.0, max_delay=20.0
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
# Check if we should never time out.
|
|
264
|
-
if etp_timeout is None:
|
|
265
|
-
# This is an infinite generator, so it should never exit.
|
|
266
|
-
yield from backoff_generator
|
|
267
|
-
return
|
|
268
|
-
|
|
269
|
-
# Generate timeout intervals until we have reached the
|
|
270
|
-
# `etp_timeout`-threshold.
|
|
271
|
-
csum = 0.0
|
|
272
|
-
for d in backoff_generator:
|
|
273
|
-
yield d
|
|
274
|
-
|
|
275
|
-
csum += d
|
|
276
|
-
|
|
277
|
-
if csum >= etp_timeout:
|
|
278
|
-
break
|
|
279
|
-
|
|
280
205
|
for ti in timeout_intervals(self.etp_timeout):
|
|
281
206
|
try:
|
|
282
207
|
# Wait for an event for `ti` seconds.
|
|
@@ -327,11 +252,7 @@ class ETPClient(ETPConnection):
|
|
|
327
252
|
"Server responded with ETPErrors:", ETPError.from_protos(errors)
|
|
328
253
|
)
|
|
329
254
|
|
|
330
|
-
|
|
331
|
-
logger.warning(f"Recived {len(bodies)} messages, but only expected one")
|
|
332
|
-
|
|
333
|
-
# ok
|
|
334
|
-
return bodies[0]
|
|
255
|
+
return bodies
|
|
335
256
|
|
|
336
257
|
@staticmethod
|
|
337
258
|
def _parse_error_info(bodies: list[ETPModel]) -> list[ErrorInfo]:
|
|
@@ -416,7 +337,7 @@ class ETPClient(ETPConnection):
|
|
|
416
337
|
|
|
417
338
|
logger.debug("Client closed")
|
|
418
339
|
|
|
419
|
-
async def close(self):
|
|
340
|
+
async def close(self) -> None:
|
|
420
341
|
"""Closing method that tears down the ETP-connection via the
|
|
421
342
|
`ETPClient.__aexit__`-method, and closes the websockets connection.
|
|
422
343
|
This method should _only_ be used if the user has set up a connection
|
|
@@ -473,7 +394,7 @@ class ETPClient(ETPConnection):
|
|
|
473
394
|
|
|
474
395
|
return "store"
|
|
475
396
|
|
|
476
|
-
|
|
397
|
+
msgs = await self.send(
|
|
477
398
|
RequestSession(
|
|
478
399
|
applicationName=self.application_name,
|
|
479
400
|
applicationVersion=self.application_version,
|
|
@@ -497,6 +418,10 @@ class ETPClient(ETPConnection):
|
|
|
497
418
|
),
|
|
498
419
|
)
|
|
499
420
|
)
|
|
421
|
+
|
|
422
|
+
assert len(msgs) == 1
|
|
423
|
+
msg = msgs[0]
|
|
424
|
+
|
|
500
425
|
assert msg and isinstance(msg, OpenSession)
|
|
501
426
|
|
|
502
427
|
self.is_connected = True
|
|
@@ -510,12 +435,15 @@ class ETPClient(ETPConnection):
|
|
|
510
435
|
async def authorize(
|
|
511
436
|
self, authorization: str, supplemental_authorization: T.Mapping[str, str] = {}
|
|
512
437
|
):
|
|
513
|
-
|
|
438
|
+
msgs = await self.send(
|
|
514
439
|
Authorize(
|
|
515
440
|
authorization=authorization,
|
|
516
441
|
supplementalAuthorization=supplemental_authorization,
|
|
517
442
|
)
|
|
518
443
|
)
|
|
444
|
+
assert len(msgs) == 1
|
|
445
|
+
msg = msgs[0]
|
|
446
|
+
|
|
519
447
|
assert msg and isinstance(msg, AuthorizeResponse)
|
|
520
448
|
|
|
521
449
|
return msg
|
|
@@ -547,8 +475,10 @@ class ETPClient(ETPConnection):
|
|
|
547
475
|
raise Exception("Max one / in dataspace name")
|
|
548
476
|
return DataspaceURI.from_name(ds)
|
|
549
477
|
|
|
550
|
-
def list_objects(
|
|
551
|
-
|
|
478
|
+
async def list_objects(
|
|
479
|
+
self, dataspace_uri: DataspaceURI | str, depth: int = 1
|
|
480
|
+
) -> GetResourcesResponse:
|
|
481
|
+
responses = await self.send(
|
|
552
482
|
GetResources(
|
|
553
483
|
scope=ContextScopeKind.TARGETS_OR_SELF,
|
|
554
484
|
context=ContextInfo(
|
|
@@ -559,6 +489,8 @@ class ETPClient(ETPConnection):
|
|
|
559
489
|
),
|
|
560
490
|
)
|
|
561
491
|
)
|
|
492
|
+
assert len(responses) == 1
|
|
493
|
+
return responses[0]
|
|
562
494
|
|
|
563
495
|
#
|
|
564
496
|
# dataspace
|
|
@@ -567,10 +499,17 @@ class ETPClient(ETPConnection):
|
|
|
567
499
|
async def get_dataspaces(
|
|
568
500
|
self, store_last_write_filter: int = None
|
|
569
501
|
) -> GetDataspacesResponse:
|
|
570
|
-
|
|
502
|
+
responses = await self.send(
|
|
571
503
|
GetDataspaces(store_last_write_filter=store_last_write_filter)
|
|
572
504
|
)
|
|
573
505
|
|
|
506
|
+
assert all(
|
|
507
|
+
[isinstance(response, GetDataspacesResponse) for response in responses]
|
|
508
|
+
), "Expected GetDataspacesResponse"
|
|
509
|
+
assert len(responses) == 1
|
|
510
|
+
|
|
511
|
+
return responses[0]
|
|
512
|
+
|
|
574
513
|
async def put_dataspaces(
|
|
575
514
|
self,
|
|
576
515
|
legaltags: list[str],
|
|
@@ -578,13 +517,13 @@ class ETPClient(ETPConnection):
|
|
|
578
517
|
owners: list[str],
|
|
579
518
|
viewers: list[str],
|
|
580
519
|
*dataspace_uris: DataspaceURI,
|
|
581
|
-
):
|
|
520
|
+
) -> dict[str, str]:
|
|
582
521
|
_uris = list(map(DataspaceURI.from_any, dataspace_uris))
|
|
583
522
|
for i in _uris:
|
|
584
523
|
if i.raw_uri.count("/") > 4: # includes the 3 eml
|
|
585
524
|
raise Exception("Max one / in dataspace name")
|
|
586
525
|
time = self.timestamp
|
|
587
|
-
|
|
526
|
+
responses = await self.send(
|
|
588
527
|
PutDataspaces(
|
|
589
528
|
dataspaces={
|
|
590
529
|
d.raw_uri: Dataspace(
|
|
@@ -607,15 +546,19 @@ class ETPClient(ETPConnection):
|
|
|
607
546
|
}
|
|
608
547
|
)
|
|
609
548
|
)
|
|
610
|
-
assert
|
|
611
|
-
|
|
612
|
-
)
|
|
549
|
+
assert all(
|
|
550
|
+
[isinstance(response, PutDataspacesResponse) for response in responses]
|
|
551
|
+
), "Expected PutDataspacesResponse"
|
|
613
552
|
|
|
614
|
-
|
|
615
|
-
|
|
553
|
+
successes = {}
|
|
554
|
+
for response in responses:
|
|
555
|
+
successes = {**successes, **response.success}
|
|
556
|
+
|
|
557
|
+
assert len(successes) == len(dataspace_uris), (
|
|
558
|
+
f"expected {len(dataspace_uris)} successes"
|
|
616
559
|
)
|
|
617
560
|
|
|
618
|
-
return
|
|
561
|
+
return successes
|
|
619
562
|
|
|
620
563
|
async def put_dataspaces_no_raise(
|
|
621
564
|
self,
|
|
@@ -624,22 +567,30 @@ class ETPClient(ETPConnection):
|
|
|
624
567
|
owners: list[str],
|
|
625
568
|
viewers: list[str],
|
|
626
569
|
*dataspace_uris: DataspaceURI,
|
|
627
|
-
):
|
|
570
|
+
) -> dict[str, str]:
|
|
628
571
|
try:
|
|
629
572
|
return await self.put_dataspaces(
|
|
630
573
|
legaltags, otherRelevantDataCountries, owners, viewers, *dataspace_uris
|
|
631
574
|
)
|
|
632
575
|
except ETPError:
|
|
633
|
-
|
|
576
|
+
return {}
|
|
634
577
|
|
|
635
|
-
async def delete_dataspaces(self, *dataspace_uris: DataspaceURI):
|
|
578
|
+
async def delete_dataspaces(self, *dataspace_uris: DataspaceURI) -> dict[str, str]:
|
|
636
579
|
_uris = list(map(str, dataspace_uris))
|
|
637
580
|
|
|
638
|
-
|
|
639
|
-
assert
|
|
640
|
-
|
|
581
|
+
responses = await self.send(DeleteDataspaces(uris=dict(zip(_uris, _uris))))
|
|
582
|
+
assert all(
|
|
583
|
+
[isinstance(response, DeleteDataspacesResponse) for response in responses]
|
|
584
|
+
), "Expected DeleteDataspacesResponse"
|
|
585
|
+
|
|
586
|
+
successes = {}
|
|
587
|
+
for response in responses:
|
|
588
|
+
successes = {**successes, **response.success}
|
|
589
|
+
|
|
590
|
+
assert len(successes) == len(dataspace_uris), (
|
|
591
|
+
f"expected {len(dataspace_uris)} successes"
|
|
641
592
|
)
|
|
642
|
-
return
|
|
593
|
+
return successes
|
|
643
594
|
|
|
644
595
|
async def get_data_objects(self, *uris: T.Union[DataObjectURI, str]):
|
|
645
596
|
tasks = []
|
|
@@ -647,7 +598,8 @@ class ETPClient(ETPConnection):
|
|
|
647
598
|
task = self.send(GetDataObjects(uris={str(uri): str(uri)}))
|
|
648
599
|
tasks.append(task)
|
|
649
600
|
|
|
650
|
-
|
|
601
|
+
task_responses = await asyncio.gather(*tasks)
|
|
602
|
+
responses = [r for tr in task_responses for r in tr]
|
|
651
603
|
assert len(responses) == len(uris)
|
|
652
604
|
|
|
653
605
|
data_objects = []
|
|
@@ -680,7 +632,8 @@ class ETPClient(ETPConnection):
|
|
|
680
632
|
)
|
|
681
633
|
tasks.append(task)
|
|
682
634
|
|
|
683
|
-
|
|
635
|
+
task_responses = await asyncio.gather(*tasks)
|
|
636
|
+
responses = [r for tr in task_responses for r in tr]
|
|
684
637
|
|
|
685
638
|
errors = []
|
|
686
639
|
for response in responses:
|
|
@@ -742,12 +695,16 @@ class ETPClient(ETPConnection):
|
|
|
742
695
|
):
|
|
743
696
|
_uris = list(map(str, uris))
|
|
744
697
|
|
|
745
|
-
|
|
698
|
+
responses = await self.send(
|
|
746
699
|
DeleteDataObjects(
|
|
747
700
|
uris=dict(zip(_uris, _uris)),
|
|
748
701
|
prune_contained_objects=prune_contained_objects,
|
|
749
702
|
)
|
|
750
703
|
)
|
|
704
|
+
|
|
705
|
+
assert len(responses) == 1
|
|
706
|
+
response = responses[0]
|
|
707
|
+
|
|
751
708
|
assert isinstance(response, DeleteDataObjectsResponse), (
|
|
752
709
|
"Expected DeleteDataObjectsResponse"
|
|
753
710
|
)
|
|
@@ -755,23 +712,34 @@ class ETPClient(ETPConnection):
|
|
|
755
712
|
return response.deleted_uris
|
|
756
713
|
|
|
757
714
|
async def start_transaction(
|
|
758
|
-
self, dataspace_uri: DataspaceURI, read_only: bool = True
|
|
715
|
+
self, dataspace_uri: DataspaceURI | str, read_only: bool = True
|
|
759
716
|
) -> Uuid:
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
)
|
|
717
|
+
dataspace_uri = str(DataspaceURI.from_any(dataspace_uri))
|
|
718
|
+
responses = await self.send(
|
|
719
|
+
StartTransaction(read_only=read_only, dataspace_uris=[dataspace_uri])
|
|
764
720
|
)
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
721
|
+
assert all(
|
|
722
|
+
[isinstance(response, StartTransactionResponse) for response in responses]
|
|
723
|
+
), "Expected StartTransactionResponse"
|
|
724
|
+
|
|
725
|
+
assert len(responses) == 1
|
|
726
|
+
response = responses[0]
|
|
727
|
+
|
|
728
|
+
if not response.successful:
|
|
729
|
+
raise ETPTransactionFailure(f"Failed starting transaction {dataspace_uri}")
|
|
730
|
+
|
|
731
|
+
return response.transaction_uuid
|
|
769
732
|
|
|
770
733
|
async def commit_transaction(self, transaction_uuid: Uuid):
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
734
|
+
responses = await self.send(
|
|
735
|
+
CommitTransaction(transaction_uuid=transaction_uuid)
|
|
736
|
+
)
|
|
737
|
+
assert len(responses) == 1
|
|
738
|
+
response = responses[0]
|
|
739
|
+
|
|
740
|
+
if response.successful is False:
|
|
741
|
+
raise ETPTransactionFailure(response.failure_reason)
|
|
742
|
+
return response
|
|
775
743
|
|
|
776
744
|
async def rollback_transaction(self, transaction_id: Uuid):
|
|
777
745
|
return await self.send(RollbackTransaction(transactionUuid=transaction_id))
|
|
@@ -877,9 +845,12 @@ class ETPClient(ETPConnection):
|
|
|
877
845
|
#
|
|
878
846
|
|
|
879
847
|
async def get_array_metadata(self, *uids: DataArrayIdentifier):
|
|
880
|
-
|
|
848
|
+
responses = await self.send(
|
|
881
849
|
GetDataArrayMetadata(dataArrays={i.path_in_resource: i for i in uids})
|
|
882
850
|
)
|
|
851
|
+
assert len(responses) == 1
|
|
852
|
+
response = responses[0]
|
|
853
|
+
|
|
883
854
|
assert isinstance(response, GetDataArrayMetadataResponse)
|
|
884
855
|
|
|
885
856
|
if len(response.array_metadata) != len(uids):
|
|
@@ -899,9 +870,13 @@ class ETPClient(ETPConnection):
|
|
|
899
870
|
):
|
|
900
871
|
return await self._get_array_chunked(uid)
|
|
901
872
|
|
|
902
|
-
|
|
873
|
+
responses = await self.send(
|
|
903
874
|
GetDataArrays(dataArrays={uid.path_in_resource: uid})
|
|
904
875
|
)
|
|
876
|
+
|
|
877
|
+
assert len(responses) == 1
|
|
878
|
+
response = responses[0]
|
|
879
|
+
|
|
905
880
|
assert isinstance(response, GetDataArraysResponse), (
|
|
906
881
|
"Expected GetDataArraysResponse"
|
|
907
882
|
)
|
|
@@ -920,10 +895,13 @@ class ETPClient(ETPConnection):
|
|
|
920
895
|
path_in_resource=path_in_resource,
|
|
921
896
|
)
|
|
922
897
|
|
|
923
|
-
|
|
898
|
+
responses = await self.send(
|
|
924
899
|
GetDataArrayMetadata(data_arrays={dai.path_in_resource: dai}),
|
|
925
900
|
)
|
|
926
901
|
|
|
902
|
+
assert len(responses) == 1
|
|
903
|
+
response = responses[0]
|
|
904
|
+
|
|
927
905
|
self.assert_response(response, GetDataArrayMetadataResponse)
|
|
928
906
|
assert (
|
|
929
907
|
len(response.array_metadata) == 1
|
|
@@ -986,7 +964,12 @@ class ETPClient(ETPConnection):
|
|
|
986
964
|
)
|
|
987
965
|
tasks.append(task)
|
|
988
966
|
|
|
989
|
-
|
|
967
|
+
task_responses = await asyncio.gather(*tasks)
|
|
968
|
+
responses = [
|
|
969
|
+
response
|
|
970
|
+
for task_response in task_responses
|
|
971
|
+
for response in task_response
|
|
972
|
+
]
|
|
990
973
|
|
|
991
974
|
data_blocks = []
|
|
992
975
|
for i, response in enumerate(responses):
|
|
@@ -1021,10 +1004,13 @@ class ETPClient(ETPConnection):
|
|
|
1021
1004
|
return data
|
|
1022
1005
|
|
|
1023
1006
|
# Download the full array in one go.
|
|
1024
|
-
|
|
1007
|
+
responses = await self.send(
|
|
1025
1008
|
GetDataArrays(data_arrays={dai.path_in_resource: dai}),
|
|
1026
1009
|
)
|
|
1027
1010
|
|
|
1011
|
+
assert len(responses) == 1
|
|
1012
|
+
response = responses[0]
|
|
1013
|
+
|
|
1028
1014
|
self.assert_response(response, GetDataArraysResponse)
|
|
1029
1015
|
assert (
|
|
1030
1016
|
len(response.data_arrays) == 1
|
|
@@ -1056,7 +1042,7 @@ class ETPClient(ETPConnection):
|
|
|
1056
1042
|
now = self.timestamp
|
|
1057
1043
|
|
|
1058
1044
|
# Allocate space on server for the array.
|
|
1059
|
-
|
|
1045
|
+
responses = await self.send(
|
|
1060
1046
|
PutUninitializedDataArrays(
|
|
1061
1047
|
data_arrays={
|
|
1062
1048
|
dai.path_in_resource: PutUninitializedDataArrayType(
|
|
@@ -1073,6 +1059,9 @@ class ETPClient(ETPConnection):
|
|
|
1073
1059
|
),
|
|
1074
1060
|
)
|
|
1075
1061
|
|
|
1062
|
+
assert len(responses) == 1
|
|
1063
|
+
response = responses[0]
|
|
1064
|
+
|
|
1076
1065
|
self.assert_response(response, PutUninitializedDataArraysResponse)
|
|
1077
1066
|
assert len(response.success) == 1 and dai.path_in_resource in response.success
|
|
1078
1067
|
|
|
@@ -1122,7 +1111,14 @@ class ETPClient(ETPConnection):
|
|
|
1122
1111
|
tasks.append(task)
|
|
1123
1112
|
|
|
1124
1113
|
# Upload all blocks.
|
|
1125
|
-
|
|
1114
|
+
task_responses = await asyncio.gather(*tasks)
|
|
1115
|
+
|
|
1116
|
+
# Flatten list of responses.
|
|
1117
|
+
responses = [
|
|
1118
|
+
response
|
|
1119
|
+
for task_response in task_responses
|
|
1120
|
+
for response in task_response
|
|
1121
|
+
]
|
|
1126
1122
|
|
|
1127
1123
|
# Check for successful responses.
|
|
1128
1124
|
for response in responses:
|
|
@@ -1139,7 +1135,7 @@ class ETPClient(ETPConnection):
|
|
|
1139
1135
|
etp_array_data = utils_arrays.get_etp_data_array_from_numpy(data)
|
|
1140
1136
|
|
|
1141
1137
|
# Pass entire array in one message.
|
|
1142
|
-
|
|
1138
|
+
responses = await self.send(
|
|
1143
1139
|
PutDataArrays(
|
|
1144
1140
|
data_arrays={
|
|
1145
1141
|
dai.path_in_resource: PutDataArraysType(
|
|
@@ -1150,6 +1146,9 @@ class ETPClient(ETPConnection):
|
|
|
1150
1146
|
)
|
|
1151
1147
|
)
|
|
1152
1148
|
|
|
1149
|
+
assert len(responses) == 1
|
|
1150
|
+
response = responses[0]
|
|
1151
|
+
|
|
1153
1152
|
self.assert_response(response, PutDataArraysResponse)
|
|
1154
1153
|
assert len(response.success) == 1 and dai.path_in_resource in response.success
|
|
1155
1154
|
|
|
@@ -1171,7 +1170,7 @@ class ETPClient(ETPConnection):
|
|
|
1171
1170
|
if data.nbytes > self.max_array_size:
|
|
1172
1171
|
return await self._put_array_chunked(uid, data)
|
|
1173
1172
|
|
|
1174
|
-
|
|
1173
|
+
responses = await self.send(
|
|
1175
1174
|
PutDataArrays(
|
|
1176
1175
|
data_arrays={
|
|
1177
1176
|
uid.path_in_resource: PutDataArraysType(
|
|
@@ -1182,6 +1181,9 @@ class ETPClient(ETPConnection):
|
|
|
1182
1181
|
)
|
|
1183
1182
|
)
|
|
1184
1183
|
|
|
1184
|
+
assert len(responses) == 1
|
|
1185
|
+
response = responses[0]
|
|
1186
|
+
|
|
1185
1187
|
assert isinstance(response, PutDataArraysResponse), (
|
|
1186
1188
|
"Expected PutDataArraysResponse"
|
|
1187
1189
|
)
|
|
@@ -1205,9 +1207,13 @@ class ETPClient(ETPConnection):
|
|
|
1205
1207
|
starts=starts.tolist(),
|
|
1206
1208
|
counts=counts.tolist(),
|
|
1207
1209
|
)
|
|
1208
|
-
|
|
1210
|
+
responses = await self.send(
|
|
1209
1211
|
GetDataSubarrays(dataSubarrays={uid.path_in_resource: payload})
|
|
1210
1212
|
)
|
|
1213
|
+
|
|
1214
|
+
assert len(responses) == 1
|
|
1215
|
+
response = responses[0]
|
|
1216
|
+
|
|
1211
1217
|
assert isinstance(response, GetDataSubarraysResponse), (
|
|
1212
1218
|
"Expected GetDataSubarraysResponse"
|
|
1213
1219
|
)
|
|
@@ -1246,9 +1252,13 @@ class ETPClient(ETPConnection):
|
|
|
1246
1252
|
f"{dataarray.data.item.__class__.__name__}"
|
|
1247
1253
|
)
|
|
1248
1254
|
|
|
1249
|
-
|
|
1255
|
+
responses = await self.send(
|
|
1250
1256
|
PutDataSubarrays(dataSubarrays={uid.path_in_resource: payload})
|
|
1251
1257
|
)
|
|
1258
|
+
|
|
1259
|
+
assert len(responses) == 1
|
|
1260
|
+
response = responses[0]
|
|
1261
|
+
|
|
1252
1262
|
assert isinstance(response, PutDataSubarraysResponse), (
|
|
1253
1263
|
"Expected PutDataSubarraysResponse"
|
|
1254
1264
|
)
|
|
@@ -1370,9 +1380,13 @@ class ETPClient(ETPConnection):
|
|
|
1370
1380
|
)
|
|
1371
1381
|
),
|
|
1372
1382
|
)
|
|
1373
|
-
|
|
1383
|
+
responses = await self.send(
|
|
1374
1384
|
PutUninitializedDataArrays(dataArrays={uid.path_in_resource: payload})
|
|
1375
1385
|
)
|
|
1386
|
+
|
|
1387
|
+
assert len(responses) == 1
|
|
1388
|
+
response = responses[0]
|
|
1389
|
+
|
|
1376
1390
|
assert isinstance(response, PutUninitializedDataArraysResponse), (
|
|
1377
1391
|
"Expected PutUninitializedDataArraysResponse"
|
|
1378
1392
|
)
|
|
@@ -1521,7 +1535,12 @@ class etp_connect:
|
|
|
1521
1535
|
) -> None:
|
|
1522
1536
|
self.uri = uri
|
|
1523
1537
|
self.data_partition_id = data_partition_id
|
|
1524
|
-
|
|
1538
|
+
|
|
1539
|
+
if isinstance(authorization, SecretStr):
|
|
1540
|
+
self.authorization = authorization
|
|
1541
|
+
else:
|
|
1542
|
+
self.authorization = SecretStr(authorization)
|
|
1543
|
+
|
|
1525
1544
|
self.etp_timeout = etp_timeout
|
|
1526
1545
|
self.max_message_size = max_message_size
|
|
1527
1546
|
self.subprotocols = ["etp12.energistics.org"]
|
|
@@ -1586,3 +1605,30 @@ class etp_connect:
|
|
|
1586
1605
|
max_message_size=self.max_message_size,
|
|
1587
1606
|
) as etp_client:
|
|
1588
1607
|
yield etp_client
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def timeout_intervals(total_timeout: float) -> Generator[float]:
|
|
1611
|
+
# Local function generating progressively longer timeout intervals.
|
|
1612
|
+
|
|
1613
|
+
# Use the timeout-interval generator from the Python websockets
|
|
1614
|
+
# library.
|
|
1615
|
+
backoff_generator = websockets.client.backoff(
|
|
1616
|
+
initial_delay=5.0, min_delay=5.0, max_delay=20.0
|
|
1617
|
+
)
|
|
1618
|
+
|
|
1619
|
+
# Check if we should never time out.
|
|
1620
|
+
if total_timeout is None:
|
|
1621
|
+
# This is an infinite generator, so it should never exit.
|
|
1622
|
+
yield from backoff_generator
|
|
1623
|
+
return
|
|
1624
|
+
|
|
1625
|
+
# Generate timeout intervals until we have reached the
|
|
1626
|
+
# `total_timeout`-threshold.
|
|
1627
|
+
csum = 0.0
|
|
1628
|
+
for d in backoff_generator:
|
|
1629
|
+
yield d
|
|
1630
|
+
|
|
1631
|
+
csum += d
|
|
1632
|
+
|
|
1633
|
+
if csum >= total_timeout:
|
|
1634
|
+
break
|