pyetp 0.0.31__py3-none-any.whl → 0.0.33__py3-none-any.whl

pyetp/client.py

@@ -5,10 +5,8 @@ import sys
 import typing as T
 import uuid
 from collections import defaultdict
-from contextlib import asynccontextmanager
 from types import TracebackType
 import time
-
 import numpy as np
 import websockets
 from etpproto.connection import (CommunicationProtocol, ConnectionType,
@@ -18,12 +16,16 @@ from pydantic import SecretStr
 from scipy.interpolate import griddata
 from xtgeo import RegularSurface
 
+
 import pyetp.resqml_objects as ro
+#import energyml.resqml.v2_0_1.resqmlv2 as ro
+#import energyml.eml.v2_0.commonv2 as roc
 from pyetp import utils_arrays, utils_xml
 from pyetp.config import SETTINGS
 from pyetp.types import *
 from pyetp.uri import DataObjectURI, DataspaceURI
-from pyetp.utils import short_id
+from pyetp.utils import short_id, batched
+from asyncio import timeout
 
 try:
     # for py >3.11, we can raise grouped exceptions
@@ -32,27 +34,24 @@ except ImportError:
     def ExceptionGroup(msg, errors):
         return errors[0]
 
-try:
-    from asyncio import timeout
-except ImportError:
-    import async_timeout
+# try:
+#     from asyncio import timeout
+# except ImportError:
+#     import async_timeout
 
-    @asynccontextmanager
-    async def timeout(delay: T.Optional[float]) -> T.Any:
-        try:
-            async with async_timeout.timeout(delay):
-                yield None
-        except asyncio.CancelledError as e:
-            raise asyncio.TimeoutError(f'Timeout ({delay}s)') from e
+#     @asynccontextmanager
+#     async def timeout(delay: T.Optional[float]) -> T.Any:
+#         try:
+#             async with async_timeout.timeout(delay):
+#                 yield None
+#         except asyncio.CancelledError as e:
+#             raise asyncio.TimeoutError(f'Timeout ({delay}s)') from e
 
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 
-MAXPAYLOADSIZE = 10_000_000  # 10MB
-
-
 class ETPError(Exception):
     def __init__(self, message: str, code: int):
         self.message = message
@@ -88,16 +87,16 @@ class ETPClient(ETPConnection):
     _recv_events: T.Dict[int, asyncio.Event]
     _recv_buffer: T.Dict[int, T.List[ETPModel]]
 
-    def __init__(self, ws: websockets.WebSocketClientProtocol, default_dataspace_uri: T.Union[DataspaceURI, None], timeout=10.):
+    def __init__(self, ws: websockets.WebSocketClientProtocol, timeout=10.):
         super().__init__(connection_type=ConnectionType.CLIENT)
         self._recv_events = {}
         self._recv_buffer = defaultdict(lambda: list())  # type: ignore
-        self._default_duri = default_dataspace_uri
         self.ws = ws
 
         self.timeout = timeout
-        self.client_info.endpoint_capabilities['MaxWebSocketMessagePayloadSize'] = MAXPAYLOADSIZE
+        self.client_info.endpoint_capabilities['MaxWebSocketMessagePayloadSize'] = SETTINGS.MaxWebSocketMessagePayloadSize
         self.__recvtask = asyncio.create_task(self.__recv__())
+        self.max_concurrent_requests = 1
 
     #
     # client
@@ -121,7 +120,7 @@ class ETPClient(ETPConnection):
         # create future recv event
         self._recv_events[msg.header.message_id] = asyncio.Event()
 
-        async for msg_part in msg.encode_message_generator(self.max_size, self):
+        for msg_part in msg.encode_message_generator(self.max_size, self):
             await self.ws.send(msg_part)
 
         return msg.header.message_id
@@ -216,21 +215,22 @@ class ETPClient(ETPConnection):
 
     async def request_session(self):
         # Handshake protocol
-
+        etp_version = Version(major=1, minor=2, revision=0, patch=0)
         msg = await self.send(
             RequestSession(
                 applicationName=SETTINGS.application_name,
                 applicationVersion=SETTINGS.application_version,
                 clientInstanceId=uuid.uuid4(),  # type: ignore
                 requestedProtocols=[
-                    SupportedProtocol(protocol=p.value, protocolVersion=Version(major=1, minor=2), role='store')
-                    for p in [CommunicationProtocol.DISCOVERY, CommunicationProtocol.STORE, CommunicationProtocol.DATA_ARRAY, CommunicationProtocol.DATASPACE]
+                    SupportedProtocol(protocol=p.value, protocolVersion=etp_version, role='store')
+                    for p in CommunicationProtocol
                 ],
                 supportedDataObjects=[SupportedDataObject(qualifiedType="resqml20.*"), SupportedDataObject(qualifiedType="eml20.*")],
                 currentDateTime=self.timestamp,
                 earliestRetainedChangeTime=0,
                 endpointCapabilities=dict(
-                    MaxWebSocketMessagePayloadSize=DataValue(item=self.max_size)
+                    MaxWebSocketMessagePayloadSize=DataValue(item=self.max_size),
+                    MaxWebSocketFramePayloadSize=DataValue(item=10000)
                 )
             )
         )
@@ -246,10 +246,7 @@ class ETPClient(ETPConnection):
 
     async def authorize(self, authorization: str, supplemental_authorization: T.Mapping[str, str] = {}):
 
-        from etptypes.energistics.etp.v12.protocol.core.authorize import \
-            Authorize
-        from etptypes.energistics.etp.v12.protocol.core.authorize_response import \
-            AuthorizeResponse
+
 
         msg = await self.send(
             Authorize(
@@ -275,36 +272,28 @@ class ETPClient(ETPConnection):
     def timestamp(self):
         return int(datetime.datetime.now(datetime.timezone.utc).timestamp())
 
-    @property
-    def default_dataspace_uri(self):
-        return self._default_duri
-
-    @default_dataspace_uri.setter
-    def default_dataspace_uri(self, v: T.Union[DataspaceURI, str, None]):
-        self._default_duri = None if v is None else DataspaceURI.from_any(v)
-
-    def get_dataspace_or_default_uri(self, ds: T.Union[DataspaceURI, str, None]) -> DataspaceURI:
-        """Returns default dataspace or user spefied one"""
-
-        if ds is not None:
-            return DataspaceURI.from_any(ds)
-
-        if self._default_duri is None:
-            raise ValueError("Could not get dataspace from userinput or default")
-
-        return self._default_duri
 
+    def dataspace_uri(self, ds: str) -> DataspaceURI:
+        return DataspaceURI.from_name(ds)
+    
+    def list_objects(self, dataspace_uri: DataspaceURI, depth: int = 1) -> list:
+        return self.send(GetResources(
+                scope=ContextScopeKind.TARGETS_OR_SELF,
+                context=ContextInfo(
+                    uri=dataspace_uri.raw_uri,
+                    depth=depth,
+                    dataObjectTypes=[],
+                    navigableEdges=RelationshipKind.PRIMARY,)
+                )
+            )
     #
     # dataspace
     #
 
-    async def put_dataspaces(self, *uris: T.Union[DataspaceURI, str]):
-        from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces import \
-            PutDataspaces
-        from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces_response import \
-            PutDataspacesResponse
+    async def put_dataspaces(self, *dataspace_uris: DataspaceURI):
+
 
-        _uris = list(map(DataspaceURI.from_any, uris))
+        _uris = list(map(DataspaceURI.from_any, dataspace_uris))
 
         time = self.timestamp
         response = await self.send(
@@ -315,23 +304,20 @@ class ETPClient(ETPConnection):
         )
         assert isinstance(response, PutDataspacesResponse), "Expected PutDataspacesResponse"
 
-        assert len(response.success) == len(uris), f"expected {len(uris)} success's"
+        assert len(response.success) == len(dataspace_uris), f"expected {len(dataspace_uris)} success's"
 
         return response.success
 
-    async def put_dataspaces_no_raise(self, *uris: T.Union[DataspaceURI, str]):
+    async def put_dataspaces_no_raise(self, *dataspace_uris: DataspaceURI):
         try:
-            return await self.put_dataspaces(*uris)
+            return await self.put_dataspaces(*dataspace_uris)
         except ETPError:
             pass
 
-    async def delete_dataspaces(self, *uris: T.Union[DataspaceURI, str]):
-        from etptypes.energistics.etp.v12.protocol.dataspace.delete_dataspaces import \
-            DeleteDataspaces
-        from etptypes.energistics.etp.v12.protocol.dataspace.delete_dataspaces_response import \
-            DeleteDataspacesResponse
+    async def delete_dataspaces(self, *dataspace_uris: DataspaceURI):
 
-        _uris = list(map(str, uris))
+
+        _uris = list(map(str, dataspace_uris))
 
         response = await self.send(DeleteDataspaces(uris=dict(zip(_uris, _uris))))
         assert isinstance(response, DeleteDataspacesResponse), "Expected DeleteDataspacesResponse"
@@ -343,11 +329,6 @@ class ETPClient(ETPConnection):
 
     async def get_data_objects(self, *uris: T.Union[DataObjectURI, str]):
 
-        from etptypes.energistics.etp.v12.protocol.store.get_data_objects import \
-            GetDataObjects
-        from etptypes.energistics.etp.v12.protocol.store.get_data_objects_response import \
-            GetDataObjectsResponse
-
         _uris = list(map(str, uris))
 
         msg = await self.send(
@@ -360,10 +341,7 @@ class ETPClient(ETPConnection):
 
     async def put_data_objects(self, *objs: DataObject):
 
-        from etptypes.energistics.etp.v12.protocol.store.put_data_objects import \
-            PutDataObjects
-        from etptypes.energistics.etp.v12.protocol.store.put_data_objects_response import \
-            PutDataObjectsResponse
+
 
         response = await self.send(
             PutDataObjects(dataObjects={f"{p.resource.name}_{short_id()}": p for p in objs})
@@ -378,12 +356,10 @@ class ETPClient(ETPConnection):
         data_objects = await self.get_data_objects(*uris)
         return utils_xml.parse_resqml_objects(data_objects)
 
-    async def put_resqml_objects(self, *objs: ro.AbstractObject, dataspace: T.Union[DataspaceURI, str, None] = None):
-        from etptypes.energistics.etp.v12.datatypes.object.resource import \
-            Resource
+    async def put_resqml_objects(self, *objs: ro.AbstractObject, dataspace_uri: DataspaceURI):
+
         time = self.timestamp
-        duri = self.get_dataspace_or_default_uri(dataspace)
-        uris = [DataObjectURI.from_obj(duri, obj) for obj in objs]
+        uris = [DataObjectURI.from_obj(dataspace_uri, obj) for obj in objs]
         dobjs = [DataObject(
             format="xml",
             data=utils_xml.resqml_to_xml(obj),
@@ -403,10 +379,7 @@ class ETPClient(ETPConnection):
         return uris
 
     async def delete_data_objects(self, *uris: T.Union[DataObjectURI, str], pruneContainedObjects=False):
-        from etptypes.energistics.etp.v12.protocol.store.delete_data_objects import \
-            DeleteDataObjects
-        from etptypes.energistics.etp.v12.protocol.store.delete_data_objects_response import \
-            DeleteDataObjectsResponse
+
 
         _uris = list(map(str, uris))
 
@@ -420,7 +393,6 @@ class ETPClient(ETPConnection):
         assert isinstance(response, DeleteDataObjectsResponse), "Expected DeleteDataObjectsResponse"
 
         return response.deleted_uris
-
     #
     # xtgeo
     #
@@ -446,7 +418,7 @@ class ETPClient(ETPConnection):
         y_ind = (y-patch.geometry.points.supporting_geometry.origin.coordinate2)/patch.geometry.points.supporting_geometry.offset[1].spacing.value
         return round(x_ind), round(y_ind)
 
-    async def get_surface_value_x_y(self, epc_uri: T.Union[DataObjectURI, str], gri_uri: T.Union[DataObjectURI, str], x: T.Union[int, float], y: T.Union[int, float], method: T.Literal["bilinear", "nearest"]):
+    async def get_surface_value_x_y(self, epc_uri: T.Union[DataObjectURI, str], gri_uri: T.Union[DataObjectURI, str],crs_uri: T.Union[DataObjectURI, str], x: T.Union[int, float], y: T.Union[int, float], method: T.Literal["bilinear", "nearest"]):
         gri, = await self.get_resqml_objects(gri_uri) # parallelized using subarray
         xori = gri.grid2d_patch.geometry.points.supporting_geometry.origin.coordinate1
         yori = gri.grid2d_patch.geometry.points.supporting_geometry.origin.coordinate2
@@ -462,7 +434,7 @@ class ETPClient(ETPConnection):
                     uri=str(epc_uri), pathInResource=gri.grid2d_patch.geometry.points.zvalues.values.path_in_hdf_file
                 )
         if max_x_index_in_gri <= 10 or max_y_index_in_gri <= 10:
-            surf = await self.get_xtgeo_surface(epc_uri, gri_uri)
+            surf = await self.get_xtgeo_surface(epc_uri, gri_uri, crs_uri)
             return surf.get_value_from_xy((x, y), sampling=method)
 
         x_ind, y_ind = self.find_closest_index(x, y, gri.grid2d_patch)
@@ -499,21 +471,12 @@ class ETPClient(ETPConnection):
         )
         return regridded.get_value_from_xy((x, y))
 
-    async def get_xtgeo_surface(self, epc_uri: T.Union[DataObjectURI, str], gri_uri: T.Union[DataObjectURI, str], crs_uri: T.Union[DataObjectURI, str, None] = None):
-        if crs_uri is None:
-            logger.debug("NO crs")
-            gri, = await self.get_resqml_objects(gri_uri)
-            crs_uuid = gri.grid2d_patch.geometry.local_crs.uuid
-            dataspace_uri = self.get_dataspace_or_default_uri(epc_uri)
-            crs_eml = f"{dataspace_uri}/resqml20.LocalDepth3dCrs({crs_uuid})"
-            crs, = await self.get_resqml_objects(crs_eml)
-            logger.debug("got crs")
-        else:
-            gri, crs, = await self.get_resqml_objects(gri_uri, crs_uri)
+    async def get_xtgeo_surface(self, epc_uri: T.Union[DataObjectURI, str], gri_uri: T.Union[DataObjectURI, str], crs_uri: T.Union[DataObjectURI, str]):
+        gri, crs, = await self.get_resqml_objects(gri_uri, crs_uri)
         rotation = crs.areal_rotation.value
         # some checks
 
-        assert isinstance(gri, ro.Grid2dRepresentation), "obj must be Grid2dRepresentation"
+        assert isinstance(gri, ro.Grid2dRepresentation), "obj must be Grid2DRepresentation"
         sgeo = gri.grid2d_patch.geometry.points.supporting_geometry  # type: ignore
         if sys.version_info[1] != 10:
             assert isinstance(gri.grid2d_patch.geometry.points, ro.Point3dZValueArray), "Points must be Point3dZValueArray"
@@ -537,13 +500,24 @@ class ETPClient(ETPConnection):
             rotation=rotation,
             masked=True
         )
-
-    async def put_xtgeo_surface(self, surface: RegularSurface, epsg_code=23031, dataspace: T.Union[DataspaceURI, str, None] = None):
+    async def start_transaction(self, dataspace_uri: DataspaceURI, readOnly :bool= True) -> uuid.UUID:
+        trans_id = await self.send(StartTransaction(readOnly=readOnly, dataspaceUris=[dataspace_uri.raw_uri]))
+        return uuid.UUID(bytes=trans_id.transaction_uuid)
+    
+    async def commit_transaction(self, transaction_id: uuid.UUID):
+        trans_id = await self.send(CommitTransaction(transaction_uuid=transaction_id))
+        return trans_id
+    
+    async def rollback_transaction(self, transaction_id: uuid.UUID):
+        return await self.send(RollbackTransaction(transactionUuid=transaction_id))
+    
+    async def put_xtgeo_surface(self, surface: RegularSurface, epsg_code: int, dataspace_uri: DataspaceURI):
         """Returns (epc_uri, crs_uri, gri_uri)"""
         assert surface.values is not None, "cannot upload empty surface"
-
+        
+        
         epc, crs, gri = utils_xml.parse_xtgeo_surface_to_resqml_grid(surface, epsg_code)
-        epc_uri, crs_uri, gri_uri = await self.put_resqml_objects(epc, crs, gri, dataspace=dataspace)
+        epc_uri, crs_uri, gri_uri = await self.put_resqml_objects(epc, crs, gri, dataspace_uri=dataspace_uri)
         response = await self.put_array(
             DataArrayIdentifier(
                 uri=epc_uri.raw_uri if isinstance(epc_uri, DataObjectURI) else epc_uri,
@@ -552,7 +526,7 @@ class ETPClient(ETPConnection):
             surface.values.filled(np.nan).astype(np.float32)
         )
 
-        return epc_uri, crs_uri, gri_uri
+        return epc_uri, gri_uri, crs_uri
 
     #
     # resqpy meshes
@@ -562,7 +536,7 @@ class ETPClient(ETPConnection):
         uns, = await self.get_resqml_objects(uns_uri)
 
         # some checks
-        assert isinstance(uns, ro.UnstructuredGridRepresentation), "obj must be Grid2dRepresentation"
+        assert isinstance(uns, ro.UnstructuredGridRepresentation), "obj must be UnstructuredGridRepresentation"
         assert isinstance(uns.geometry, ro.UnstructuredGridGeometry), "geometry must be UnstructuredGridGeometry"
         if sys.version_info[1] != 10:
             assert isinstance(uns.geometry.points, ro.Point3dHdf5Array), "points must be Point3dHdf5Array"
@@ -706,14 +680,15 @@ class ETPClient(ETPConnection):
     async def put_rddms_property(self, epc_uri: T.Union[DataObjectURI , str], 
             cprop0: T.Union[ro.ContinuousProperty, ro.DiscreteProperty], 
             propertykind0: ro.PropertyKind, 
-            array_ref: np.ndarray, dataspace: T.Union[DataspaceURI , str , None] ):
+            array_ref: np.ndarray, 
+            dataspace_uri: DataspaceURI ):
 
         assert isinstance(cprop0, ro.ContinuousProperty) or isinstance(cprop0, ro.DiscreteProperty), "prop must be a Property"
         assert len(cprop0.patch_of_values) == 1, "property obj must have exactly one patch of values"
 
         st = time.time()
-        propkind_uri = [""] if (propertykind0 is None) else (await self.put_resqml_objects(propertykind0, dataspace=dataspace))
-        cprop_uri = await self.put_resqml_objects(cprop0, dataspace=dataspace)
+        propkind_uri = [""] if (propertykind0 is None) else (await self.put_resqml_objects(propertykind0, dataspace_uri=dataspace_uri))
+        cprop_uri = await self.put_resqml_objects(cprop0, dataspace_uri=dataspace_uri)
         delay = time.time() - st
         logger.debug(f"pyetp: put_rddms_property: put objects took {delay} s")
         
@@ -731,13 +706,13 @@ class ETPClient(ETPConnection):
 
     async def put_epc_mesh(
         self, epc_filename: str, title_in: str, property_titles: T.List[str], projected_epsg: int, 
-        dataspace: T.Union[DataspaceURI , str , None]
+        dataspace_uri: DataspaceURI
     ):
         uns, crs, epc, timeseries, hexa = utils_xml.convert_epc_mesh_to_resqml_mesh(epc_filename, title_in, projected_epsg)
-        epc_uri, crs_uri, uns_uri = await self.put_resqml_objects(epc, crs, uns, dataspace=dataspace)
+        epc_uri, crs_uri, uns_uri = await self.put_resqml_objects(epc, crs, uns, dataspace_uri=dataspace_uri)
         timeseries_uri = ""
         if timeseries is not None:
-            timeseries_uris = await self.put_resqml_objects(timeseries, dataspace=dataspace)
+            timeseries_uris = await self.put_resqml_objects(timeseries, dataspace_uri=dataspace_uri)
             timeseries_uri = list(timeseries_uris)[0] if (len(list(timeseries_uris)) > 0) else ""
         
         #
@@ -808,7 +783,7 @@ class ETPClient(ETPConnection):
 
             cprop_uris = []
             for cprop0, prop, time_index in zip(cprop0s, props, time_indices):
-                cprop_uri, propkind_uri = await self.put_rddms_property(epc_uri, cprop0, propertykind0, prop.array_ref(), dataspace)
+                cprop_uri, propkind_uri = await self.put_rddms_property(epc_uri, cprop0, propertykind0, prop.array_ref(), dataspace_uri)
                 cprop_uris.extend(cprop_uri)
             prop_rddms_uris[propname] = [propkind_uri, cprop_uris]
 
@@ -881,10 +856,7 @@ class ETPClient(ETPConnection):
     #
 
     async def get_array_metadata(self, *uids: DataArrayIdentifier):
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata import \
-            GetDataArrayMetadata
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata_response import \
-            GetDataArrayMetadataResponse
+
 
         response = await self.send(
             GetDataArrayMetadata(dataArrays={i.path_in_resource: i for i in uids})
@@ -898,10 +870,7 @@ class ETPClient(ETPConnection):
         return [response.array_metadata[i.path_in_resource] for i in uids]
 
     async def get_array(self, uid: DataArrayIdentifier):
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays import \
-            GetDataArrays
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays_response import \
-            GetDataArraysResponse
+
 
         # Check if we can upload the full array in one go.
         meta, = await self.get_array_metadata(uid)
@@ -917,12 +886,7 @@ class ETPClient(ETPConnection):
         return utils_arrays.to_numpy(arrays[0])
 
     async def put_array(self, uid: DataArrayIdentifier, data: np.ndarray):
-        from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_arrays_type import \
-            PutDataArraysType
-        from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays import \
-            PutDataArrays
-        from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays_response import \
-            PutDataArraysResponse
+
 
         # Check if we can upload the full array in one go.
         if data.nbytes > self.max_array_size:
@@ -940,12 +904,7 @@ class ETPClient(ETPConnection):
         starts = np.array(starts).astype(np.int64)
         counts = np.array(counts).astype(np.int64)
 
-        from etptypes.energistics.etp.v12.datatypes.data_array_types.get_data_subarrays_type import \
-            GetDataSubarraysType
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays import \
-            GetDataSubarrays
-        from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays_response import \
-            GetDataSubarraysResponse
+
 
         logger.debug(f"get_subarray {starts=:} {counts=:}")
 
@@ -962,22 +921,15 @@ class ETPClient(ETPConnection):
         arrays = list(response.data_subarrays.values())
         return utils_arrays.to_numpy(arrays[0])
 
-    async def put_subarray(self, uid: DataArrayIdentifier, data: np.ndarray, starts: T.Union[np.ndarray, T.List[int]], counts: T.Union[np.ndarray, T.List[int]], put_uninitialized=False):
-        from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_subarrays_type import \
-            PutDataSubarraysType
-        from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays import \
-            PutDataSubarrays
-        from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays_response import \
-            PutDataSubarraysResponse
+    async def put_subarray(self, uid: DataArrayIdentifier, data: np.ndarray, starts: T.Union[np.ndarray, T.List[int]], counts: T.Union[np.ndarray, T.List[int]]):
+
 
         # starts [start_X, starts_Y]
         # counts [count_X, count_Y]
         starts = np.array(starts).astype(np.int64) # len = 2 [x_start_index, y_start_index]
         counts = np.array(counts).astype(np.int64) # len = 2
         ends = starts + counts # len = 2
-        if put_uninitialized:
-            transport_array_type = utils_arrays.get_transport(data.dtype)
-            await self._put_uninitialized_data_array(uid, data.shape, transport_array_type=transport_array_type)
+
 
         slices = tuple(map(lambda se: slice(se[0], se[1]), zip(starts, ends)))
         dataarray = utils_arrays.to_data_array(data[slices])
@@ -994,7 +946,6 @@ class ETPClient(ETPConnection):
             PutDataSubarrays(dataSubarrays={uid.path_in_resource: payload})
         )
         assert isinstance(response, PutDataSubarraysResponse), "Expected PutDataSubarraysResponse"
-
         assert len(response.success) == 1, "expected one success"
         return response.success
 
@@ -1049,11 +1000,13 @@ class ETPClient(ETPConnection):
             slices = tuple(map(lambda se: slice(se[0], se[1]), zip(starts-offset, ends-offset)))
             buffer[slices] = array
             return
-
-        r = await asyncio.gather(*[
-            populate(starts, counts)
-            for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)
-        ])
+        coro = [populate(starts, counts) for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)]
+        for i in batched(coro, self.max_concurrent_requests):
+            await asyncio.gather(*i)    
+        # r = await asyncio.gather(*[
+        #     populate(starts, counts)
+        #     for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)
+        # ])
 
         return buffer
 
@@ -1065,18 +1018,16 @@ class ETPClient(ETPConnection):
         coro = []
         for starts, counts in self._get_chunk_sizes(data.shape, data.dtype):
             params.append([starts, counts])
+            #await self.put_subarray(uid, data, starts, counts)
             coro.append(self.put_subarray(uid, data, starts, counts))
-        r = await asyncio.gather(*coro)
+        for i in batched(coro, self.max_concurrent_requests):
+            await asyncio.gather(*i)    
+        #r = await asyncio.gather(*coro)
 
         return {uid.uri: ''}
 
     async def _put_uninitialized_data_array(self, uid: DataArrayIdentifier, shape: T.Tuple[int, ...], transport_array_type=AnyArrayType.ARRAY_OF_FLOAT, logical_array_type=AnyLogicalArrayType.ARRAY_OF_BOOLEAN):
-        from etptypes.energistics.etp.v12.datatypes.data_array_types.put_uninitialized_data_array_type import \
-            PutUninitializedDataArrayType
-        from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays import \
-            PutUninitializedDataArrays
-        from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays_response import \
-            PutUninitializedDataArraysResponse
+
 
         payload = PutUninitializedDataArrayType(
             uid=uid,
@@ -1104,7 +1055,6 @@ class connect:
         self.authorization = authorization
         self.data_partition = SETTINGS.data_partition
         self.timeout = SETTINGS.etp_timeout
-        self.default_dataspace_uri = DataspaceURI.from_name(SETTINGS.dataspace)
 
     # ... = await connect(...)
 
@@ -1127,12 +1077,12 @@ class connect:
             self.server_url,
             subprotocols=[ETPClient.SUB_PROTOCOL],  # type: ignore
             extra_headers=headers,
-            max_size=MAXPAYLOADSIZE,
+            max_size=SETTINGS.MaxWebSocketMessagePayloadSize,
             ping_timeout=self.timeout,
             open_timeout=None,
         )
 
-        self.client = ETPClient(ws, default_dataspace_uri=self.default_dataspace_uri, timeout=self.timeout)
+        self.client = ETPClient(ws, timeout=self.timeout)
 
         try:
             await self.client.request_session()

pyetp/config.py

@@ -1,8 +1,6 @@
 from typing import Optional
 
-from pydantic import AnyUrl, BaseSettings, Field, RedisDsn, SecretStr
-
-from pyetp.uri import DataspaceURI
+from pydantic import AnyUrl, BaseSettings, Field
 
 
 class WebSocketUrl(AnyUrl):
@@ -18,17 +16,14 @@ class Settings(BaseSettings):
             'redis_password': {'env': ['redis_password', 'redispass']}
         }
 
-    application_name: str = Field(default='etpClient')
-    application_version: str = Field(default='0.0.1')
+    application_name: str = Field(default='pyetp')
+    application_version: str = Field(default='0.0.33')
 
-    dataspace: str = Field(default='demo/pss-data-gateway')
+    #dataspace: str = Field(default='demo/pss-data-gateway')
     etp_url: WebSocketUrl = Field(default='wss://host.com')
-    etp_timeout: float = Field(default=60., description="Timeout in seconds")
+    etp_timeout: float = Field(default=15., description="Timeout in seconds")
     data_partition: Optional[str] = None
-
-    @property
-    def duri(self):
-        return DataspaceURI.from_name(self.dataspace)
+    MaxWebSocketMessagePayloadSize: int = Field(default=100000)
 
 
 SETTINGS = Settings()

pyetp/types.py

@@ -41,7 +41,72 @@ from etptypes.energistics.etp.v12.protocol.core.protocol_exception import \
     ProtocolException
 from etptypes.energistics.etp.v12.protocol.core.request_session import \
     RequestSession
-
+from etptypes.energistics.etp.v12.protocol.transaction.start_transaction import StartTransaction
+from etptypes.energistics.etp.v12.protocol.transaction.commit_transaction import CommitTransaction
+from etptypes.energistics.etp.v12.protocol.transaction.rollback_transaction import RollbackTransaction
+from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_subarrays_type import \
+    PutDataSubarraysType
+from etptypes.energistics.etp.v12.datatypes.object.relationship_kind import RelationshipKind
+from etptypes.energistics.etp.v12.datatypes.object.context_info import ContextInfo
+from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays import \
+    PutDataSubarrays
+from etptypes.energistics.etp.v12.datatypes.object.context_scope_kind import ContextScopeKind
+from etptypes.energistics.etp.v12.protocol.data_array.put_data_subarrays_response import \
+    PutDataSubarraysResponse
+from etptypes.energistics.etp.v12.datatypes.data_array_types.get_data_subarrays_type import \
+    GetDataSubarraysType
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays import \
+    GetDataSubarrays
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_subarrays_response import \
+    GetDataSubarraysResponse
+from etptypes.energistics.etp.v12.datatypes.data_array_types.put_data_arrays_type import \
+    PutDataArraysType
+from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays import \
+    PutDataArrays
+from etptypes.energistics.etp.v12.protocol.data_array.put_data_arrays_response import \
+    PutDataArraysResponse
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays import \
+    GetDataArrays
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_arrays_response import \
+    GetDataArraysResponse
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata import \
+    GetDataArrayMetadata
+from etptypes.energistics.etp.v12.protocol.data_array.get_data_array_metadata_response import \
+    GetDataArrayMetadataResponse
+from etptypes.energistics.etp.v12.datatypes.data_array_types.put_uninitialized_data_array_type import \
+    PutUninitializedDataArrayType
+from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays import \
+    PutUninitializedDataArrays
+from etptypes.energistics.etp.v12.protocol.data_array.put_uninitialized_data_arrays_response import \
+    PutUninitializedDataArraysResponse
+from etptypes.energistics.etp.v12.protocol.store.delete_data_objects import \
+    DeleteDataObjects
+from etptypes.energistics.etp.v12.protocol.store.delete_data_objects_response import \
+    DeleteDataObjectsResponse
+from etptypes.energistics.etp.v12.datatypes.object.resource import \
+    Resource
+from etptypes.energistics.etp.v12.protocol.discovery.get_resources import \
+    GetResources
+from etptypes.energistics.etp.v12.protocol.store.put_data_objects import \
+    PutDataObjects
+from etptypes.energistics.etp.v12.protocol.store.put_data_objects_response import \
+    PutDataObjectsResponse
+from etptypes.energistics.etp.v12.protocol.store.get_data_objects import \
+    GetDataObjects
+from etptypes.energistics.etp.v12.protocol.store.get_data_objects_response import \
+    GetDataObjectsResponse
+from etptypes.energistics.etp.v12.protocol.dataspace.delete_dataspaces import \
+    DeleteDataspaces
+from etptypes.energistics.etp.v12.protocol.dataspace.delete_dataspaces_response import \
+    DeleteDataspacesResponse
+from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces import \
+    PutDataspaces
+from etptypes.energistics.etp.v12.protocol.dataspace.put_dataspaces_response import \
+    PutDataspacesResponse
+from etptypes.energistics.etp.v12.protocol.core.authorize import \
+    Authorize
+from etptypes.energistics.etp.v12.protocol.core.authorize_response import \
+    AuthorizeResponse
 #
 # NOTE we want to `from etptypes.energistics.etp.v12 import datatypes, protocol` and use this
 # however this not supported with pylance as of yet, hence so many imports are reexported here

pyetp/uri.py

@@ -4,12 +4,11 @@ from uuid import UUID
 from etpproto.uri import DataObjectURI as _DataObjectURI
 from etpproto.uri import DataspaceUri as _DataspaceURI
 from pydantic import BaseConfig
-
-import pyetp.resqml_objects as ro
+from pyetp.resqml_objects import AbstractObject
+#from energyml.eml.v2_0.commonv2 import AbstractObject
 
 
 class _Mixin:
-    # print full url as default
     raw_uri: str
 
     def __str__(self):
@@ -66,7 +65,7 @@ class DataObjectURI(_DataObjectURI, _Mixin):
         return cls(f"{duri}/{domain_and_version}.{obj_type}({uuid})")
 
     @classmethod
-    def from_obj(cls, dataspace: Union[DataspaceURI , str], obj: ro.AbstractObject):
+    def from_obj(cls, dataspace: Union[DataspaceURI , str], obj: AbstractObject):
 
         objname = obj.__class__.__name__
         if getattr(obj, 'Meta', None):

pyetp/utils.py

@@ -1,6 +1,15 @@
 import random
 import string
-
-
+from itertools import islice
+from typing import Iterable
 def short_id(length=8):
     return ''.join([random.choice(string.ascii_letters + string.digits + '-_') for _ in range(length)])
+
+def batched(iterable:Iterable, n: int):
+    "Batch data into tuples of length n. The last batch may be shorter."
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError('n must be at least one')
+    it = iter(iterable)
+    while batch := tuple(islice(it, n)):
+        yield batch

pyetp/utils_xml.py

@@ -2,15 +2,21 @@ import datetime
 import logging
 import typing as T
 from uuid import uuid4
-
+import numpy as np
 import lxml.etree as ET
 from xsdata.formats.dataclass.context import XmlContext
 from xsdata.formats.dataclass.parsers import XmlParser
 from xsdata.formats.dataclass.serializers import XmlSerializer
 from xsdata.formats.dataclass.serializers.config import SerializerConfig
 from xsdata.models.datatype import XmlDateTime
+import resqpy.model as rq
+import resqpy.time_series as rts
+import resqpy.unstructured as rug
 
+import resqpy.property as rqp
 import pyetp.resqml_objects as ro
+#import energyml.resqml.v2_0_1.resqmlv2 as ro
+#import energyml.eml.v2_0.commonv2 as roc
 from pyetp.config import SETTINGS
 from pyetp.types import DataObject
 
@@ -21,8 +27,7 @@ if T.TYPE_CHECKING:
     from xtgeo import RegularSurface
 
 
-schema_version = "2.0"
-
+schema_version = "2.0.1"
 
 def get_data_object_type(obj: ro.AbstractObject):
     return obj.__class__.__name__
@@ -83,8 +88,8 @@ def create_common_crs(title: str, projected_epsg, rotation: float = 0.0):
         projected_uom=ro.LengthUom.M,
         vertical_uom=ro.LengthUom.M,
         zincreasing_downward=True,
-        vertical_crs=ro.VerticalUnknownCrs(
-            unknown="unknown",
+        vertical_crs=ro.VerticalCrsEpsgCode(
+            epsg_code=projected_epsg
         ),
         projected_crs=ro.ProjectedCrsEpsgCode(
             epsg_code=projected_epsg,
@@ -139,7 +144,7 @@ def instantiate_resqml_grid(name: str, rotation: float, x0: float, y0: float, dx
             fastest_axis_count=ny,
             slowest_axis_count=nx,
             geometry=ro.PointGeometry(
-                local_crs=ro.DataObjectReference(
+                local_crs= ro.DataObjectReference(
                     # NOTE: See Energistics Identifier Specification 4.0
                     # (it is downloaded alongside the RESQML v2.0.1
                     # standard) section 4.1 for an explanation on the
@@ -147,7 +152,8 @@ def instantiate_resqml_grid(name: str, rotation: float, x0: float, y0: float, dx
                     content_type=f"application/x-resqml+xml;version={schema_version};type={get_data_object_type(crs)}",
                     title=crs.citation.title,
                     uuid=crs.uuid,
-                ),
+                )
+                ,
                 points=ro.Point3dZValueArray(
                     supporting_geometry=ro.Point3dLatticeArray(
                         origin=ro.Point3d(
@@ -237,7 +243,7 @@ def uom_for_prop_title(pt: str):
         return ro.ResqmlUom.VALUE
     return ro.ResqmlUom.EUC
 
-def create_resqml_property(prop_title, continuous, indexable_element, uns, epc, min_val=0.0, max_val=1.0, 
+def create_resqml_property(prop_title:str, continuous: bool, indexable_element: ro.IndexableElements, uns: ro.UnstructuredGridRepresentation, epc: ro.EpcExternalPartReference, min_val=0.0, max_val=1.0, 
                            timeseries=None, time_index=-1, pre_existing_propertykind = None):
     timeindex_ref = None
     use_timeseries = timeseries is not None
@@ -253,7 +259,6 @@ def create_resqml_property(prop_title, continuous, indexable_element, uns, epc,
         )
 
     r_uom = ro.ResqmlUom( value= uom_for_prop_title(prop_title) )
-    # r_uom = ro.ResqmlUom( uom )
 
     if (pre_existing_propertykind is None):
         pk_uuid = uuid4()
@@ -357,8 +362,8 @@ def create_resqml_property(prop_title, continuous, indexable_element, uns, epc,
         )
     return cprop0, propertykind0
 
-def create_resqml_mesh(rmdi, rmdts, geotimes, projected_epsg):  #(rddms_mesh_data_initial, rddms_upload_data_timestep)
-    import numpy as np
+def create_resqml_mesh(rmdi, rmdts, geotimes, projected_epsg: int):  #(rddms_mesh_data_initial, rddms_upload_data_timestep)
+    
     ro_timestamps = []
     for i in geotimes:
         ro_timestamps.append(
@@ -432,7 +437,7 @@ def create_resqml_mesh(rmdi, rmdts, geotimes, projected_epsg):  #(rddms_mesh_dat
         node_count=node_count,
         face_count=face_count,
         cell_shape=cellshape,
-        points=ro.Point3dHdf5Array(
+        points=ro.Point3DHdf5Array(
             coordinates=ro.Hdf5Dataset(
                 path_in_hdf_file=f"/RESQML/{str(hexa_uuid)}/points",
                 hdf_proxy=ro.DataObjectReference(
@@ -514,10 +519,8 @@ def create_resqml_mesh(rmdi, rmdts, geotimes, projected_epsg):  #(rddms_mesh_dat
     return uns, crs, epc, timeseries
 
 
-def convert_epc_mesh_to_resqml_mesh(epc_filename, title_in, projected_epsg):
-    import resqpy.model as rq
-    import resqpy.time_series as rts
-    import resqpy.unstructured as rug
+def convert_epc_mesh_to_resqml_mesh(epc_filename: str, title_in: str, projected_epsg: int):
+
 
     title = title_in or "hexamesh"
 
@@ -661,9 +664,8 @@ def convert_epc_mesh_to_resqml_mesh(epc_filename, title_in, projected_epsg):
     return uns, crs, epc, timeseries, hexa
 
 
-def convert_epc_mesh_property_to_resqml_mesh(epc_filename, hexa, prop_title, uns, epc, timeseries=None, time_indices: list[int] = []):
-    import resqpy.model as rq
-    import resqpy.property as rqp
+def convert_epc_mesh_property_to_resqml_mesh(epc_filename, hexa, prop_title, uns:ro.UnstructuredGridRepresentation, epc:ro.EpcExternalPartReference, timeseries=None, time_indices: list[int] = []):
+
 
     model = rq.Model(epc_filename)
     assert model is not None

{pyetp-0.0.31.dist-info → pyetp-0.0.33.dist-info}/METADATA

@@ -1,20 +1,18 @@
 Metadata-Version: 2.3
 Name: pyetp
-Version: 0.0.31
+Version: 0.0.33
 Summary: Interface with OSDU RDDMS using ETP protocol
 Author: Adam Cheng
 Author-email: 52572642+adamchengtkc@users.noreply.github.com
-Requires-Python: >=3.9,<3.13
+Requires-Python: >=3.11,<3.13
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: async-lru (>=2.0.4,<3.0.0)
 Requires-Dist: async-timeout (>=5.0,<6.0) ; python_version < "3.11"
-Requires-Dist: etpproto (>=1.0.5,<2.0.0)
+Requires-Dist: etpproto (>=1.0.7,<2.0.0)
 Requires-Dist: lxml (>=4.9.4,<6.0)
 Requires-Dist: numpy (>=1.26.3,<2.0.0)
 Requires-Dist: pyarrow (>=15.0.0,<16.0.0)

pyetp-0.0.33.dist-info/RECORD

@@ -0,0 +1,14 @@
+pyetp/__init__.py,sha256=FSlPJ3AXoMe3vDQ0z8k417aKJWwt7sxbNPsLgdc21tI,123
+pyetp/client.py,sha256=71DV3WPXpOX4fL119aR_5z09L3NDsS5zMP1mSMHGr9s,46154
+pyetp/config.py,sha256=BmCWo09GXO9CoAwtmmmAflNeRHF-5d14JFwsBBrKnQA,865
+pyetp/resqml_objects/__init__.py,sha256=Sr_1bktzS8JMmE9oi6pcW6BuERm7gH0u50rBs58zI7Q,50492
+pyetp/resqml_objects/generated.py,sha256=P-hfL8btOE6uAmtaGXbSdWMl8Z9iJx-r6FRzSvmzHvU,783914
+pyetp/types.py,sha256=HMNZ4_cJqhxqzXiYsNT-Ywvtfji9PARX4d6MixoQn1k,6397
+pyetp/uri.py,sha256=6NE_-QvtaL_z5NCUbxehc28ul-70DgXpMiEGaIJPqNc,2996
+pyetp/utils.py,sha256=4U_9ndZGzN3B0xMoeSglmxdg7jgfYP5v_mE6tF2Lqmc,518
+pyetp/utils_arrays.py,sha256=HHhw23La4SG7UJyd5Q_OJ5WlMq98QpwHBk553YyyYOk,4331
+pyetp/utils_xml.py,sha256=VLTelVsUZrUDZEGES4xMU8MA9zSW-C83IDiHD_B8wT0,33861
+pyetp-0.0.33.dist-info/LICENSE.md,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pyetp-0.0.33.dist-info/METADATA,sha256=38-PLw8E-sAbkpTIdZqzyviKXEXlIPvO9m7adac2NHw,2293
+pyetp-0.0.33.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+pyetp-0.0.33.dist-info/RECORD,,

{pyetp-0.0.31.dist-info → pyetp-0.0.33.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any

pyetp-0.0.31.dist-info/RECORD

@@ -1,14 +0,0 @@
-pyetp/__init__.py,sha256=FSlPJ3AXoMe3vDQ0z8k417aKJWwt7sxbNPsLgdc21tI,123
-pyetp/client.py,sha256=MJmS-HfQKmbcfMriakDcwwoBdVgMQAF6LcpgdFx20TU,49638
-pyetp/config.py,sha256=Z0Q3dH0olM4CKLckDA_EV23XK_al2Ey8D9JrKb3s5R0,949
-pyetp/resqml_objects/__init__.py,sha256=Sr_1bktzS8JMmE9oi6pcW6BuERm7gH0u50rBs58zI7Q,50492
-pyetp/resqml_objects/generated.py,sha256=P-hfL8btOE6uAmtaGXbSdWMl8Z9iJx-r6FRzSvmzHvU,783914
-pyetp/types.py,sha256=E-CvzI74kiQ52J42DQ-5wGUC84l3-glYs3NQ36HIs1I,2559
-pyetp/uri.py,sha256=57wTOH7EAGXluDiGxIe34DAiftPR91jEnbRB5thaG8k,2963
-pyetp/utils.py,sha256=WxfG13bItprl457H3EEsckF1uDwF-hzPhVG7QRX0LB8,158
-pyetp/utils_arrays.py,sha256=HHhw23La4SG7UJyd5Q_OJ5WlMq98QpwHBk553YyyYOk,4331
-pyetp/utils_xml.py,sha256=mBnWERnlMEjoPb6D9EYalkUShDvEOvo7zxw-Hz0dJ1A,33647
-pyetp-0.0.31.dist-info/LICENSE.md,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pyetp-0.0.31.dist-info/METADATA,sha256=KOsETuuwuuBEw64Zz5WI4GC87oIof-mtE27hLOzinsY,2393
-pyetp-0.0.31.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-pyetp-0.0.31.dist-info/RECORD,,