pyetp 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl

pyetp/client.py

@@ -96,7 +96,6 @@ class ETPClient(ETPConnection):
         self.timeout = timeout
         self.client_info.endpoint_capabilities['MaxWebSocketMessagePayloadSize'] = SETTINGS.MaxWebSocketMessagePayloadSize
         self.__recvtask = asyncio.create_task(self.__recv__())
-        self.max_concurrent_requests = SETTINGS.max_concurrent_requests
 
     #
     # client
@@ -274,6 +273,8 @@ class ETPClient(ETPConnection):
 
 
     def dataspace_uri(self, ds: str) -> DataspaceURI:
+        if ds.count("/") > 1:
+            raise Exception(f"Max one / in dataspace name")
         return DataspaceURI.from_name(ds)
     
     def list_objects(self, dataspace_uri: DataspaceURI, depth: int = 1) -> list:
@@ -291,10 +292,10 @@ class ETPClient(ETPConnection):
     #
 
     async def put_dataspaces(self, *dataspace_uris: DataspaceURI):
-
-
         _uris = list(map(DataspaceURI.from_any, dataspace_uris))
-
+        for i in _uris:
+            if i.raw_uri.count("/") > 4: # includes the 3 eml
+                raise Exception(f"Max one / in dataspace name")
         time = self.timestamp
         response = await self.send(
             PutDataspaces(dataspaces={
@@ -500,34 +501,36 @@ class ETPClient(ETPConnection):
             rotation=rotation,
             masked=True
         )
-    async def start_transaction(self, dataspace_uri: DataspaceURI, readOnly :bool= True) -> uuid.UUID:
+    async def start_transaction(self, dataspace_uri: DataspaceURI, readOnly :bool= True) -> Uuid:
         trans_id = await self.send(StartTransaction(readOnly=readOnly, dataspaceUris=[dataspace_uri.raw_uri]))
         if trans_id.successful is False:
             raise Exception(f"Failed starting transaction {dataspace_uri.raw_uri}")
-        return uuid.UUID(bytes=trans_id.transaction_uuid)
+        return Uuid(trans_id.transaction_uuid) #uuid.UUID(bytes=trans_id.transaction_uuid)
     
-    async def commit_transaction(self, transaction_id: uuid.UUID):
-        r = await self.send(CommitTransaction(transaction_uuid=transaction_id))
+    async def commit_transaction(self, transaction_id: Uuid):
+        r = await self.send(CommitTransaction(transactionUuid=transaction_id))
         if r.successful is False:
             raise Exception(r.failure_reason)
         return r
     
-    async def rollback_transaction(self, transaction_id: uuid.UUID):
+    async def rollback_transaction(self, transaction_id: Uuid):
         return await self.send(RollbackTransaction(transactionUuid=transaction_id))
     
     async def put_xtgeo_surface(self, surface: RegularSurface, epsg_code: int, dataspace_uri: DataspaceURI):
         """Returns (epc_uri, crs_uri, gri_uri)"""
         assert surface.values is not None, "cannot upload empty surface"
         
-        
+        t_id = await self.start_transaction(dataspace_uri, False)
         epc, crs, gri = utils_xml.parse_xtgeo_surface_to_resqml_grid(surface, epsg_code)
         epc_uri, crs_uri, gri_uri = await self.put_resqml_objects(epc, crs, gri, dataspace_uri=dataspace_uri)
-        response = await self.put_array(
+
+        await self.put_array(
             DataArrayIdentifier(
                 uri=epc_uri.raw_uri if isinstance(epc_uri, DataObjectURI) else epc_uri,
                 pathInResource=gri.grid2d_patch.geometry.points.zvalues.values.path_in_hdf_file  # type: ignore
             ),
-            surface.values.filled(np.nan).astype(np.float32)
+            surface.values.filled(np.nan).astype(np.float32),
+            t_id
         )
 
         return epc_uri, gri_uri, crs_uri
@@ -691,6 +694,7 @@ class ETPClient(ETPConnection):
         assert len(cprop0.patch_of_values) == 1, "property obj must have exactly one patch of values"
 
         st = time.time()
+        t_id = await self.start_transaction(dataspace_uri, False)
         propkind_uri = [""] if (propertykind0 is None) else (await self.put_resqml_objects(propertykind0, dataspace_uri=dataspace_uri))
         cprop_uri = await self.put_resqml_objects(cprop0, dataspace_uri=dataspace_uri)
         delay = time.time() - st
@@ -703,6 +707,7 @@ class ETPClient(ETPConnection):
                 pathInResource=cprop0.patch_of_values[0].values.values.path_in_hdf_file,
             ),
             array_ref,  # type: ignore
+            t_id
         )
         delay = time.time() - st
         logger.debug(f"pyetp: put_rddms_property: put array ({array_ref.shape}) took {delay} s")
@@ -713,6 +718,7 @@ class ETPClient(ETPConnection):
         dataspace_uri: DataspaceURI
     ):
         uns, crs, epc, timeseries, hexa = utils_xml.convert_epc_mesh_to_resqml_mesh(epc_filename, title_in, projected_epsg)
+        t_id = await self.start_transaction(dataspace_uri, False)
         epc_uri, crs_uri, uns_uri = await self.put_resqml_objects(epc, crs, uns, dataspace_uri=dataspace_uri)
         timeseries_uri = ""
         if timeseries is not None:
@@ -769,7 +775,7 @@ class ETPClient(ETPConnection):
             ),
             hexa.cell_face_is_right_handed  # type: ignore
         )
-
+        await self.commit_transaction(t_id)
         #
         # mesh properties: one Property, one array of values, and an optional PropertyKind per property
         #
@@ -889,21 +895,25 @@ class ETPClient(ETPConnection):
         arrays = list(response.data_arrays.values())
         return utils_arrays.to_numpy(arrays[0])
 
-    async def put_array(self, uid: DataArrayIdentifier, data: np.ndarray):
+    async def put_array(self, uid: DataArrayIdentifier, data: np.ndarray, transaction_id: Uuid | None = None):
 
 
         # Check if we can upload the full array in one go.
         if data.nbytes > self.max_array_size:
-            return await self._put_array_chuncked(uid, data)
-
+            return await self._put_array_chuncked(uid, data, transaction_id)
+        
         response = await self.send(
             PutDataArrays(
                 dataArrays={uid.path_in_resource: PutDataArraysType(uid=uid, array=utils_arrays.to_data_array(data))})
         )
+
         assert isinstance(response, PutDataArraysResponse), "Expected PutDataArraysResponse"
         assert len(response.success) == 1, "expected one success from put_array"
+        if isinstance(transaction_id, Uuid):
+            await self.commit_transaction(transaction_id)
         return response.success
 
+
     async def get_subarray(self, uid: DataArrayIdentifier, starts: T.Union[np.ndarray, T.List[int]], counts: T.Union[np.ndarray, T.List[int]]):
         starts = np.array(starts).astype(np.int64)
         counts = np.array(counts).astype(np.int64)
@@ -1002,31 +1012,33 @@ class ETPClient(ETPConnection):
             slices = tuple(map(lambda se: slice(se[0], se[1]), zip(starts-offset, ends-offset)))
             buffer[slices] = array
             return
-        coro = [populate(starts, counts) for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)]
-        logger.debug(f"Concurrent request: {self.max_concurrent_requests}")
-        for i in batched(coro, self.max_concurrent_requests):
-            await asyncio.gather(*i)    
-        # r = await asyncio.gather(*[
-        #     populate(starts, counts)
-        #     for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)
-        # ])
+        # coro = [populate(starts, counts) for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)]
+        # logger.debug(f"Concurrent request: {self.max_concurrent_requests}")
+        # for i in batched(coro, self.max_concurrent_requests):
+        #     await asyncio.gather(*i)    
+        r = await asyncio.gather(*[
+            populate(starts, counts)
+            for starts, counts in self._get_chunk_sizes(buffer_shape, dtype, offset)
+        ])
 
         return buffer
 
-    async def _put_array_chuncked(self, uid: DataArrayIdentifier, data: np.ndarray):
+    async def _put_array_chuncked(self, uid: DataArrayIdentifier, data: np.ndarray, transaction_id: Uuid | None = None):
         transport_array_type = utils_arrays.get_transport(data.dtype)
+
         await self._put_uninitialized_data_array(uid, data.shape, transport_array_type=transport_array_type)
-        params = []
-        coro = []
-        for starts, counts in self._get_chunk_sizes(data.shape, data.dtype):
-            params.append([starts, counts])
-            #await self.put_subarray(uid, data, starts, counts)
-            coro.append(self.put_subarray(uid, data, starts, counts))
-        logger.debug(f"Concurrent request: {self.max_concurrent_requests}")
-        for i in batched(coro, self.max_concurrent_requests):
-            await asyncio.gather(*i)    
-        #r = await asyncio.gather(*coro)
+        if isinstance(transaction_id, Uuid):
+            await self.commit_transaction(transaction_id)
 
+        ds_uri = DataspaceURI.from_any(uid.uri)
+        t_id = None
+        if isinstance(transaction_id, Uuid):
+            t_id = await self.start_transaction(ds_uri, False)
+        for starts, counts in self._get_chunk_sizes(data.shape, data.dtype):
+            await self.put_subarray(uid, data, starts, counts)
+        if isinstance(t_id, Uuid):
+            await self.commit_transaction(t_id)
+        
         return {uid.uri: ''}
 
     async def _put_uninitialized_data_array(self, uid: DataArrayIdentifier, shape: T.Tuple[int, ...], transport_array_type=AnyArrayType.ARRAY_OF_FLOAT, logical_array_type=AnyLogicalArrayType.ARRAY_OF_BOOLEAN):

pyetp/config.py

@@ -23,8 +23,7 @@ class Settings(BaseSettings):
     etp_url: WebSocketUrl = Field(default='wss://host.com')
     etp_timeout: float = Field(default=15., description="Timeout in seconds")
     data_partition: Optional[str] = None
-    MaxWebSocketMessagePayloadSize: int = Field(default=30000)
-    max_concurrent_requests: int = Field(default = 5)
+    MaxWebSocketMessagePayloadSize: int = Field(default=500000000)
 
 
 SETTINGS = Settings()

pyetp/types.py

@@ -25,6 +25,7 @@ from etptypes.energistics.etp.v12.datatypes.object.data_object import \
 from etptypes.energistics.etp.v12.datatypes.object.dataspace import Dataspace
 from etptypes.energistics.etp.v12.datatypes.server_capabilities import \
     ServerCapabilities
+from etptypes.energistics.etp.v12.datatypes.uuid import Uuid
 from etptypes.energistics.etp.v12.datatypes.supported_data_object import \
     SupportedDataObject
 from etptypes.energistics.etp.v12.datatypes.supported_protocol import \

pyetp/utils_xml.py

@@ -437,7 +437,7 @@ def create_resqml_mesh(rmdi, rmdts, geotimes, projected_epsg: int):  #(rddms_mes
         node_count=node_count,
         face_count=face_count,
         cell_shape=cellshape,
-        points=ro.Point3DHdf5Array(
+        points=ro.Point3dHdf5Array(
             coordinates=ro.Hdf5Dataset(
                 path_in_hdf_file=f"/RESQML/{str(hexa_uuid)}/points",
                 hdf_proxy=ro.DataObjectReference(

{pyetp-0.0.35.dist-info → pyetp-0.0.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pyetp
-Version: 0.0.35
+Version: 0.0.37
 Summary: Interface with OSDU RDDMS using ETP protocol
 Author: Adam Cheng
 Author-email: 52572642+adamchengtkc@users.noreply.github.com

pyetp-0.0.37.dist-info/RECORD

@@ -0,0 +1,14 @@
+pyetp/__init__.py,sha256=FSlPJ3AXoMe3vDQ0z8k417aKJWwt7sxbNPsLgdc21tI,123
+pyetp/client.py,sha256=H2fSpR4tNM4dfXvldlmI4VxemepOTarNGKkCuQos108,47173
+pyetp/config.py,sha256=eKSpS7EXOYchZPoh0bHsNTQIAcipG-nFYKpfdtEz3g8,868
+pyetp/resqml_objects/__init__.py,sha256=Sr_1bktzS8JMmE9oi6pcW6BuERm7gH0u50rBs58zI7Q,50492
+pyetp/resqml_objects/generated.py,sha256=P-hfL8btOE6uAmtaGXbSdWMl8Z9iJx-r6FRzSvmzHvU,783914
+pyetp/types.py,sha256=k3lvtgYmLO5aZGkQE2CChQ0RO-fB-Bkc9AC9uCrAC94,6458
+pyetp/uri.py,sha256=6NE_-QvtaL_z5NCUbxehc28ul-70DgXpMiEGaIJPqNc,2996
+pyetp/utils.py,sha256=4U_9ndZGzN3B0xMoeSglmxdg7jgfYP5v_mE6tF2Lqmc,518
+pyetp/utils_arrays.py,sha256=HHhw23La4SG7UJyd5Q_OJ5WlMq98QpwHBk553YyyYOk,4331
+pyetp/utils_xml.py,sha256=OLGKPxFsz2OzsJOEJDgQXIZ9wwvanlOr6jD3i3KHVZI,33861
+pyetp-0.0.37.dist-info/LICENSE.md,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pyetp-0.0.37.dist-info/METADATA,sha256=cjlX-Fm_oO4KK7LEduqtuVMx96bomR0frV5SryyVRKg,2344
+pyetp-0.0.37.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+pyetp-0.0.37.dist-info/RECORD,,

pyetp-0.0.35.dist-info/RECORD

@@ -1,14 +0,0 @@
-pyetp/__init__.py,sha256=FSlPJ3AXoMe3vDQ0z8k417aKJWwt7sxbNPsLgdc21tI,123
-pyetp/client.py,sha256=YlNrmhiuNkyMC8dJ-km1TSjFrMF867becxK8Y6Fzgxg,46534
-pyetp/config.py,sha256=WCs9n5Cd2igSVQzSNKPjnYP_H-61BYSHEeQ5Jh1oQlc,918
-pyetp/resqml_objects/__init__.py,sha256=Sr_1bktzS8JMmE9oi6pcW6BuERm7gH0u50rBs58zI7Q,50492
-pyetp/resqml_objects/generated.py,sha256=P-hfL8btOE6uAmtaGXbSdWMl8Z9iJx-r6FRzSvmzHvU,783914
-pyetp/types.py,sha256=HMNZ4_cJqhxqzXiYsNT-Ywvtfji9PARX4d6MixoQn1k,6397
-pyetp/uri.py,sha256=6NE_-QvtaL_z5NCUbxehc28ul-70DgXpMiEGaIJPqNc,2996
-pyetp/utils.py,sha256=4U_9ndZGzN3B0xMoeSglmxdg7jgfYP5v_mE6tF2Lqmc,518
-pyetp/utils_arrays.py,sha256=HHhw23La4SG7UJyd5Q_OJ5WlMq98QpwHBk553YyyYOk,4331
-pyetp/utils_xml.py,sha256=VLTelVsUZrUDZEGES4xMU8MA9zSW-C83IDiHD_B8wT0,33861
-pyetp-0.0.35.dist-info/LICENSE.md,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pyetp-0.0.35.dist-info/METADATA,sha256=bPWvQUgMF77eVBwkMLjJefMKU2TtXnOzHvw9_JKu1zY,2344
-pyetp-0.0.35.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-pyetp-0.0.35.dist-info/RECORD,,