osducli 0.0.42__py3-none-any.whl → 0.0.44__py3-none-any.whl

osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.42"
+__VERSION__ = "0.0.44"

osducli/cliclient.py

@@ -157,12 +157,14 @@ class CliOsduClient(BaseClient):
         return url
 
     def check_status_code(self, response: requests.Response, ok_status_codes: list = None):
+        """Check the status code of the response and raise an exception if not in the list of ok status codes"""
         if ok_status_codes is None:
             ok_status_codes = [200]
         if response.status_code not in ok_status_codes:
             raise HTTPError(response=response)
 
     def get_search_client(self) -> SearchClient:
+        """Get a client for the search service"""
         search_url = self.url_from_config(CONFIG_SEARCH_URL, "")
         if search_url.endswith("/"):
             search_url = search_url[:-1]
@@ -174,6 +176,7 @@ class CliOsduClient(BaseClient):
         )
 
     def get_storage_record_client(self) -> RecordClient:
+        """Get a client for the storage record service"""
         storage_url = self.url_from_config(CONFIG_STORAGE_URL, "")
         if storage_url.endswith("/"):
             storage_url = storage_url[:-1]
@@ -185,6 +188,7 @@ class CliOsduClient(BaseClient):
         )
 
     def get_entitlements_client(self) -> EntitlementsClient:
+        """Get a client for the entitlements service"""
         entitlements_url = self.url_from_config(CONFIG_ENTITLEMENTS_URL, "")
         if entitlements_url.endswith("/"):
             entitlements_url = entitlements_url[:-1]
@@ -195,7 +199,15 @@ class CliOsduClient(BaseClient):
             token_refresher=self.token_refresher
         )
 
-    def get_wellbore_ddms_client(self, url_extra_path: str) -> WellboreDdmsClient:
+    def get_wellbore_ddms_client(
+        self,
+        url_extra_path: str
+    ) -> WellboreDdmsClient:
+        """Get a client for the wellbore ddms service
+
+        Args:
+            url_extra_path (str): extra path to add to the base path
+        """
         wellbore_ddms_url = self.url_from_config(CONFIG_WELLBORE_DDMS_URL, url_extra_path)
         return WellboreDdmsClient(
             wellbore_ddms_url=wellbore_ddms_url,
@@ -205,9 +217,12 @@ class CliOsduClient(BaseClient):
         )
 
     def cli_get(
-        self, config_url_key: str, url_extra_path: str, ok_status_codes: list = None
+        self,
+        config_url_key: str,
+        url_extra_path: str,
+        ok_status_codes: list = None
     ) -> requests.Response:
-        """[summary]
+        """Basic GET call to the given url, returning the response object.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -220,9 +235,12 @@ class CliOsduClient(BaseClient):
         return response
 
     def cli_get_returning_json(
-        self, config_url_key: str, url_extra_path: str, ok_status_codes: list = None
+        self,
+        config_url_key: str,
+        url_extra_path: str,
+        ok_status_codes: list = None
     ) -> dict:
-        """[summary]
+        """Basic GET call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -241,7 +259,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> dict:
-        """[summary]
+        """Basic POST call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -253,6 +271,8 @@ class CliOsduClient(BaseClient):
             dict: returned json
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.POST, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response.json()
@@ -263,7 +283,7 @@ class CliOsduClient(BaseClient):
         url_extra_path: str,
         ok_status_codes: list = None,
     ) -> requests.Response:
-        """[summary]
+        """Basic DELETE call to the given url.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -274,7 +294,11 @@ class CliOsduClient(BaseClient):
             requests.Response: Response object from the HTTP call
         """
         url = self.url_from_config(config_url_key, url_extra_path)
-        response = self.make_request(method=HttpMethod.DELETE, url=url)
+
+        # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+        response = self.make_request(method=HttpMethod.DELETE, url=url,
+                                     bearer_token=self.token_refresher.refresh_token())
+
         self.check_status_code(response, ok_status_codes)
         return response
 
@@ -285,7 +309,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> requests.Response:
-        """[summary]
+        """Basic PUT call to the given url.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -294,6 +318,8 @@ class CliOsduClient(BaseClient):
             ok_status_codes (list, optional): Optional status codes to check for successful call.
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.PUT, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response
@@ -305,7 +331,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> dict:
-        """PUT the data to the given url.
+        """Basic PUT call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -317,6 +343,8 @@ class CliOsduClient(BaseClient):
             dict: returned json
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.PUT, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response.json()

osducli/commands/storage/add.py

@@ -15,12 +15,12 @@
 import json
 
 import click
+from osdu_api.model.storage.record import Record
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 from osducli.log import get_logger
-from osducli.util.file import get_files_from_path
+from osducli.util.file import get_files_from_path_with_suffix
 
 logger = get_logger(__name__)
 
@@ -51,41 +51,57 @@ def _click_command(state: State, path: str, batch: int):
     return add_records(state, path, batch)
 
 
-def add_records(state: State, path: str, batch: int) -> dict:
+def add_records(state: State, path: str, batch: int = None):
     """Add or update a record
 
     Args:
         state (State): Global state
         path (str): Path to a record or records to add.
         batch (int): Batch size per API call. If None then ingest as is
-    Returns:
-        dict: Response from service
     """
-    if batch is not None:
-        raise NotImplementedError("--batch is not supported yet for storage add")
-
-    connection = CliOsduClient(state.config)
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
 
-    files = get_files_from_path(path)
+    files = get_files_from_path_with_suffix(path, ".json")
     logger.debug("Files list: %s", files)
 
-    # TODO: Check if loaded file is already an array, or a single file
-    # TODO: Batch uploads
-    responses = []
+    if batch is not None:
+        logger.info("Batching records with size %s", batch)
+        file_batches = chunk_list(files, batch)
+        for file_batch in file_batches:
+            response = add_record_batch(record_client, file_batch)
+            handle_response(client, response)
+    else:
+        response = add_record_batch(record_client, files)
+        handle_response(client, response)
+
+
+def add_record_batch(record_client, files):
+    record_list = []
     for filepath in files:
-        if filepath.endswith(".json"):
-            with open(filepath, encoding="utf-8") as file:
-                storage_object = json.load(file)
-
-                logger.info("Processing file %s.", filepath)
-                if isinstance(storage_object, list):
-                    payload = storage_object
-                else:
-                    payload = [storage_object]
-
-                response_json = None
-                response_json = connection.cli_put_returning_json(
-                    CONFIG_STORAGE_URL, "records", payload, [200, 201]
-                )
-                responses.append(response_json)
-    return responses
+        with open(filepath, encoding="utf-8") as file:
+            storage_object = json.load(file)
+
+            logger.info("Processing file %s.", filepath)
+            record_list.append(Record.from_dict(storage_object))
+
+    return record_client.create_update_records(record_list)
+
+
+def chunk_list(lst, size):
+    return [lst[i:i + size] for i in range(0, len(lst), size)]
+
+
+def handle_response(client, response):
+    client.check_status_code(response, [200, 201])
+    response_json = response.json()
+    count = response_json["recordCount"]
+    added = response_json["recordIds"]
+    skipped = response_json.get("skippedRecordIds", [])
+
+    print(f"Record count: {count}")
+    print(json.dumps(added, indent=2))
+
+    if skipped:
+        print("Skipped records:")
+        print(json.dumps(skipped, indent=2))

osducli/commands/storage/delete.py

@@ -10,13 +10,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Storage service versions command"""
+"""Storage service delete command"""
 
 import click
 
 from osducli.click_cli import CustomClickCommand, State, global_params
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 
 
 # click entry point
@@ -36,9 +35,11 @@ def delete(state: State, id: str):  # pylint: disable=invalid-name,redefined-bui
         state (State): Global state
         id (str): id to delete
     """
-    connection = CliOsduClient(state.config)
-    url = "records/" + id
-    connection.cli_delete(CONFIG_STORAGE_URL, url, [200, 204])
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
+    # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+    response = record_client.delete_record(recordId=id, bearer_token=client.token_refresher.refresh_token())
+    client.check_status_code(response, [200, 204])
 
     if state.is_user_friendly_mode():
         print("1 record deleted")

osducli/util/file.py

@@ -12,6 +12,7 @@
 
 import errno
 import os
+import glob
 
 
 def get_files_from_path(path: str) -> list:
@@ -34,6 +35,23 @@ def get_files_from_path(path: str) -> list:
     return allfiles
 
 
+def get_files_from_path_with_suffix(path: str, suffix: str) -> list:
+    """Given a path get a list of all files with a certain suffix.
+
+    Args:
+        path (str): path
+        suffix (str): file suffix to filter by
+
+    Returns:
+        list: list of file paths
+    """
+    if os.path.isfile(path):
+        return [path] if path.endswith(suffix) else []
+
+    pattern = os.path.join(path, '**', f'*{suffix}')
+    return glob.glob(pattern, recursive=True)
+
+
 def ensure_directory_exists(directory: str):
     """Create a directory if it doesn't exist"""
     if not os.path.isdir(directory):

{osducli-0.0.42.dist-info → osducli-0.0.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: osducli
-Version: 0.0.42
+Version: 0.0.44
 Summary: OSDU command line
 Author-email: Equinor ASA <mhew@equinor.com>
 License: Apache-2.0
@@ -17,7 +17,7 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.md
 Requires-Dist: click
 Requires-Dist: jmespath
-Requires-Dist: osdu-api[all]==0.28.0rc912
+Requires-Dist: osdu-api[all]==0.28.0rc949
 Requires-Dist: requests
 Requires-Dist: tabulate
 Requires-Dist: packaging
@@ -26,6 +26,10 @@ Requires-Dist: msal-extensions
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault
 Requires-Dist: boto3
+Requires-Dist: google
+Requires-Dist: google-auth
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: google-cloud-storage
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -79,6 +83,17 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.44
+------
+
+- Fix handling of json data in put and post requests
+
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42.dist-info → osducli-0.0.44.dist-info}/RECORD

@@ -1,7 +1,7 @@
-osducli/__init__.py,sha256=OQNSCEnmAuTDwPc4eEhDM7Etcj9zrPFnm1p_Xi7kZr4,615
+osducli/__init__.py,sha256=ibjJ0fg460aSRsjskiE_LSNoQ5I7qcPYsI5C1Z7MFxI,615
 osducli/__main__.py,sha256=4HonhahSaS3Un0f93qnbs_pmJ5pbbVHU-qdhsssVkB0,3661
 osducli/click_cli.py,sha256=7lpYqufxRSMrNZM9M-VANOYKwP0wo-1oxt2eSVHSW_M,10845
-osducli/cliclient.py,sha256=WVVTwIMpIZ1Y4DoRD33kQmSRhiAYXlD5ua57onXVGJo,12152
+osducli/cliclient.py,sha256=sUQyIiomrf-WIcFn9C6UjISvHANLjijI-SVLxQ3T2MY,13242
 osducli/config.py,sha256=lSDxp15JiJQ_J_YjdvI1q243uexzxZpXSYMsSgQIgdQ,7802
 osducli/log.py,sha256=a5pzV2QIOqFEWCGFj01FaZk9PEzMkAr6duHei_dCme8,1549
 osducli/state.py,sha256=b0GHH2oZ-4ogtmHenNkYwNRSE3OpSTL0XklbKVjUAKQ,3335
@@ -73,8 +73,8 @@ osducli/commands/status/__init__.py,sha256=5g1PB_tU0sqP7aplnihjfhMzVVQdlB3Y01QIe
 osducli/commands/status/status.py,sha256=yh6KVUpNYbrLJIH-7GjMyo2MpR8Wvycr4KdkxUI1V5A,4543
 osducli/commands/storage/__init__.py,sha256=Ilc97Kje3VVLZIBJucd_2jsW1GF5En00xF-A6kiLAFM,585
 osducli/commands/storage/_const.py,sha256=a7oIWewDVdgbE1bHcg8QTQDppVT48pgP1Yl-l8tCRhs,847
-osducli/commands/storage/add.py,sha256=ND8EYLN9jo6lCQQnjaf7PgcvzFN216XRE1V6cEuNAu4,2937
-osducli/commands/storage/delete.py,sha256=ybW6Il0uWwwc0VaDDSZdWB916jt8d24Sz6lZPfvZwyo,1464
+osducli/commands/storage/add.py,sha256=nDKP0490dU18Dtpaun3rluaCguJqtAFAxkS1P7WtG2Q,3386
+osducli/commands/storage/delete.py,sha256=ElhE5fdC1yybG7BsVY9Li0x8TG1WeQpkraWWLCB_wR0,1612
 osducli/commands/storage/get.py,sha256=FgjoN_1cY90cbU1osh53XviGvTZaPtRjthUmtP2Y2-k,2136
 osducli/commands/storage/info.py,sha256=CGbA92GJYkWcLXHx1Z4i9g7Auk-ws2Cg6n6X7_m27iw,1345
 osducli/commands/storage/list.py,sha256=bIN76XW70Xml0CCldNvjN-qCumqICjZkQPXs7LTY69M,1413
@@ -102,13 +102,13 @@ osducli/commands/workflow/status.py,sha256=GNSaJrAHrCBjLkYDP0rYtsGicHsvZYLX5otLb
 osducli/commands/workflow/unregister.py,sha256=lTh7yoRE8Sb3W-03L2v16cs0G-0QtfNqFf8fg2t97o4,1415
 osducli/util/__init__.py,sha256=p2bZquIGOPfofHV6GIzWRUv_b9O_pIH5BTyv_gfQT6c,553
 osducli/util/exceptions.py,sha256=VVvJqxLLyYSsitwe9SVfsR7iPXIRaOJzOm7uCxfELjg,790
-osducli/util/file.py,sha256=1QHMoV2duB_ZGsILYWZNSyY99CrgoIS6oP4RSyutJ7I,1349
+osducli/util/file.py,sha256=866XSn7AWFXWgQ9B6OvPZVhb5p8VELbv_wOsUzc55-o,1821
 osducli/util/prompt.py,sha256=0i3eNnxOHRQstvsvfiKnN0lIxXu6sEXIcU8txeYRhNs,7492
 osducli/util/pypi.py,sha256=-DW5CThkKKiOwLp2tg85BmrLKZzkMI9pu8DyWNPZH6E,1192
 osducli/util/service_info.py,sha256=YsVvoRoeG1osFjql2AgVkGoj1TePuhBZf3CQXl2a9As,2577
-osducli-0.0.42.dist-info/LICENSE.md,sha256=9xnGPjJkOAgd0kH4QLlvOIYKqY1M49gWUQpoUxAU-9Y,12788
-osducli-0.0.42.dist-info/METADATA,sha256=58IQOVULe5hcLbAgkNeBTfiiJSTtDaaWw7ikDscFEQA,5925
-osducli-0.0.42.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-osducli-0.0.42.dist-info/entry_points.txt,sha256=gASIcihV0BSJDZOUK-zTzb8RiccZCvKfVZMna9RsEIg,47
-osducli-0.0.42.dist-info/top_level.txt,sha256=lqiP5fuyH8lx7c2emYoIVZNxZAPX-bSwnMH789wxUAY,8
-osducli-0.0.42.dist-info/RECORD,,
+osducli-0.0.44.dist-info/LICENSE.md,sha256=9xnGPjJkOAgd0kH4QLlvOIYKqY1M49gWUQpoUxAU-9Y,12788
+osducli-0.0.44.dist-info/METADATA,sha256=_9Md40hzE-qk91awyx96oZ-Op-MnlE-FK98chhGP6lQ,6190
+osducli-0.0.44.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+osducli-0.0.44.dist-info/entry_points.txt,sha256=gASIcihV0BSJDZOUK-zTzb8RiccZCvKfVZMna9RsEIg,47
+osducli-0.0.44.dist-info/top_level.txt,sha256=lqiP5fuyH8lx7c2emYoIVZNxZAPX-bSwnMH789wxUAY,8
+osducli-0.0.44.dist-info/RECORD,,