osducli 0.0.42__tar.gz → 0.0.44__tar.gz

{osducli-0.0.42/src/osducli.egg-info → osducli-0.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: osducli
-Version: 0.0.42
+Version: 0.0.44
 Summary: OSDU command line
 Author-email: Equinor ASA <mhew@equinor.com>
 License: Apache-2.0
@@ -17,7 +17,7 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.md
 Requires-Dist: click
 Requires-Dist: jmespath
-Requires-Dist: osdu-api[all]==0.28.0rc912
+Requires-Dist: osdu-api[all]==0.28.0rc949
 Requires-Dist: requests
 Requires-Dist: tabulate
 Requires-Dist: packaging
@@ -26,6 +26,10 @@ Requires-Dist: msal-extensions
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault
 Requires-Dist: boto3
+Requires-Dist: google
+Requires-Dist: google-auth
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: google-cloud-storage
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -79,6 +83,17 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.44
+------
+
+- Fix handling of json data in put and post requests
+
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.44}/README.rst

@@ -35,6 +35,17 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.44
+------
+
+- Fix handling of json data in put and post requests
+
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.44}/pyproject.toml

@@ -26,7 +26,7 @@ keywords = ["osdu"]
 dependencies = [
     "click",
     "jmespath",
-    "osdu-api[all]==0.28.0rc912",
+    "osdu-api[all]==0.28.0rc949",
     "requests",
     "tabulate",
     "packaging",
@@ -37,6 +37,11 @@ dependencies = [
     "azure-keyvault",
     # AWS specific
     "boto3",
+    # GCP specific
+    "google",
+    "google-auth",
+    "google-auth-oauthlib",
+    "google-cloud-storage",
 ]
 
 [project.urls]

{osducli-0.0.42 → osducli-0.0.44}/src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.42"
+__VERSION__ = "0.0.44"

{osducli-0.0.42 → osducli-0.0.44}/src/osducli/cliclient.py

@@ -157,12 +157,14 @@ class CliOsduClient(BaseClient):
         return url
 
     def check_status_code(self, response: requests.Response, ok_status_codes: list = None):
+        """Check the status code of the response and raise an exception if not in the list of ok status codes"""
         if ok_status_codes is None:
             ok_status_codes = [200]
         if response.status_code not in ok_status_codes:
             raise HTTPError(response=response)
 
     def get_search_client(self) -> SearchClient:
+        """Get a client for the search service"""
         search_url = self.url_from_config(CONFIG_SEARCH_URL, "")
         if search_url.endswith("/"):
             search_url = search_url[:-1]
@@ -174,6 +176,7 @@ class CliOsduClient(BaseClient):
         )
 
     def get_storage_record_client(self) -> RecordClient:
+        """Get a client for the storage record service"""
         storage_url = self.url_from_config(CONFIG_STORAGE_URL, "")
         if storage_url.endswith("/"):
             storage_url = storage_url[:-1]
@@ -185,6 +188,7 @@ class CliOsduClient(BaseClient):
         )
 
     def get_entitlements_client(self) -> EntitlementsClient:
+        """Get a client for the entitlements service"""
         entitlements_url = self.url_from_config(CONFIG_ENTITLEMENTS_URL, "")
         if entitlements_url.endswith("/"):
             entitlements_url = entitlements_url[:-1]
@@ -195,7 +199,15 @@ class CliOsduClient(BaseClient):
             token_refresher=self.token_refresher
         )
 
-    def get_wellbore_ddms_client(self, url_extra_path: str) -> WellboreDdmsClient:
+    def get_wellbore_ddms_client(
+        self,
+        url_extra_path: str
+    ) -> WellboreDdmsClient:
+        """Get a client for the wellbore ddms service
+
+        Args:
+            url_extra_path (str): extra path to add to the base path
+        """
         wellbore_ddms_url = self.url_from_config(CONFIG_WELLBORE_DDMS_URL, url_extra_path)
         return WellboreDdmsClient(
             wellbore_ddms_url=wellbore_ddms_url,
@@ -205,9 +217,12 @@ class CliOsduClient(BaseClient):
         )
 
     def cli_get(
-        self, config_url_key: str, url_extra_path: str, ok_status_codes: list = None
+        self,
+        config_url_key: str,
+        url_extra_path: str,
+        ok_status_codes: list = None
     ) -> requests.Response:
-        """[summary]
+        """Basic GET call to the given url, returning the response object.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -220,9 +235,12 @@ class CliOsduClient(BaseClient):
         return response
 
     def cli_get_returning_json(
-        self, config_url_key: str, url_extra_path: str, ok_status_codes: list = None
+        self,
+        config_url_key: str,
+        url_extra_path: str,
+        ok_status_codes: list = None
     ) -> dict:
-        """[summary]
+        """Basic GET call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -241,7 +259,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> dict:
-        """[summary]
+        """Basic POST call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -253,6 +271,8 @@ class CliOsduClient(BaseClient):
             dict: returned json
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.POST, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response.json()
@@ -263,7 +283,7 @@ class CliOsduClient(BaseClient):
         url_extra_path: str,
         ok_status_codes: list = None,
     ) -> requests.Response:
-        """[summary]
+        """Basic DELETE call to the given url.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -274,7 +294,11 @@ class CliOsduClient(BaseClient):
             requests.Response: Response object from the HTTP call
         """
         url = self.url_from_config(config_url_key, url_extra_path)
-        response = self.make_request(method=HttpMethod.DELETE, url=url)
+
+        # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+        response = self.make_request(method=HttpMethod.DELETE, url=url,
+                                     bearer_token=self.token_refresher.refresh_token())
+
         self.check_status_code(response, ok_status_codes)
         return response
 
@@ -285,7 +309,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> requests.Response:
-        """[summary]
+        """Basic PUT call to the given url.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -294,6 +318,8 @@ class CliOsduClient(BaseClient):
             ok_status_codes (list, optional): Optional status codes to check for successful call.
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.PUT, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response
@@ -305,7 +331,7 @@ class CliOsduClient(BaseClient):
         data: str | dict,
         ok_status_codes: list = None,
     ) -> dict:
-        """PUT the data to the given url.
+        """Basic PUT call to the given url, returning the json.
 
         Args:
             config_url_key (str): key in configuration for the base path
@@ -317,6 +343,8 @@ class CliOsduClient(BaseClient):
             dict: returned json
         """
         url = self.url_from_config(config_url_key, url_extra_path)
+        if isinstance(data, dict):
+            data = json.dumps(data)
         response = self.make_request(method=HttpMethod.PUT, url=url, data=data)
         self.check_status_code(response, ok_status_codes)
         return response.json()

{osducli-0.0.42 → osducli-0.0.44}/src/osducli/commands/storage/add.py

@@ -15,12 +15,12 @@
 import json
 
 import click
+from osdu_api.model.storage.record import Record
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 from osducli.log import get_logger
-from osducli.util.file import get_files_from_path
+from osducli.util.file import get_files_from_path_with_suffix
 
 logger = get_logger(__name__)
 
@@ -51,41 +51,57 @@ def _click_command(state: State, path: str, batch: int):
     return add_records(state, path, batch)
 
 
-def add_records(state: State, path: str, batch: int) -> dict:
+def add_records(state: State, path: str, batch: int = None):
     """Add or update a record
 
     Args:
         state (State): Global state
         path (str): Path to a record or records to add.
         batch (int): Batch size per API call. If None then ingest as is
-    Returns:
-        dict: Response from service
     """
-    if batch is not None:
-        raise NotImplementedError("--batch is not supported yet for storage add")
-
-    connection = CliOsduClient(state.config)
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
 
-    files = get_files_from_path(path)
+    files = get_files_from_path_with_suffix(path, ".json")
     logger.debug("Files list: %s", files)
 
-    # TODO: Check if loaded file is already an array, or a single file
-    # TODO: Batch uploads
-    responses = []
+    if batch is not None:
+        logger.info("Batching records with size %s", batch)
+        file_batches = chunk_list(files, batch)
+        for file_batch in file_batches:
+            response = add_record_batch(record_client, file_batch)
+            handle_response(client, response)
+    else:
+        response = add_record_batch(record_client, files)
+        handle_response(client, response)
+
+
+def add_record_batch(record_client, files):
+    record_list = []
     for filepath in files:
-        if filepath.endswith(".json"):
-            with open(filepath, encoding="utf-8") as file:
-                storage_object = json.load(file)
-
-                logger.info("Processing file %s.", filepath)
-                if isinstance(storage_object, list):
-                    payload = storage_object
-                else:
-                    payload = [storage_object]
-
-                response_json = None
-                response_json = connection.cli_put_returning_json(
-                    CONFIG_STORAGE_URL, "records", payload, [200, 201]
-                )
-                responses.append(response_json)
-    return responses
+        with open(filepath, encoding="utf-8") as file:
+            storage_object = json.load(file)
+
+            logger.info("Processing file %s.", filepath)
+            record_list.append(Record.from_dict(storage_object))
+
+    return record_client.create_update_records(record_list)
+
+
+def chunk_list(lst, size):
+    return [lst[i:i + size] for i in range(0, len(lst), size)]
+
+
+def handle_response(client, response):
+    client.check_status_code(response, [200, 201])
+    response_json = response.json()
+    count = response_json["recordCount"]
+    added = response_json["recordIds"]
+    skipped = response_json.get("skippedRecordIds", [])
+
+    print(f"Record count: {count}")
+    print(json.dumps(added, indent=2))
+
+    if skipped:
+        print("Skipped records:")
+        print(json.dumps(skipped, indent=2))

{osducli-0.0.42 → osducli-0.0.44}/src/osducli/commands/storage/delete.py

@@ -10,13 +10,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Storage service versions command"""
+"""Storage service delete command"""
 
 import click
 
 from osducli.click_cli import CustomClickCommand, State, global_params
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 
 
 # click entry point
@@ -36,9 +35,11 @@ def delete(state: State, id: str):  # pylint: disable=invalid-name,redefined-bui
         state (State): Global state
         id (str): id to delete
     """
-    connection = CliOsduClient(state.config)
-    url = "records/" + id
-    connection.cli_delete(CONFIG_STORAGE_URL, url, [200, 204])
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
+    # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+    response = record_client.delete_record(recordId=id, bearer_token=client.token_refresher.refresh_token())
+    client.check_status_code(response, [200, 204])
 
     if state.is_user_friendly_mode():
         print("1 record deleted")

{osducli-0.0.42 → osducli-0.0.44}/src/osducli/util/file.py

@@ -12,6 +12,7 @@
 
 import errno
 import os
+import glob
 
 
 def get_files_from_path(path: str) -> list:
@@ -34,6 +35,23 @@ def get_files_from_path(path: str) -> list:
     return allfiles
 
 
+def get_files_from_path_with_suffix(path: str, suffix: str) -> list:
+    """Given a path get a list of all files with a certain suffix.
+
+    Args:
+        path (str): path
+        suffix (str): file suffix to filter by
+
+    Returns:
+        list: list of file paths
+    """
+    if os.path.isfile(path):
+        return [path] if path.endswith(suffix) else []
+
+    pattern = os.path.join(path, '**', f'*{suffix}')
+    return glob.glob(pattern, recursive=True)
+
+
 def ensure_directory_exists(directory: str):
     """Create a directory if it doesn't exist"""
     if not os.path.isdir(directory):

{osducli-0.0.42 → osducli-0.0.44/src/osducli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: osducli
-Version: 0.0.42
+Version: 0.0.44
 Summary: OSDU command line
 Author-email: Equinor ASA <mhew@equinor.com>
 License: Apache-2.0
@@ -17,7 +17,7 @@ Description-Content-Type: text/x-rst
 License-File: LICENSE.md
 Requires-Dist: click
 Requires-Dist: jmespath
-Requires-Dist: osdu-api[all]==0.28.0rc912
+Requires-Dist: osdu-api[all]==0.28.0rc949
 Requires-Dist: requests
 Requires-Dist: tabulate
 Requires-Dist: packaging
@@ -26,6 +26,10 @@ Requires-Dist: msal-extensions
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault
 Requires-Dist: boto3
+Requires-Dist: google
+Requires-Dist: google-auth
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: google-cloud-storage
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -79,6 +83,17 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.44
+------
+
+- Fix handling of json data in put and post requests
+
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.44}/src/osducli.egg-info/requires.txt

@@ -1,6 +1,6 @@
 click
 jmespath
-osdu-api[all]==0.28.0rc912
+osdu-api[all]==0.28.0rc949
 requests
 tabulate
 packaging
@@ -9,6 +9,10 @@ msal-extensions
 azure-identity
 azure-keyvault
 boto3
+google
+google-auth
+google-auth-oauthlib
+google-cloud-storage
 
 [dev]
 black

{osducli-0.0.42 → osducli-0.0.44}/tests/test_cliclient.py

@@ -15,6 +15,7 @@
 from unittest.mock import MagicMock, Mock
 
 import pytest
+import json
 from osdu_api.clients.base_client import BaseClient
 from osdu_api.model.http_method import HttpMethod
 
@@ -159,6 +160,8 @@ def test_cli_put(mocker, cli_client, config, path, data):
     response = cli_client.cli_put(config, path, data)
 
     mock_put.assert_called_once()
+    if isinstance(data, dict):
+        data = json.dumps(data)
     mock_put.assert_called_with(method=HttpMethod.PUT, url="https://dummy.com/core_" + config + path, data=data)
     assert response_mock == response
 
@@ -205,11 +208,14 @@ def test_cli_delete(mocker, cli_client, config, path):
     response_mock = Mock()
     response_mock.status_code = 200
     mock_delete = mocker.patch.object(cli_client, "make_request", return_value=response_mock)
+    mock_token_refresher = mocker.Mock()
+    mock_token_refresher.refresh_token.return_value = "some-token"
+    cli_client.token_refresher = mock_token_refresher
 
     response = cli_client.cli_delete(config, path)
 
     mock_delete.assert_called_once()
-    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_" + config + path)
+    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_" + config + path, bearer_token="some-token")
     assert response_mock == response
 
 @pytest.mark.parametrize("ok_status_codes, actual_status_code",
@@ -224,11 +230,14 @@ def test_cli_delete_status_codes(mocker, cli_client, ok_status_codes, actual_sta
     response_mock = Mock()
     response_mock.status_code = actual_status_code
     mock_delete = mocker.patch.object(cli_client, "make_request", return_value=response_mock)
+    mock_token_refresher = mocker.Mock()
+    mock_token_refresher.refresh_token.return_value = "some-token"
+    cli_client.token_refresher = mock_token_refresher
 
     response = cli_client.cli_delete("config", "/path", ok_status_codes)
 
     mock_delete.assert_called_once()
-    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_config/path")
+    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_config/path", bearer_token="some-token")
     assert response_mock == response
 
 # endregion test cli_delete