osducli 0.0.42__tar.gz → 0.0.43__tar.gz

{osducli-0.0.42/src/osducli.egg-info → osducli-0.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: osducli
-Version: 0.0.42
+Version: 0.0.43
 Summary: OSDU command line
 Author-email: Equinor ASA <mhew@equinor.com>
 License: Apache-2.0
@@ -26,6 +26,10 @@ Requires-Dist: msal-extensions
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault
 Requires-Dist: boto3
+Requires-Dist: google
+Requires-Dist: google-auth
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: google-cloud-storage
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -79,6 +83,12 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.43}/README.rst

@@ -35,6 +35,12 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.43}/pyproject.toml

@@ -37,6 +37,11 @@ dependencies = [
     "azure-keyvault",
     # AWS specific
     "boto3",
+    # GCP specific
+    "google",
+    "google-auth",
+    "google-auth-oauthlib",
+    "google-cloud-storage",
 ]
 
 [project.urls]

{osducli-0.0.42 → osducli-0.0.43}/src/osducli/__init__.py

@@ -12,4 +12,4 @@
 
 """ OSDU command line environment"""
 
-__VERSION__ = "0.0.42"
+__VERSION__ = "0.0.43"

{osducli-0.0.42 → osducli-0.0.43}/src/osducli/cliclient.py

@@ -274,7 +274,11 @@ class CliOsduClient(BaseClient):
             requests.Response: Response object from the HTTP call
         """
         url = self.url_from_config(config_url_key, url_extra_path)
-        response = self.make_request(method=HttpMethod.DELETE, url=url)
+
+        # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+        response = self.make_request(method=HttpMethod.DELETE, url=url,
+                                     bearer_token=self.token_refresher.refresh_token())
+
         self.check_status_code(response, ok_status_codes)
         return response
 

{osducli-0.0.42 → osducli-0.0.43}/src/osducli/commands/storage/add.py

@@ -15,12 +15,12 @@
 import json
 
 import click
+from osdu_api.model.storage.record import Record
 
 from osducli.click_cli import CustomClickCommand, State, command_with_output
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 from osducli.log import get_logger
-from osducli.util.file import get_files_from_path
+from osducli.util.file import get_files_from_path_with_suffix
 
 logger = get_logger(__name__)
 
@@ -51,41 +51,57 @@ def _click_command(state: State, path: str, batch: int):
     return add_records(state, path, batch)
 
 
-def add_records(state: State, path: str, batch: int) -> dict:
+def add_records(state: State, path: str, batch: int = None):
     """Add or update a record
 
     Args:
         state (State): Global state
         path (str): Path to a record or records to add.
         batch (int): Batch size per API call. If None then ingest as is
-    Returns:
-        dict: Response from service
     """
-    if batch is not None:
-        raise NotImplementedError("--batch is not supported yet for storage add")
-
-    connection = CliOsduClient(state.config)
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
 
-    files = get_files_from_path(path)
+    files = get_files_from_path_with_suffix(path, ".json")
     logger.debug("Files list: %s", files)
 
-    # TODO: Check if loaded file is already an array, or a single file
-    # TODO: Batch uploads
-    responses = []
+    if batch is not None:
+        logger.info("Batching records with size %s", batch)
+        file_batches = chunk_list(files, batch)
+        for file_batch in file_batches:
+            response = add_record_batch(record_client, file_batch)
+            handle_response(client, response)
+    else:
+        response = add_record_batch(record_client, files)
+        handle_response(client, response)
+
+
+def add_record_batch(record_client, files):
+    record_list = []
     for filepath in files:
-        if filepath.endswith(".json"):
-            with open(filepath, encoding="utf-8") as file:
-                storage_object = json.load(file)
-
-                logger.info("Processing file %s.", filepath)
-                if isinstance(storage_object, list):
-                    payload = storage_object
-                else:
-                    payload = [storage_object]
-
-                response_json = None
-                response_json = connection.cli_put_returning_json(
-                    CONFIG_STORAGE_URL, "records", payload, [200, 201]
-                )
-                responses.append(response_json)
-    return responses
+        with open(filepath, encoding="utf-8") as file:
+            storage_object = json.load(file)
+
+            logger.info("Processing file %s.", filepath)
+            record_list.append(Record.from_dict(storage_object))
+
+    return record_client.create_update_records(record_list)
+
+
+def chunk_list(lst, size):
+    return [lst[i:i + size] for i in range(0, len(lst), size)]
+
+
+def handle_response(client, response):
+    client.check_status_code(response, [200, 201])
+    response_json = response.json()
+    count = response_json["recordCount"]
+    added = response_json["recordIds"]
+    skipped = response_json.get("skippedRecordIds", [])
+
+    print(f"Record count: {count}")
+    print(json.dumps(added, indent=2))
+
+    if skipped:
+        print("Skipped records:")
+        print(json.dumps(skipped, indent=2))

{osducli-0.0.42 → osducli-0.0.43}/src/osducli/commands/storage/delete.py

@@ -10,13 +10,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Storage service versions command"""
+"""Storage service delete command"""
 
 import click
 
 from osducli.click_cli import CustomClickCommand, State, global_params
 from osducli.cliclient import CliOsduClient, handle_cli_exceptions
-from osducli.config import CONFIG_STORAGE_URL
 
 
 # click entry point
@@ -36,9 +35,11 @@ def delete(state: State, id: str):  # pylint: disable=invalid-name,redefined-bui
         state (State): Global state
         id (str): id to delete
     """
-    connection = CliOsduClient(state.config)
-    url = "records/" + id
-    connection.cli_delete(CONFIG_STORAGE_URL, url, [200, 204])
+    client = CliOsduClient(state.config)
+    record_client = client.get_storage_record_client()
+    # TODO: Fix bug in SDK for DELETE. Workaround is to give bearer_token
+    response = record_client.delete_record(recordId=id, bearer_token=client.token_refresher.refresh_token())
+    client.check_status_code(response, [200, 204])
 
     if state.is_user_friendly_mode():
         print("1 record deleted")

{osducli-0.0.42 → osducli-0.0.43}/src/osducli/util/file.py

@@ -12,6 +12,7 @@
 
 import errno
 import os
+import glob
 
 
 def get_files_from_path(path: str) -> list:
@@ -34,6 +35,23 @@ def get_files_from_path(path: str) -> list:
     return allfiles
 
 
+def get_files_from_path_with_suffix(path: str, suffix: str) -> list:
+    """Given a path get a list of all files with a certain suffix.
+
+    Args:
+        path (str): path
+        suffix (str): file suffix to filter by
+
+    Returns:
+        list: list of file paths
+    """
+    if os.path.isfile(path):
+        return [path] if path.endswith(suffix) else []
+
+    pattern = os.path.join(path, '**', f'*{suffix}')
+    return glob.glob(pattern, recursive=True)
+
+
 def ensure_directory_exists(directory: str):
     """Create a directory if it doesn't exist"""
     if not os.path.isdir(directory):

{osducli-0.0.42 → osducli-0.0.43/src/osducli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: osducli
-Version: 0.0.42
+Version: 0.0.43
 Summary: OSDU command line
 Author-email: Equinor ASA <mhew@equinor.com>
 License: Apache-2.0
@@ -26,6 +26,10 @@ Requires-Dist: msal-extensions
 Requires-Dist: azure-identity
 Requires-Dist: azure-keyvault
 Requires-Dist: boto3
+Requires-Dist: google
+Requires-Dist: google-auth
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: google-cloud-storage
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -79,6 +83,12 @@ For more information, specify the `-h` flag:
 Change Log
 ==========
 
+0.0.43
+------
+
+- Fix storage add command
+- Implement storage add batching
+
 0.0.42
 ------
 

{osducli-0.0.42 → osducli-0.0.43}/src/osducli.egg-info/requires.txt

@@ -9,6 +9,10 @@ msal-extensions
 azure-identity
 azure-keyvault
 boto3
+google
+google-auth
+google-auth-oauthlib
+google-cloud-storage
 
 [dev]
 black

{osducli-0.0.42 → osducli-0.0.43}/tests/test_cliclient.py

@@ -205,11 +205,14 @@ def test_cli_delete(mocker, cli_client, config, path):
     response_mock = Mock()
     response_mock.status_code = 200
     mock_delete = mocker.patch.object(cli_client, "make_request", return_value=response_mock)
+    mock_token_refresher = mocker.Mock()
+    mock_token_refresher.refresh_token.return_value = "some-token"
+    cli_client.token_refresher = mock_token_refresher
 
     response = cli_client.cli_delete(config, path)
 
     mock_delete.assert_called_once()
-    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_" + config + path)
+    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_" + config + path, bearer_token="some-token")
     assert response_mock == response
 
 @pytest.mark.parametrize("ok_status_codes, actual_status_code",
@@ -224,11 +227,14 @@ def test_cli_delete_status_codes(mocker, cli_client, ok_status_codes, actual_sta
     response_mock = Mock()
     response_mock.status_code = actual_status_code
     mock_delete = mocker.patch.object(cli_client, "make_request", return_value=response_mock)
+    mock_token_refresher = mocker.Mock()
+    mock_token_refresher.refresh_token.return_value = "some-token"
+    cli_client.token_refresher = mock_token_refresher
 
     response = cli_client.cli_delete("config", "/path", ok_status_codes)
 
     mock_delete.assert_called_once()
-    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_config/path")
+    mock_delete.assert_called_with(method=HttpMethod.DELETE, url="https://dummy.com/core_config/path", bearer_token="some-token")
     assert response_mock == response
 
 # endregion test cli_delete