ciocore 8.0.0rc1__py2.py3-none-any.whl → 8.1.0__py2.py3-none-any.whl

ciocore/uploader/_uploader.py

@@ -640,8 +640,7 @@ class Uploader(object):
             (UploadWorker, [], {"thread_count": self.args["thread_count"]}),
         ]
 
-        manager = worker.JobManager(job_description)
-        manager.start()
+        manager = worker.JobManager(job_description)        
         return manager
 
     @common.dec_catch_exception(raise_=True)
@@ -744,7 +743,7 @@ class Uploader(object):
         return True
 
     def mark_upload_failed(self, error_message, upload_id):
-        logger.error("Upload failed: \n%s" % error_message)
+        logger.error("Upload failed: %s", error_message)
 
         # report error_message to the app
         self.api_client.make_request(
@@ -793,6 +792,7 @@ class Uploader(object):
 
             # create worker pools
             self.manager = self.create_manager(project)
+            self.manager.start()
 
             # create reporters
             logger.debug("creating report status thread...")
@@ -931,16 +931,17 @@ class Uploader(object):
 
                 if error_messages:
                     self.mark_upload_failed(
-                        error_message="\n".join(error_messages), upload_id=upload_id
+                        error_message="Uploader ERROR: {}".format("\n".join(error_messages)), 
+                                                                  upload_id=upload_id
                     )
 
                     log_file = loggeria.LOG_PATH
-                    sys.stderr.write("Error uploading files:\n")
+                    sys.stderr.write("\nError uploading files:\n")
 
                     for err_msg in error_messages:
-                        sys.stderr.write("\t%s\n", err_msg)
+                        sys.stderr.write("\t{}\n".format(err_msg))
 
-                    sys.stderr.write("\nSee log %s for more details\n\n", log_file)
+                    sys.stderr.write("\nSee log {} for more details\n\n".format(log_file))
 
                 self.error_messages = []
 

ciocore/worker.py

@@ -252,7 +252,7 @@ class ThreadWorker(object):
             return
         
         # don't to anything if we were not provided an out_queue
-        logger.debug("Attempting to put job to out_queue (%s -> %s)", job,self, self.out_queue)
+        logger.debug("Attempting to put job to out_queue (%s -> %s)", job, self.out_queue)
 
         if not self.out_queue:
             return
@@ -261,7 +261,7 @@ class ThreadWorker(object):
         if not WORKING:
             return
         
-        logger.debug("Adding job to out_queue (%s -> %s)", job,self, self.out_queue)
+        logger.debug("Adding job to out_queue (%s -> %s)", job, self.out_queue)
         
         # add item to job
         self.out_queue.put(job)
@@ -491,10 +491,15 @@ class JobManager():
         while not self.error_handler_stop:
             
             try:
-                error = self.error_queue.get(True, 2)
+                error = self.error_queue.get(True, 0.5)
                 
             except queue.Empty:
-                continue
+
+                if self.error:
+                    break
+                
+                else:
+                    continue
             
             logger.error('Got something from the error queue: %s', error)
             self.error.append(error)
@@ -503,7 +508,7 @@ class JobManager():
             except ValueError:
                 pass
             
-            self.stop_work(force=True)
+        self.stop_work(force=True)
 
     def start_error_handler(self):
         

{ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 8.0.0rc1
+Version: 8.1.0
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -51,10 +51,18 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Version:8.0.0-rc.1 -- 05 Apr 2024
+## Version:8.1.0 -- 10 Jun 2024
 
-Major Updates in Version 8.0.0
+* Adds two new calls to the API to get usage; get_compute_usage() and get_storage_usage()
 
+## Version:8.0.1 -- 22 Apr 2024
+
+* The uploader now sends all error messages to the dashboard - not just the first error.
+* Fixed a bug where --log-to-console was not not working as expected in the command line upload subcommand.
+* Fixed some errors in DEBUG-level logging messages.
+
+
+## Version:8.0.0 -- 07 Apr 2024
 
 * The `conductor` commandline interface (CLI) has undergone a complete overhaul to improve its functionality and performance. It is now installed as a Python console script, which simplifies the installation process on all platforms and ensures that the tool is always accessible. In addition, this standardized installation allows for better integration into our Companion application and potentially customers' own proprietary tools.
 * The commandline downloader has been rewritten to improve its performance and reliability. In previous versions, the downloader would fetch all download URLs from the server before starting any downloads. This approach was inefficient and would lead to performance issues when handling jobs with large numbers of tasks. The new downloader fetches tasks in small batches, which allows the download to start almost immediately, regardless of the number of tasks. It provides several new commandline flags, such as filtering based on filenames, and better logging, giving users more control and visibility over the download process. In addition, the new downloader has been written as an event based system that can be seamlessly integrated into other pipeline tools and GUIs. Pipeline TDs need only to register handlers to any of the events that the downloader emits.

{ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-ciocore/VERSION,sha256=oaAkSyjgoKqpHRAw-3LYleaedh24W2mf8BvHs5j0-ak,10
+ciocore/VERSION,sha256=tLQREuN8_s0dGaKmIQt0W8MTSuMhtDpdcGwzW80I8nk,5
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
-ciocore/api_client.py,sha256=TyHXGmK4uTmKV93O_IpAvttZhbexIIZTGP-IVn_WyWM,24599
-ciocore/cli.py,sha256=9OdoazUovfUHxpiAHahH7HJjJcdRLO0_QsdPXjGjlW8,15509
+ciocore/api_client.py,sha256=SBxEwAiwn2XtH7T_ipefUbWhczXjoNdNbQBur1RV-Bw,32810
+ciocore/cli.py,sha256=jZ1lOKQiUcrMhsVmD9SVmPMFwHtgDF4SaoAf2-PBS54,15449
 ciocore/client_db.py,sha256=tTz3bl2xeDPPcYSDS3g3QgV_xYihJMx0Kj6OeN2klK0,12978
 ciocore/common.py,sha256=mBIS6KiYoQsjWe6aIFUGRRvCMl8BIN2kmLZ4J8icap8,14982
 ciocore/compat.py,sha256=5uEXPSog_jxsDMaHBswAKEtfyXT25VgU6WNGIhz9PHU,256
@@ -18,22 +18,22 @@ ciocore/package_tree.py,sha256=vkORKXxQ7dO8l0_96eFwm-5AUVL0rP9bhgWYhW_v3lo,15649
 ciocore/post_install.py,sha256=zu5Ctz2ANbKD-f5G2ODLIhKkWENBi4F3UKKu50OEWrg,1000
 ciocore/retry.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ciocore/validator.py,sha256=f_K7gxz122W_i5AxVx6dKhckOygl8TnmQiVj7tyX5zw,2344
-ciocore/worker.py,sha256=h6JophGBWqLdLdvphlzkkVMt0ojLw7_1Vrz4dikPRYE,21743
+ciocore/worker.py,sha256=YnE0mGqkky9uxPMyDRmvdt1p1ER_GN39AeZPIGw6f4U,21832
 ciocore/auth/__init__.py,sha256=cdS-xZzMq41yXM5cz8sUlcYgo8CJYh8HcCCWmhbDgf0,606
 ciocore/auth/server.py,sha256=8btX9-EokUl6q55V8muDmEV2tvvbTBD0BHeWFbwkzUc,3892
-ciocore/docsite/404.html,sha256=DQCrQsaKh8M0zlCFZmjFHUtqy7_gx_i2Rpc3OoDdDI4,17207
-ciocore/docsite/index.html,sha256=p_Dq_6_8cwZZPDI5zjahfCWUkfbSj14NJtoQFAZv0WI,20945
+ciocore/docsite/404.html,sha256=zUCnDe9r5yFZh8MXcTmzDaFyYBuRHz1gvuyJS2rYiTI,17207
+ciocore/docsite/index.html,sha256=xz215H_KHWBYI1HT7IRpAZ4d1ptUiO8Y4DlOmbEPgPk,20945
 ciocore/docsite/logo.png,sha256=gArgFFWdw8w985-0TkuGIgU_pW9sziEMZdqytXb5WLo,2825
-ciocore/docsite/objects.inv,sha256=s2FKStLlVIQbfG7U2-nw-7rz4unvd1W0u00YtLBxAKo,758
+ciocore/docsite/objects.inv,sha256=XwmLactPEWWC4fAWqHNPBXGsluRxLLTrwDLQqq51ONY,775
 ciocore/docsite/sitemap.xml,sha256=M_V85zl0y2adRvzJAnoCxlZH_Hl7TLnIb1A-6l_xGmI,109
-ciocore/docsite/sitemap.xml.gz,sha256=5eSPEBgK40Dy0VTOM8gl-guV4pLORNHnn-M_KPwVDQk,127
-ciocore/docsite/apidoc/api_client/index.html,sha256=mHDq-zdWuilDKUluzmQNdOsImRoGEnnK5Ul2ApajgdU,170218
-ciocore/docsite/apidoc/apidoc/index.html,sha256=NQn8wjapxa7O2IQhsbE7Y-VMfMFL6Tby-L7qIF6K3m4,26171
-ciocore/docsite/apidoc/config/index.html,sha256=1GWkyClM7LJlLBpx07BqCi3xDkU-SKVPjNUEeKuwbNs,72559
-ciocore/docsite/apidoc/data/index.html,sha256=9DonjpASPYZGYRHjyaodkK3AxSlTHsvm1xdq2cqMxPI,50850
-ciocore/docsite/apidoc/hardware_set/index.html,sha256=DMlpzQCy2ypJ078tcFNrZAvG2sU7P5-ZwU4ZRCObtQg,123042
-ciocore/docsite/apidoc/package_environment/index.html,sha256=M2L6Pu00jYj5AdhHQXrNZjOKVWCXYtgL8I34zL6B0i4,69248
-ciocore/docsite/apidoc/package_tree/index.html,sha256=wKW5dsHBlcDyvCGakSNzYHX8z99F1Vhu2NyqAW6c1ko,109393
+ciocore/docsite/sitemap.xml.gz,sha256=rZcI755TcvgYJvqQWgKtBrKCVKxIJd1pLTDyxi4NFv8,127
+ciocore/docsite/apidoc/api_client/index.html,sha256=mh9VwU1J3Ks6MWvo5a-PcUt5GYdm8s7KsIVDWfnOeXY,188372
+ciocore/docsite/apidoc/apidoc/index.html,sha256=KGsHBY6GBF380w338GfOIMWnwAyyLvw2i1RQFORdwlE,26171
+ciocore/docsite/apidoc/config/index.html,sha256=n3koKPRUza5-uHUt8hK8H8XyH6HG_U2CsGobdaN4KfY,72559
+ciocore/docsite/apidoc/data/index.html,sha256=QhqiXUd_3qk_jGRYB5JiqlJTem8Jg7aKa5mPPDT1978,50850
+ciocore/docsite/apidoc/hardware_set/index.html,sha256=4GEBYy5uLVlJqvdG71RdyrN7sKKuxszxipgmRi_NCbM,123042
+ciocore/docsite/apidoc/package_environment/index.html,sha256=1d4hVlBXPy4bldkBH1GsRjVd-bxdYANmB-Mg7xIAC_4,69248
+ciocore/docsite/apidoc/package_tree/index.html,sha256=7GaOHXeM_oKH8Db25f2-h5kUI1R6pcp0bXxW1COCJxs,109393
 ciocore/docsite/assets/_mkdocstrings.css,sha256=K3bqYEmxlOHQ3-M11JNbBWHCBDBLarkFRm8HuEYrAG4,341
 ciocore/docsite/assets/images/favicon.png,sha256=AjhUxD_Eslt5XuSVHIAZ494Fk__rb5GLXR8qm0elfP4,1870
 ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js,sha256=vMxCR_BtNIcbmbPV6j8Z-YDLQ9ckt4RzvGuzCTg034s,97250
@@ -76,12 +76,12 @@ ciocore/docsite/assets/stylesheets/main.83068744.min.css,sha256=gwaHRErdgDoSsYab
 ciocore/docsite/assets/stylesheets/main.83068744.min.css.map,sha256=q8dWQkgGSfCcYT2Le1J0Tmxpcqxol5kZKaokp9d8Af0,42278
 ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css,sha256=7MiWsGpIt19TLduugHr1OQv1TJ9rEsndnkc2V8D9VjM,12245
 ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css.map,sha256=T8ZT8Mw8ws4vxyx_b1QAwyC9_pnv7r0KVaUdu2ST7ug,3639
-ciocore/docsite/cmdline/docs/index.html,sha256=2SFe5X1oq58UDuYBvUXCcgSBseP5AMgfu-ldKKK1WXA,20051
-ciocore/docsite/cmdline/downloader/index.html,sha256=DAVErhWEPuEtkQr7y9D_wjylzM9j4nWIWvTi4_gMoaA,23171
-ciocore/docsite/cmdline/packages/index.html,sha256=j1YeMgdwZ6FyJGJii05AdtUWKRjyu1fOr3_bKJsLfEU,20923
-ciocore/docsite/cmdline/uploader/index.html,sha256=y0728lIAYC4b9mfpjNVfwQqXoqJrsdG49UY5u7B1p94,25123
-ciocore/docsite/how-to-guides/index.html,sha256=OFrFBTb9tuMn60Jd4oeM10Mw-d-0pyqwoKoo9g1biBM,20100
-ciocore/docsite/search/search_index.json,sha256=ZMAbHLAlwOTujgA32XJTA7SHogRYh9i-ioPFOxjjJ0s,182633
+ciocore/docsite/cmdline/docs/index.html,sha256=Vte3vd2TtRY34Ft6cKvUREeQrWEI9jgdTkQ0XzjZuss,20051
+ciocore/docsite/cmdline/downloader/index.html,sha256=917XE3waukGnIXI9qxl0zYEf3vHVIQPdY3ZybmABAqc,23171
+ciocore/docsite/cmdline/packages/index.html,sha256=Ssb2R7eSubBNR824oc3jjhJfrG3JKOPojA-uiAczgeQ,20923
+ciocore/docsite/cmdline/uploader/index.html,sha256=TtLbHNlu4ofZ4bQBqam87RelWdHnVDSL7_rw_YqBbC8,25123
+ciocore/docsite/how-to-guides/index.html,sha256=rF-DMyk4k_jGUJRF3dKtMMiOgXS8OvBqAn-zxWy3AGs,20100
+ciocore/docsite/search/search_index.json,sha256=vBLU-rkC0pxOPX0O3MNKXRKGfgsaS1-91KsV17Gf9SY,189092
 ciocore/docsite/stylesheets/extra.css,sha256=_Cxe9Dhg1BBi6Kqaz_iZD9z9VyqxA9vtONRjP4PVic0,354
 ciocore/docsite/stylesheets/tables.css,sha256=LE_zwGRxGcdPIy-9QiVPecOzlEBSqZb_WP5vDkFE0ZM,3235
 ciocore/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -95,7 +95,7 @@ ciocore/downloader/perpetual_downloader.py,sha256=cD7lnBH75-c-ZVVPHZc1vSnDhgJOnG
 ciocore/downloader/registry.py,sha256=_JIOuqpWkJkgJGN33nt-DCvqN9Gw3xeFhzPq4RUxIoE,2903
 ciocore/downloader/reporter.py,sha256=p1NK9k6iQ-jt7lRvZR0xFz0cGb2yo8tQcjlvYKR9SWM,4501
 ciocore/uploader/__init__.py,sha256=hxRFJf5Lo86rtRObFXSjjot8nybQd-SebSfYCbgZwow,24
-ciocore/uploader/_uploader.py,sha256=AaCwRo1rGkdMA-XACx9k4Z73R97ndJUVMemUZNV-Ark,37353
+ciocore/uploader/_uploader.py,sha256=40nzqO5DuFi4sx31VvjWxZPNkrWsWqM9jtFVxs_-o3o,37479
 ciocore/uploader/upload_stats/__init__.py,sha256=Lg1y4zq1i0cwc6Hh2K1TAQDYymLff49W-uIo1xjcvdI,5309
 ciocore/uploader/upload_stats/stats_formats.py,sha256=giNirtObU66VALWghPFSRhg3q_vw5MvESsnXhb_I3y8,2402
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -103,9 +103,9 @@ tests/extra_env_fixtures.py,sha256=8qvU4d8SXGKzRVNR5whVqKCQOwOVMiFVfbKBAjxa2gE,1
 tests/instance_type_fixtures.py,sha256=uIzQduqKQVgjllMuyXaYnRC-pwqk5lnTx3NY2M5Nujo,4320
 tests/package_fixtures.py,sha256=CsJnhB7oYzIxJH7b1tCOPyvnnVSCqEbSPhtCnsHL-nA,5070
 tests/project_fixtures.py,sha256=iBm_th_JtAw76vlNu7Jjhh9tLH4oOaNi-MgtPzCV7yQ,138
-tests/test_api_client.py,sha256=tnYkIYvDILGJRatZM0nrtr83x8yBJstMP0Rt151U2gI,6757
+tests/test_api_client.py,sha256=4jhj-YrBPcLj7XZn3ngguau2DPxpCYGMSuqPJ3hW0GQ,14497
 tests/test_base_downloader.py,sha256=SS7tWKv2ZZhpUDk4UCg1TkrNrpntjSewgzLl1mEubSE,3603
-tests/test_cli.py,sha256=yWWGLip8WgnDAB0vg7uhBU5eeEJieFLQ78Pn7IN-uWI,6088
+tests/test_cli.py,sha256=_WTs2SWlEgd6wtg1hmOBlFnbWVdFLvqp0KqNhy-y2e8,5532
 tests/test_common.py,sha256=tY_-SY-JmJX09UehFs9RIDqZ785AmhfTl6eVKJeIUFY,763
 tests/test_config.py,sha256=-_G682Ss3Zr1FmcMkjla1zAZprX2tQKpKc5_wD28rII,13340
 tests/test_data.py,sha256=NIBXpCjG3Os3vpc1CkiVONrebro8D_jqQyJ0N3kbucU,5433
@@ -120,8 +120,8 @@ tests/test_uploader.py,sha256=B1llTJt_fqR6e_V_Jxfw9z73QgkFlEPU87xLYGzt-TQ,2914
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-8.0.0rc1.dist-info/METADATA,sha256=OiC555SBSXja3tKUjJCh3NWrvYpRdzUTLGHP4nSlMaY,18196
-ciocore-8.0.0rc1.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-ciocore-8.0.0rc1.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-8.0.0rc1.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-8.0.0rc1.dist-info/RECORD,,
+ciocore-8.1.0.dist-info/METADATA,sha256=lXpW_6Xfu_wAL__6D_ePH3DcB7oyBRtIMNvv_3sK3bo,18564
+ciocore-8.1.0.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+ciocore-8.1.0.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-8.1.0.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-8.1.0.dist-info/RECORD,,

tests/test_api_client.py

@@ -2,9 +2,11 @@
 
    isort:skip_file
 """
+import datetime
+import json
 import sys
 import unittest
-import json
+
 try:
     from unittest import mock
 except ImportError:
@@ -13,6 +15,8 @@ except ImportError:
 from ciocore import api_client
 
 from ciocore.api_client import request_extra_environment
+from ciocore.api_client import _get_compute_usage, get_compute_usage
+from ciocore.api_client import _get_storage_usage, get_storage_usage
 
 class ApiClientTest(unittest.TestCase):
     @staticmethod
@@ -164,3 +168,172 @@ class TestRequestExtraEnvironment(unittest.TestCase):
 
         self.assertEqual("Error: Could not get account environment!", str(context.exception))
         mock_ApiClient.assert_called_once()
+
+class TestGetComputeUsage(unittest.TestCase):
+
+    def setUp(self):
+        
+        # Precison is rounded to 4 places to avoid FP issues.
+        self.compute_response_data = {
+            "data": [
+                {
+                    "cores": 0.5, 
+                    "instance_cost": 0.0200, 
+                    "license_cost": 0.0200, 
+                    "minutes": 6.9700, 
+                    "self_link": 0, 
+                    "start_time": "Tue, 09 Jan 2024 18:00:00 GMT"
+                }, 
+                {
+                    "cores": 0.4, 
+                    "instance_cost": 0.0200, 
+                    "license_cost": 0.0200, 
+                    "minutes": 6.9600, 
+                    "self_link": 1, 
+                    "start_time": "Tue, 09 Jan 2024 19:00:00 GMT"
+                }, 
+                {
+                    "cores": 0.9613, 
+                    "instance_cost": 0.0400, 
+                    "license_cost": 0.0800, 
+                    "minutes": 7.2100, 
+                    "self_link": 2, 
+                    "start_time": "Tue, 16 Jan 2024 17:00:00 GMT"
+                }, 
+            ]
+        }
+     
+        self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.compute_response_data))
+        self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
+
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_query_raw_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks for successful execution
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "123"
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        start_time = datetime.datetime(2024, 1, 1)
+        end_time = datetime.datetime(2024, 3, 20)
+
+        result = _get_compute_usage(start_time, end_time)
+
+        self.assertEqual(result, self.compute_response_data['data'])
+        
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks for successful execution
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "123"
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        start_time = datetime.datetime(2024, 1, 1)
+        end_time = datetime.datetime(2024, 3, 20)
+
+        result = get_compute_usage(start_time=start_time, end_time=end_time)
+
+        self.assertEqual(result, {  '2024-01-09': {'cost': 0.08,
+                                                 'corehours': 0.9, 
+                                                 'walltime': 13.93},
+                                    '2024-01-16': { 'cost': 0.12,
+                                                    'corehours': 0.9613, 
+                                                    'walltime': 7.21}})
+        
+    @mock.patch("ciocore.api_client.ApiClient")
+    def test_get_compute_usage_api_failure(self, mock_ApiClient):
+        # Set up mock for API failure
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
+
+        start_time = datetime.datetime(2024, 1, 1)
+        end_time = datetime.datetime(2024, 3, 20)
+        
+        # Assert exception raised when the API call fails
+        with self.assertRaises(Exception) as context:
+            get_compute_usage(start_time, end_time)
+
+        self.assertIn('Failed to query compute usage', str(context.exception))
+        mock_ApiClient.assert_called_once()    
+class TestGetStorageUsage(unittest.TestCase):
+
+    def setUp(self):
+               
+        self.storage_response_data = {
+            "data": [
+                {
+                "cost": "28.96", 
+                "cost_per_day": [
+                    4.022,
+                    4.502, 
+                    4.502, 
+                    5.102, 
+                    5.102, 
+                    5.732
+                ], 
+                "currency": "USD", 
+                "daily_price": "0.006", 
+                "end_date": "2024-01-07", 
+                "gibs_per_day": [
+                    679.714,
+                    750.34, 
+                    750.34, 
+                    850.36, 
+                    850.35, 
+                    955.32
+                ], 
+                "gibs_used": "806.07", 
+                "monthly_price": "0.18", 
+                "start_date": "2024-01-01", 
+                "storage_unit": "GiB"
+                }
+            ]
+            }
+        
+        self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.storage_response_data))
+        self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
+        
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_query_raw_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks for successful execution
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "123"
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        start_time = datetime.datetime(2024, 5, 1)
+        end_time = datetime.datetime(2024, 5, 17)
+
+        result = _get_storage_usage(start_time, end_time)
+
+        self.assertEqual(result, self.storage_response_data['data'][0])
+
+    @mock.patch("ciocore.api_client.ApiClient")
+    @mock.patch("ciocore.api_client.read_conductor_credentials")
+    @mock.patch("ciocore.api_client.account_id_from_jwt")
+    def test_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
+        # Set up mocks for successful execution
+        mock_read_conductor_credentials.return_value = "valid_token"
+        mock_account_id_from_jwt.return_value = "123"
+        mock_api_instance = mock_ApiClient.return_value
+        mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
+
+        start_time = datetime.datetime(2024, 1, 1)
+        end_time = datetime.datetime(2024, 1, 7)
+
+        result = get_storage_usage(start_time=start_time, end_time=end_time)
+        
+        self.assertEqual(result, {  '2024-01-01': {'cost': 4.022, 'GiB': 679.714},
+                                    '2024-01-02': {'cost': 4.502, 'GiB': 750.34},
+                                    '2024-01-03': {'cost': 4.502, 'GiB': 750.34},
+                                    '2024-01-04': {'cost': 5.102, 'GiB': 850.36},
+                                    '2024-01-05': {'cost': 5.102, 'GiB': 850.35},
+                                    '2024-01-06': {'cost': 5.732, 'GiB': 955.32}
+                                  })

tests/test_cli.py

@@ -13,8 +13,6 @@ class CliTestUploaderOptions(unittest.TestCase):
             "database_filepath": None,
             "location": None,
             "md5_caching": True,
-            "log_level": "INFO",
-            "log_dir": None,
             "thread_count": mock.ANY,
         }
         init_patcher = mock.patch("ciocore.cli.Uploader.__init__", autospec=True)
@@ -43,18 +41,6 @@ class CliTestUploaderOptions(unittest.TestCase):
         expected.update({"md5_caching": False})
         self.mock_init.assert_called_once_with(mock.ANY, expected)
 
-    def test_log_level_arg(self):
-        self.runner.invoke(cli_upload, ["--log_level", "DEBUG"])
-        expected = self.default_args
-        expected.update({"log_level": "DEBUG"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_log_dir_arg(self):
-        self.runner.invoke(cli_upload, ["--log_dir", "foo"])
-        expected = self.default_args
-        expected.update({"log_dir": "foo"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
     def test_thread_count_arg(self):
         self.runner.invoke(cli_upload, ["--thread_count", 4])
         expected = self.default_args