ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. ciocore/VERSION +1 -1
  2. ciocore/__init__.py +23 -1
  3. ciocore/api_client.py +655 -160
  4. ciocore/auth/__init__.py +5 -3
  5. ciocore/cli.py +501 -0
  6. ciocore/common.py +15 -13
  7. ciocore/conductor_submit.py +77 -60
  8. ciocore/config.py +127 -13
  9. ciocore/data.py +162 -77
  10. ciocore/docsite/404.html +746 -0
  11. ciocore/docsite/apidoc/api_client/index.html +3605 -0
  12. ciocore/docsite/apidoc/apidoc/index.html +909 -0
  13. ciocore/docsite/apidoc/config/index.html +1652 -0
  14. ciocore/docsite/apidoc/data/index.html +1553 -0
  15. ciocore/docsite/apidoc/hardware_set/index.html +2460 -0
  16. ciocore/docsite/apidoc/package_environment/index.html +1507 -0
  17. ciocore/docsite/apidoc/package_tree/index.html +2386 -0
  18. ciocore/docsite/assets/_mkdocstrings.css +16 -0
  19. ciocore/docsite/assets/images/favicon.png +0 -0
  20. ciocore/docsite/assets/javascripts/bundle.471ce7a9.min.js +29 -0
  21. ciocore/docsite/assets/javascripts/bundle.471ce7a9.min.js.map +7 -0
  22. ciocore/docsite/assets/javascripts/lunr/min/lunr.ar.min.js +1 -0
  23. ciocore/docsite/assets/javascripts/lunr/min/lunr.da.min.js +18 -0
  24. ciocore/docsite/assets/javascripts/lunr/min/lunr.de.min.js +18 -0
  25. ciocore/docsite/assets/javascripts/lunr/min/lunr.du.min.js +18 -0
  26. ciocore/docsite/assets/javascripts/lunr/min/lunr.el.min.js +1 -0
  27. ciocore/docsite/assets/javascripts/lunr/min/lunr.es.min.js +18 -0
  28. ciocore/docsite/assets/javascripts/lunr/min/lunr.fi.min.js +18 -0
  29. ciocore/docsite/assets/javascripts/lunr/min/lunr.fr.min.js +18 -0
  30. ciocore/docsite/assets/javascripts/lunr/min/lunr.he.min.js +1 -0
  31. ciocore/docsite/assets/javascripts/lunr/min/lunr.hi.min.js +1 -0
  32. ciocore/docsite/assets/javascripts/lunr/min/lunr.hu.min.js +18 -0
  33. ciocore/docsite/assets/javascripts/lunr/min/lunr.hy.min.js +1 -0
  34. ciocore/docsite/assets/javascripts/lunr/min/lunr.it.min.js +18 -0
  35. ciocore/docsite/assets/javascripts/lunr/min/lunr.ja.min.js +1 -0
  36. ciocore/docsite/assets/javascripts/lunr/min/lunr.jp.min.js +1 -0
  37. ciocore/docsite/assets/javascripts/lunr/min/lunr.kn.min.js +1 -0
  38. ciocore/docsite/assets/javascripts/lunr/min/lunr.ko.min.js +1 -0
  39. ciocore/docsite/assets/javascripts/lunr/min/lunr.multi.min.js +1 -0
  40. ciocore/docsite/assets/javascripts/lunr/min/lunr.nl.min.js +18 -0
  41. ciocore/docsite/assets/javascripts/lunr/min/lunr.no.min.js +18 -0
  42. ciocore/docsite/assets/javascripts/lunr/min/lunr.pt.min.js +18 -0
  43. ciocore/docsite/assets/javascripts/lunr/min/lunr.ro.min.js +18 -0
  44. ciocore/docsite/assets/javascripts/lunr/min/lunr.ru.min.js +18 -0
  45. ciocore/docsite/assets/javascripts/lunr/min/lunr.sa.min.js +1 -0
  46. ciocore/docsite/assets/javascripts/lunr/min/lunr.stemmer.support.min.js +1 -0
  47. ciocore/docsite/assets/javascripts/lunr/min/lunr.sv.min.js +18 -0
  48. ciocore/docsite/assets/javascripts/lunr/min/lunr.ta.min.js +1 -0
  49. ciocore/docsite/assets/javascripts/lunr/min/lunr.te.min.js +1 -0
  50. ciocore/docsite/assets/javascripts/lunr/min/lunr.th.min.js +1 -0
  51. ciocore/docsite/assets/javascripts/lunr/min/lunr.tr.min.js +18 -0
  52. ciocore/docsite/assets/javascripts/lunr/min/lunr.vi.min.js +1 -0
  53. ciocore/docsite/assets/javascripts/lunr/min/lunr.zh.min.js +1 -0
  54. ciocore/docsite/assets/javascripts/lunr/tinyseg.js +206 -0
  55. ciocore/docsite/assets/javascripts/lunr/wordcut.js +6708 -0
  56. ciocore/docsite/assets/javascripts/workers/search.b8dbb3d2.min.js +42 -0
  57. ciocore/docsite/assets/javascripts/workers/search.b8dbb3d2.min.js.map +7 -0
  58. ciocore/docsite/assets/stylesheets/main.3cba04c6.min.css +1 -0
  59. ciocore/docsite/assets/stylesheets/main.3cba04c6.min.css.map +1 -0
  60. ciocore/docsite/assets/stylesheets/palette.06af60db.min.css +1 -0
  61. ciocore/docsite/assets/stylesheets/palette.06af60db.min.css.map +1 -0
  62. ciocore/docsite/cmdline/docs/index.html +871 -0
  63. ciocore/docsite/cmdline/downloader/index.html +934 -0
  64. ciocore/docsite/cmdline/packages/index.html +878 -0
  65. ciocore/docsite/cmdline/uploader/index.html +995 -0
  66. ciocore/docsite/how-to-guides/index.html +869 -0
  67. ciocore/docsite/index.html +895 -0
  68. ciocore/docsite/logo.png +0 -0
  69. ciocore/docsite/objects.inv +0 -0
  70. ciocore/docsite/search/search_index.json +1 -0
  71. ciocore/docsite/sitemap.xml +3 -0
  72. ciocore/docsite/sitemap.xml.gz +0 -0
  73. ciocore/docsite/stylesheets/extra.css +26 -0
  74. ciocore/docsite/stylesheets/tables.css +167 -0
  75. ciocore/downloader/base_downloader.py +644 -0
  76. ciocore/downloader/download_runner_base.py +47 -0
  77. ciocore/downloader/job_downloader.py +119 -0
  78. ciocore/{downloader.py → downloader/legacy_downloader.py} +12 -9
  79. ciocore/downloader/log.py +73 -0
  80. ciocore/downloader/logging_download_runner.py +87 -0
  81. ciocore/downloader/perpetual_downloader.py +63 -0
  82. ciocore/downloader/registry.py +97 -0
  83. ciocore/downloader/reporter.py +135 -0
  84. ciocore/exceptions.py +8 -2
  85. ciocore/file_utils.py +51 -50
  86. ciocore/hardware_set.py +449 -0
  87. ciocore/loggeria.py +89 -20
  88. ciocore/package_environment.py +110 -48
  89. ciocore/package_query.py +182 -0
  90. ciocore/package_tree.py +319 -258
  91. ciocore/retry.py +0 -0
  92. ciocore/uploader/_uploader.py +547 -364
  93. ciocore/uploader/thread_queue_job.py +176 -0
  94. ciocore/uploader/upload_stats/__init__.py +3 -4
  95. ciocore/uploader/upload_stats/stats_formats.py +10 -4
  96. ciocore/validator.py +34 -2
  97. ciocore/worker.py +174 -151
  98. ciocore-10.0.0b3.dist-info/METADATA +928 -0
  99. ciocore-10.0.0b3.dist-info/RECORD +128 -0
  100. {ciocore-5.1.1.dist-info → ciocore-10.0.0b3.dist-info}/WHEEL +1 -1
  101. ciocore-10.0.0b3.dist-info/entry_points.txt +2 -0
  102. tests/instance_type_fixtures.py +175 -0
  103. tests/package_fixtures.py +205 -0
  104. tests/test_api_client.py +297 -12
  105. tests/test_base_downloader.py +104 -0
  106. tests/test_cli.py +149 -0
  107. tests/test_common.py +1 -7
  108. tests/test_config.py +40 -18
  109. tests/test_data.py +162 -173
  110. tests/test_downloader.py +118 -0
  111. tests/test_hardware_set.py +139 -0
  112. tests/test_job_downloader.py +213 -0
  113. tests/test_package_query.py +38 -0
  114. tests/test_package_tree.py +91 -291
  115. tests/test_submit.py +44 -18
  116. tests/test_uploader.py +1 -4
  117. ciocore/__about__.py +0 -10
  118. ciocore/cli/conductor.py +0 -191
  119. ciocore/compat.py +0 -15
  120. ciocore-5.1.1.data/scripts/conductor +0 -19
  121. ciocore-5.1.1.data/scripts/conductor.bat +0 -13
  122. ciocore-5.1.1.dist-info/METADATA +0 -408
  123. ciocore-5.1.1.dist-info/RECORD +0 -47
  124. tests/mocks/api_client_mock.py +0 -51
  125. /ciocore/{cli → downloader}/__init__.py +0 -0
  126. {ciocore-5.1.1.dist-info → ciocore-10.0.0b3.dist-info}/top_level.txt +0 -0
tests/test_api_client.py CHANGED
@@ -2,20 +2,18 @@
2
2
 
3
3
  isort:skip_file
4
4
  """
5
-
6
- import unittest
7
- import os
5
+ import datetime
6
+ import json
8
7
  import sys
8
+ import unittest
9
9
 
10
- try:
11
- from unittest import mock
12
- except ImportError:
13
- import mock
14
-
15
- from mock import MagicMock
10
+ from unittest import mock
16
11
 
17
12
  from ciocore import api_client
18
13
 
14
+ from ciocore.api_client import request_extra_environment
15
+ from ciocore.api_client import _get_compute_usage, get_compute_usage
16
+ from ciocore.api_client import _get_storage_usage, get_storage_usage
19
17
 
20
18
  class ApiClientTest(unittest.TestCase):
21
19
  @staticmethod
@@ -26,7 +24,7 @@ class ApiClientTest(unittest.TestCase):
26
24
  return True
27
25
 
28
26
  def setUp(self):
29
- self.env = {"HOME": "/users/joebloggs"}
27
+ self.env = {"USERPROFILE": "/users/joebloggs", "HOME": "/users/joebloggs"}
30
28
 
31
29
  self.api_key_dict = {"api_key": {"client_id": "123", "private_key": "secret123"}}
32
30
 
@@ -41,11 +39,298 @@ class ApiClientTest(unittest.TestCase):
41
39
 
42
40
  def test_get_standard_creds_path(self):
43
41
  with mock.patch.dict("os.environ", self.env):
44
- fn = api_client.get_creds_path(api_key=False)
42
+ fn = api_client.get_creds_path(api_key=False).replace("\\", "/")
45
43
  self.assertEqual(fn, "/users/joebloggs/.config/conductor/credentials")
46
44
 
47
45
  def test_get_api_key_creds_path(self):
48
46
  with mock.patch.dict("os.environ", self.env):
49
- fn = api_client.get_creds_path(api_key=True)
47
+ fn = api_client.get_creds_path(api_key=True).replace("\\", "/")
50
48
  self.assertEqual(fn, "/users/joebloggs/.config/conductor/api_key_credentials")
51
49
 
50
+
51
+ class TestTruncateMiddle(unittest.TestCase):
52
+
53
+ def test_truncation_not_needed(self):
54
+ self.assertEqual(api_client.truncate_middle("short", 10), "short")
55
+
56
+ def test_truncation_with_even_max_length(self):
57
+ self.assertEqual(api_client.truncate_middle("1234567890ABCDEF", 8), "1234~DEF")
58
+
59
+ def test_truncation_with_odd_max_length(self):
60
+ self.assertEqual(api_client.truncate_middle("1234567890ABCDEF", 9), "1234~CDEF")
61
+
62
+ def test_empty_string(self):
63
+ self.assertEqual(api_client.truncate_middle("", 5), "")
64
+
65
+ def test_non_string_input(self):
66
+ with self.assertRaises(TypeError):
67
+ api_client.truncate_middle(12345, 5)
68
+
69
+ class TestRegisterClient(unittest.TestCase):
70
+ USER_AGENT_MAX_PATH_LENGTH = 10 # Example max length for testing
71
+
72
+ @classmethod
73
+ def setUpClass(cls):
74
+ cls.original_executable = sys.executable
75
+ sys.executable = '/usr/bin/python3' # Example path for testing
76
+
77
+ @classmethod
78
+ def tearDownClass(cls):
79
+ sys.executable = cls.original_executable
80
+
81
+ def test_register_client_with_version(self):
82
+ client_name = 'ApiClient'
83
+ client_version = '1.0'
84
+
85
+ with mock.patch('platform.python_version', return_value='3.8.2'), \
86
+ mock.patch('platform.system', return_value='Linux'), \
87
+ mock.patch('platform.release', return_value='5.4.0-42-generic'):
88
+ user_agent = api_client.ApiClient.register_client(client_name, client_version)
89
+
90
+ expected_user_agent = (
91
+ f"ApiClient/1.0 (python 3.8.2; Linux 5.4.0-42-generic; "
92
+ )
93
+ self.assertTrue(user_agent.startswith(expected_user_agent))
94
+
95
+ def test_register_client_without_version(self):
96
+ client_name = 'ApiClient'
97
+
98
+ with mock.patch('platform.python_version', return_value='3.8.2'), \
99
+ mock.patch('platform.system', return_value='Linux'), \
100
+ mock.patch('platform.release', return_value='5.4.0-42-generic'):
101
+ user_agent = api_client.ApiClient.register_client(client_name)
102
+
103
+
104
+ expected_user_agent = (
105
+ f"ApiClient/unknown (python 3.8.2; Linux 5.4.0-42-generic; "
106
+ )
107
+
108
+ self.assertTrue(user_agent.startswith(expected_user_agent))
109
+
110
+
111
+ class TestRequestExtraEnvironment(unittest.TestCase):
112
+
113
+ def setUp(self):
114
+ self.api_response_data = {
115
+ "data": [
116
+ {"account_id": "123", "env": ["VAR1=value1", "VAR2=value2"]},
117
+ {"account_id": "456", "env": ["VAR3=value3", "VAR4=value4"]}
118
+ ]
119
+ }
120
+ self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.api_response_data))
121
+ self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
122
+
123
+ @mock.patch("ciocore.api_client.ApiClient")
124
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
125
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
126
+ def test_request_extra_environment_success(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
127
+ # Set up mocks for successful execution
128
+ mock_read_conductor_credentials.return_value = "valid_token"
129
+ mock_account_id_from_jwt.return_value = "123"
130
+ mock_api_instance = mock_ApiClient.return_value
131
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
132
+
133
+ result = request_extra_environment()
134
+
135
+ self.assertEqual(result, ["VAR1=value1", "VAR2=value2"])
136
+ mock_ApiClient.assert_called_once()
137
+ mock_read_conductor_credentials.assert_called_once_with(True)
138
+ mock_account_id_from_jwt.assert_called_once_with("valid_token")
139
+
140
+ @mock.patch("ciocore.api_client.ApiClient")
141
+ def test_request_extra_environment_api_failure(self, mock_ApiClient):
142
+ # Set up mock for API failure
143
+ mock_api_instance = mock_ApiClient.return_value
144
+ mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
145
+
146
+ # Assert exception raised when the API call fails
147
+ with self.assertRaises(Exception) as context:
148
+ request_extra_environment()
149
+
150
+ self.assertIn('Failed to get extra environment', str(context.exception))
151
+ mock_ApiClient.assert_called_once()
152
+
153
+ @mock.patch("ciocore.api_client.ApiClient")
154
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
155
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
156
+ def test_request_extra_environment_no_account_env(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
157
+ # Set up mocks to simulate valid token and account ID but no matching environment
158
+ mock_read_conductor_credentials.return_value = "valid_token"
159
+ mock_account_id_from_jwt.return_value = "invalid_id" # This won't match any 'account_id' in response
160
+ mock_api_instance = mock_ApiClient.return_value
161
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
162
+
163
+ with self.assertRaises(Exception) as context:
164
+ request_extra_environment()
165
+
166
+ self.assertEqual("Error: Could not get account environment!", str(context.exception))
167
+ mock_ApiClient.assert_called_once()
168
+
169
+ class TestGetComputeUsage(unittest.TestCase):
170
+
171
+ def setUp(self):
172
+
173
+ # Precison is rounded to 4 places to avoid FP issues.
174
+ self.compute_response_data = {
175
+ "data": [
176
+ {
177
+ "cores": 0.5,
178
+ "instance_cost": 0.0200,
179
+ "license_cost": 0.0200,
180
+ "minutes": 6.9700,
181
+ "self_link": 0,
182
+ "start_time": "Tue, 09 Jan 2024 18:00:00 GMT"
183
+ },
184
+ {
185
+ "cores": 0.4,
186
+ "instance_cost": 0.0200,
187
+ "license_cost": 0.0200,
188
+ "minutes": 6.9600,
189
+ "self_link": 1,
190
+ "start_time": "Tue, 09 Jan 2024 19:00:00 GMT"
191
+ },
192
+ {
193
+ "cores": 0.9613,
194
+ "instance_cost": 0.0400,
195
+ "license_cost": 0.0800,
196
+ "minutes": 7.2100,
197
+ "self_link": 2,
198
+ "start_time": "Tue, 16 Jan 2024 17:00:00 GMT"
199
+ },
200
+ ]
201
+ }
202
+
203
+ self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.compute_response_data))
204
+ self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
205
+
206
+ @mock.patch("ciocore.api_client.ApiClient")
207
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
208
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
209
+ def test_query_raw_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
210
+ # Set up mocks for successful execution
211
+ mock_read_conductor_credentials.return_value = "valid_token"
212
+ mock_account_id_from_jwt.return_value = "123"
213
+ mock_api_instance = mock_ApiClient.return_value
214
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
215
+
216
+ start_time = datetime.datetime(2024, 1, 1)
217
+ end_time = datetime.datetime(2024, 3, 20)
218
+
219
+ result = _get_compute_usage(start_time, end_time)
220
+
221
+ self.assertEqual(result, self.compute_response_data['data'])
222
+
223
+ @mock.patch("ciocore.api_client.ApiClient")
224
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
225
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
226
+ def test_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
227
+ # Set up mocks for successful execution
228
+ mock_read_conductor_credentials.return_value = "valid_token"
229
+ mock_account_id_from_jwt.return_value = "123"
230
+ mock_api_instance = mock_ApiClient.return_value
231
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
232
+
233
+ start_time = datetime.datetime(2024, 1, 1)
234
+ end_time = datetime.datetime(2024, 3, 20)
235
+
236
+ result = get_compute_usage(start_time=start_time, end_time=end_time)
237
+
238
+ self.assertEqual(result, { '2024-01-09': {'cost': 0.08,
239
+ 'corehours': 0.9,
240
+ 'walltime': 13.93},
241
+ '2024-01-16': { 'cost': 0.12,
242
+ 'corehours': 0.9613,
243
+ 'walltime': 7.21}})
244
+
245
+ @mock.patch("ciocore.api_client.ApiClient")
246
+ def test_get_compute_usage_api_failure(self, mock_ApiClient):
247
+ # Set up mock for API failure
248
+ mock_api_instance = mock_ApiClient.return_value
249
+ mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
250
+
251
+ start_time = datetime.datetime(2024, 1, 1)
252
+ end_time = datetime.datetime(2024, 3, 20)
253
+
254
+ # Assert exception raised when the API call fails
255
+ with self.assertRaises(Exception) as context:
256
+ get_compute_usage(start_time, end_time)
257
+
258
+ self.assertIn('Failed to query compute usage', str(context.exception))
259
+ mock_ApiClient.assert_called_once()
260
+ class TestGetStorageUsage(unittest.TestCase):
261
+
262
+ def setUp(self):
263
+
264
+ self.storage_response_data = {
265
+ "data": [
266
+ {
267
+ "cost": "28.96",
268
+ "cost_per_day": [
269
+ 4.022,
270
+ 4.502,
271
+ 4.502,
272
+ 5.102,
273
+ 5.102,
274
+ 5.732
275
+ ],
276
+ "currency": "USD",
277
+ "daily_price": "0.006",
278
+ "end_date": "2024-01-07",
279
+ "gibs_per_day": [
280
+ 679.714,
281
+ 750.34,
282
+ 750.34,
283
+ 850.36,
284
+ 850.35,
285
+ 955.32
286
+ ],
287
+ "gibs_used": "806.07",
288
+ "monthly_price": "0.18",
289
+ "start_date": "2024-01-01",
290
+ "storage_unit": "GiB"
291
+ }
292
+ ]
293
+ }
294
+
295
+ self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.storage_response_data))
296
+ self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
297
+
298
+ @mock.patch("ciocore.api_client.ApiClient")
299
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
300
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
301
+ def test_query_raw_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
302
+ # Set up mocks for successful execution
303
+ mock_read_conductor_credentials.return_value = "valid_token"
304
+ mock_account_id_from_jwt.return_value = "123"
305
+ mock_api_instance = mock_ApiClient.return_value
306
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
307
+
308
+ start_time = datetime.datetime(2024, 5, 1)
309
+ end_time = datetime.datetime(2024, 5, 17)
310
+
311
+ result = _get_storage_usage(start_time, end_time)
312
+
313
+ self.assertEqual(result, self.storage_response_data['data'][0])
314
+
315
+ @mock.patch("ciocore.api_client.ApiClient")
316
+ @mock.patch("ciocore.api_client.read_conductor_credentials")
317
+ @mock.patch("ciocore.api_client.account_id_from_jwt")
318
+ def test_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
319
+ # Set up mocks for successful execution
320
+ mock_read_conductor_credentials.return_value = "valid_token"
321
+ mock_account_id_from_jwt.return_value = "123"
322
+ mock_api_instance = mock_ApiClient.return_value
323
+ mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
324
+
325
+ start_time = datetime.datetime(2024, 1, 1)
326
+ end_time = datetime.datetime(2024, 1, 7)
327
+
328
+ result = get_storage_usage(start_time=start_time, end_time=end_time)
329
+
330
+ self.assertEqual(result, { '2024-01-01': {'cost': 4.022, 'GiB': 679.714},
331
+ '2024-01-02': {'cost': 4.502, 'GiB': 750.34},
332
+ '2024-01-03': {'cost': 4.502, 'GiB': 750.34},
333
+ '2024-01-04': {'cost': 5.102, 'GiB': 850.36},
334
+ '2024-01-05': {'cost': 5.102, 'GiB': 850.35},
335
+ '2024-01-06': {'cost': 5.732, 'GiB': 955.32}
336
+ })
@@ -0,0 +1,104 @@
1
+ from ciocore.downloader.base_downloader import (
2
+ BaseDownloader,
3
+ DEFAULT_NUM_THREADS,
4
+ DEFAULT_PROGRESS_INTERVAL,
5
+ DEFAULT_MAX_ATTEMPTS,
6
+ DEFAULT_DELAY,
7
+ DEFAULT_JITTER,
8
+ DEFAULT_PAGE_SIZE,
9
+ )
10
+ import unittest
11
+ from unittest import mock
12
+
13
+ from ciocore import api_client
14
+
15
+ from unittest.mock import patch
16
+
17
+ from concurrent.futures import ThreadPoolExecutor
18
+
19
+
20
+ class TestBaseDownloaderInit(unittest.TestCase):
21
+ def test_default_values(self):
22
+ # Create an instance of the class
23
+ downloader = BaseDownloader()
24
+
25
+ # Assertions
26
+ self.assertIsNone(downloader.output_path)
27
+ self.assertFalse(downloader.force)
28
+ self.assertEqual(downloader.num_threads, DEFAULT_NUM_THREADS)
29
+ self.assertEqual(downloader.max_queue_size, DEFAULT_NUM_THREADS * 2)
30
+ self.assertEqual(
31
+ downloader.progress_interval, DEFAULT_PROGRESS_INTERVAL / 1000.0
32
+ )
33
+ self.assertEqual(downloader.page_size, DEFAULT_PAGE_SIZE)
34
+ self.assertIsInstance(downloader.client, api_client.ApiClient)
35
+ self.assertEqual(downloader.max_attempts, DEFAULT_MAX_ATTEMPTS)
36
+ self.assertEqual(downloader.delay, DEFAULT_DELAY)
37
+ self.assertEqual(downloader.jitter, DEFAULT_JITTER)
38
+ self.assertIsNone(downloader.regex)
39
+
40
+
41
+ def test_custom_values(self):
42
+ output_path = "/path/to/destination"
43
+ num_threads = 4
44
+ progress_interval = 500
45
+ page_size = 10
46
+ force = True
47
+ regex = r"\d+"
48
+ max_attempts = 3
49
+ delay = 2
50
+ jitter = 0.5
51
+
52
+ downloader = BaseDownloader(
53
+ output_path=output_path,
54
+ num_threads=num_threads,
55
+ progress_interval=progress_interval,
56
+ page_size=page_size,
57
+ force=force,
58
+ regex=regex,
59
+ max_attempts=max_attempts,
60
+ delay=delay,
61
+ jitter=jitter,
62
+ )
63
+
64
+ # Assertions
65
+ self.assertEqual(downloader.output_path, output_path)
66
+ self.assertTrue(downloader.force)
67
+ self.assertEqual(downloader.num_threads, num_threads)
68
+ self.assertEqual(downloader.max_queue_size, num_threads * 2)
69
+ self.assertAlmostEqual(downloader.progress_interval, progress_interval / 1000.0)
70
+ self.assertEqual(downloader.page_size, page_size)
71
+ self.assertIsInstance(downloader.client, api_client.ApiClient)
72
+ self.assertEqual(downloader.max_attempts, max_attempts)
73
+ self.assertEqual(downloader.delay, delay)
74
+ self.assertEqual(downloader.jitter, jitter)
75
+ self.assertIsNotNone(downloader.regex)
76
+
77
+
78
+ class TestBaseDownloaderRun(unittest.TestCase):
79
+ def setUp(self):
80
+ self.downloader = BaseDownloader()
81
+
82
+ def tearDown(self):
83
+ pass
84
+
85
+ def test_run_method(self):
86
+ with patch(
87
+ "ciocore.downloader.base_downloader.ThreadPoolExecutor"
88
+ ) as mock_executor:
89
+ my_mock_executor = mock.MagicMock(spec=ThreadPoolExecutor)
90
+
91
+ mock_executor.return_value.__enter__.return_value = my_mock_executor
92
+
93
+ tasks = [{"id": 1, "name": "task1"}, {"id": 2, "name": "task2"}]
94
+ next_locator = False
95
+ mock_get_some_tasks = mock.MagicMock(return_value=(tasks, next_locator))
96
+
97
+ self.downloader.get_some_tasks = mock_get_some_tasks
98
+ self.downloader.download_tasks = mock.MagicMock()
99
+ self.downloader.event_queue = mock.MagicMock()
100
+
101
+ self.downloader.run()
102
+
103
+ mock_get_some_tasks.assert_called_with(None)
104
+ self.downloader.download_tasks.assert_called_with(tasks, my_mock_executor)
tests/test_cli.py ADDED
@@ -0,0 +1,149 @@
1
+ from click.testing import CliRunner
2
+ from ciocore.cli import upload as cli_upload
3
+ from ciocore.cli import download as cli_download
4
+ import unittest
5
+ import os
6
+ from unittest import mock
7
+
8
+
9
+ class CliTestUploaderOptions(unittest.TestCase):
10
+ def setUp(self):
11
+ self.runner = CliRunner()
12
+ self.default_args = {
13
+ "database_filepath": None,
14
+ "location": None,
15
+ "md5_caching": True,
16
+ "thread_count": mock.ANY,
17
+ }
18
+ init_patcher = mock.patch("ciocore.cli.Uploader.__init__", autospec=True)
19
+ self.mock_init = init_patcher.start()
20
+ self.addCleanup(init_patcher.stop)
21
+
22
+ def test_receives_full_args_dict_with_defaults_when_no_args_given(self):
23
+ self.runner.invoke(cli_upload, [])
24
+ self.mock_init.assert_called_once_with(mock.ANY, self.default_args)
25
+
26
+ def test_database_filepath_arg(self):
27
+ self.runner.invoke(cli_upload, ["--database_filepath", "foo"])
28
+ expected = self.default_args
29
+ expected.update({"database_filepath": "foo"})
30
+ self.mock_init.assert_called_once_with(mock.ANY, expected)
31
+
32
+ def test_location_arg(self):
33
+ self.runner.invoke(cli_upload, ["--location", "foo"])
34
+ expected = self.default_args
35
+ expected.update({"location": "foo"})
36
+ self.mock_init.assert_called_once_with(mock.ANY, expected)
37
+
38
+ def test_md5_caching_arg(self):
39
+ self.runner.invoke(cli_upload, ["--md5_caching", False])
40
+ expected = self.default_args
41
+ expected.update({"md5_caching": False})
42
+ self.mock_init.assert_called_once_with(mock.ANY, expected)
43
+
44
+ def test_thread_count_arg(self):
45
+ self.runner.invoke(cli_upload, ["--thread_count", 4])
46
+ expected = self.default_args
47
+ expected.update({"thread_count": 4})
48
+ self.mock_init.assert_called_once_with(mock.ANY, expected)
49
+
50
+
51
+ class CliTestUploaderArguments(unittest.TestCase):
52
+ def setUp(self):
53
+ self.runner = CliRunner()
54
+
55
+ uploader_patcher = mock.patch("ciocore.cli.Uploader", autospec=True)
56
+ self.mock_uploader = uploader_patcher.start()
57
+ self.mock_inst = self.mock_uploader.return_value
58
+ self.addCleanup(uploader_patcher.stop)
59
+
60
+ def test_path_only_branch_if_paths(self):
61
+ with self.runner.isolated_filesystem():
62
+ filenames = ["foo.txt", "bar.txt", "baz.txt", "qux.txt"]
63
+ filenames = [os.path.join(os.getcwd(), filename) for filename in filenames]
64
+ for filename in filenames:
65
+ with open(filename, "w") as f:
66
+ f.write("hello world")
67
+ self.runner.invoke(cli_upload, filenames)
68
+ self.mock_inst.assets_only.assert_called_once_with(
69
+ mock.ANY, mock.ANY, mock.ANY, mock.ANY
70
+ )
71
+ self.mock_inst.main.assert_not_called()
72
+
73
+ def test_main_branch_if_no_paths(self):
74
+ with self.runner.isolated_filesystem():
75
+ self.runner.invoke(cli_upload)
76
+ self.mock_inst.main.assert_called_once()
77
+ self.mock_inst.assets_only.assert_not_called()
78
+
79
+
80
+ class CliTestDownloader(unittest.TestCase):
81
+ def setUp(self):
82
+ self.runner = CliRunner()
83
+
84
+ dl_patcher_runner = mock.patch(
85
+ "ciocore.cli.LoggingDownloadRunner.__init__", autospec=True
86
+ )
87
+ self.mock_runner = dl_patcher_runner.start()
88
+ self.addCleanup(dl_patcher_runner.stop)
89
+
90
+ start_daemon_patcher = mock.patch(
91
+ "ciocore.cli.Downloader.start_daemon", autospec=True
92
+ )
93
+ self.mock_start_daemon = start_daemon_patcher.start()
94
+ self.addCleanup(start_daemon_patcher.stop)
95
+
96
+
97
+ def test_jobid_branch_if_job_id(self):
98
+ jid = "00000"
99
+ # self.assertTrue(True)
100
+ self.runner.invoke(cli_download, [jid])
101
+ self.mock_runner.assert_called_once( )
102
+ self.mock_start_daemon.assert_not_called()
103
+
104
+ # def test_job_id_only(self):
105
+ # jid = "00000"
106
+ # self.runner.invoke(cli_download, [jid])
107
+ # self.mock_dljobs.assert_called_once_with(
108
+ # (jid,), thread_count=mock.ANY, output_dir=mock.ANY
109
+ # )
110
+
111
+ # def test_several_job_ids(self):
112
+ # jid1 = "00000"
113
+ # jid2 = "11111"
114
+ # self.runner.invoke(cli_download, [jid1, jid2])
115
+ # self.mock_dljobs.assert_called_once_with(
116
+ # (jid1,jid2), thread_count=mock.ANY, output_dir=mock.ANY
117
+ # )
118
+
119
+ # def test_job_ids_and_others(self):
120
+ # jid1 = "00000"
121
+ # jid2 = "11111"
122
+ # tc = 4
123
+ # od = "foo"
124
+ # self.runner.invoke(
125
+ # cli_download,
126
+ # ["--thread_count", tc, "--destination", od, jid1, jid2 ],
127
+ # )
128
+ # self.mock_dljobs.assert_called_once_with(
129
+ # (jid1,jid2), thread_count=tc, output_dir=od
130
+ # )
131
+
132
+ # def test_daemon_branch_if_no_job_id(self):
133
+ # self.runner.invoke(cli_download, [])
134
+ # self.mock_start_daemon.assert_called_once()
135
+ # self.mock_dljobs.assert_not_called()
136
+
137
+ # def test_daemon_branch_args_present(self):
138
+ # tc = 4
139
+ # od = "foo"
140
+ # self.runner.invoke(cli_download)
141
+ # self.mock_start_daemon.assert_called_once_with(thread_count=mock.ANY, location=mock.ANY, output_dir=mock.ANY)
142
+
143
+ # def test_daemon_branch_args(self):
144
+ # tc = 4
145
+ # od = "/foo"
146
+ # loc = "bar"
147
+ # self.runner.invoke(cli_download, ["--thread_count", tc, "--location", loc])
148
+ # self.mock_start_daemon.assert_called_once_with(thread_count=tc, location="bar", output_dir=None)
149
+
tests/test_common.py CHANGED
@@ -9,13 +9,7 @@ FILES_PATH = os.path.join(os.path.dirname(__file__), "files")
9
9
 
10
10
 
11
11
  class TestMd5(unittest.TestCase):
12
-
13
- def test_get_base64_md5_is_correct_md5(self):
14
- from ciocore import common
15
- fn1 = os.path.join(FILES_PATH, "one")
16
- md5=common.get_base64_md5(fn1)
17
- self.assertEqual(md5, "9iVbsBxkj+lncU1SqJ6OnA==")
18
-
12
+
19
13
  def test_get_base64_md5_is_correct_type(self):
20
14
  from ciocore import common
21
15
  from builtins import str