ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ciocore/VERSION +1 -1
- ciocore/__init__.py +23 -1
- ciocore/api_client.py +655 -160
- ciocore/auth/__init__.py +5 -3
- ciocore/cli.py +501 -0
- ciocore/common.py +15 -13
- ciocore/conductor_submit.py +77 -60
- ciocore/config.py +127 -13
- ciocore/data.py +162 -77
- ciocore/docsite/404.html +746 -0
- ciocore/docsite/apidoc/api_client/index.html +3605 -0
- ciocore/docsite/apidoc/apidoc/index.html +909 -0
- ciocore/docsite/apidoc/config/index.html +1652 -0
- ciocore/docsite/apidoc/data/index.html +1553 -0
- ciocore/docsite/apidoc/hardware_set/index.html +2460 -0
- ciocore/docsite/apidoc/package_environment/index.html +1507 -0
- ciocore/docsite/apidoc/package_tree/index.html +2386 -0
- ciocore/docsite/assets/_mkdocstrings.css +16 -0
- ciocore/docsite/assets/images/favicon.png +0 -0
- ciocore/docsite/assets/javascripts/bundle.471ce7a9.min.js +29 -0
- ciocore/docsite/assets/javascripts/bundle.471ce7a9.min.js.map +7 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ar.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.da.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.de.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.du.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.el.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.es.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.fi.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.fr.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.he.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hu.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hy.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.it.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ja.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.jp.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.kn.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ko.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.multi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.nl.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.no.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.pt.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ro.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ru.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.sa.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.stemmer.support.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.sv.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ta.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.te.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.th.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.tr.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.vi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.zh.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/tinyseg.js +206 -0
- ciocore/docsite/assets/javascripts/lunr/wordcut.js +6708 -0
- ciocore/docsite/assets/javascripts/workers/search.b8dbb3d2.min.js +42 -0
- ciocore/docsite/assets/javascripts/workers/search.b8dbb3d2.min.js.map +7 -0
- ciocore/docsite/assets/stylesheets/main.3cba04c6.min.css +1 -0
- ciocore/docsite/assets/stylesheets/main.3cba04c6.min.css.map +1 -0
- ciocore/docsite/assets/stylesheets/palette.06af60db.min.css +1 -0
- ciocore/docsite/assets/stylesheets/palette.06af60db.min.css.map +1 -0
- ciocore/docsite/cmdline/docs/index.html +871 -0
- ciocore/docsite/cmdline/downloader/index.html +934 -0
- ciocore/docsite/cmdline/packages/index.html +878 -0
- ciocore/docsite/cmdline/uploader/index.html +995 -0
- ciocore/docsite/how-to-guides/index.html +869 -0
- ciocore/docsite/index.html +895 -0
- ciocore/docsite/logo.png +0 -0
- ciocore/docsite/objects.inv +0 -0
- ciocore/docsite/search/search_index.json +1 -0
- ciocore/docsite/sitemap.xml +3 -0
- ciocore/docsite/sitemap.xml.gz +0 -0
- ciocore/docsite/stylesheets/extra.css +26 -0
- ciocore/docsite/stylesheets/tables.css +167 -0
- ciocore/downloader/base_downloader.py +644 -0
- ciocore/downloader/download_runner_base.py +47 -0
- ciocore/downloader/job_downloader.py +119 -0
- ciocore/{downloader.py → downloader/legacy_downloader.py} +12 -9
- ciocore/downloader/log.py +73 -0
- ciocore/downloader/logging_download_runner.py +87 -0
- ciocore/downloader/perpetual_downloader.py +63 -0
- ciocore/downloader/registry.py +97 -0
- ciocore/downloader/reporter.py +135 -0
- ciocore/exceptions.py +8 -2
- ciocore/file_utils.py +51 -50
- ciocore/hardware_set.py +449 -0
- ciocore/loggeria.py +89 -20
- ciocore/package_environment.py +110 -48
- ciocore/package_query.py +182 -0
- ciocore/package_tree.py +319 -258
- ciocore/retry.py +0 -0
- ciocore/uploader/_uploader.py +547 -364
- ciocore/uploader/thread_queue_job.py +176 -0
- ciocore/uploader/upload_stats/__init__.py +3 -4
- ciocore/uploader/upload_stats/stats_formats.py +10 -4
- ciocore/validator.py +34 -2
- ciocore/worker.py +174 -151
- ciocore-10.0.0b3.dist-info/METADATA +928 -0
- ciocore-10.0.0b3.dist-info/RECORD +128 -0
- {ciocore-5.1.1.dist-info → ciocore-10.0.0b3.dist-info}/WHEEL +1 -1
- ciocore-10.0.0b3.dist-info/entry_points.txt +2 -0
- tests/instance_type_fixtures.py +175 -0
- tests/package_fixtures.py +205 -0
- tests/test_api_client.py +297 -12
- tests/test_base_downloader.py +104 -0
- tests/test_cli.py +149 -0
- tests/test_common.py +1 -7
- tests/test_config.py +40 -18
- tests/test_data.py +162 -173
- tests/test_downloader.py +118 -0
- tests/test_hardware_set.py +139 -0
- tests/test_job_downloader.py +213 -0
- tests/test_package_query.py +38 -0
- tests/test_package_tree.py +91 -291
- tests/test_submit.py +44 -18
- tests/test_uploader.py +1 -4
- ciocore/__about__.py +0 -10
- ciocore/cli/conductor.py +0 -191
- ciocore/compat.py +0 -15
- ciocore-5.1.1.data/scripts/conductor +0 -19
- ciocore-5.1.1.data/scripts/conductor.bat +0 -13
- ciocore-5.1.1.dist-info/METADATA +0 -408
- ciocore-5.1.1.dist-info/RECORD +0 -47
- tests/mocks/api_client_mock.py +0 -51
- /ciocore/{cli → downloader}/__init__.py +0 -0
- {ciocore-5.1.1.dist-info → ciocore-10.0.0b3.dist-info}/top_level.txt +0 -0
tests/test_api_client.py
CHANGED
|
@@ -2,20 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
isort:skip_file
|
|
4
4
|
"""
|
|
5
|
-
|
|
6
|
-
import
|
|
7
|
-
import os
|
|
5
|
+
import datetime
|
|
6
|
+
import json
|
|
8
7
|
import sys
|
|
8
|
+
import unittest
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
from unittest import mock
|
|
12
|
-
except ImportError:
|
|
13
|
-
import mock
|
|
14
|
-
|
|
15
|
-
from mock import MagicMock
|
|
10
|
+
from unittest import mock
|
|
16
11
|
|
|
17
12
|
from ciocore import api_client
|
|
18
13
|
|
|
14
|
+
from ciocore.api_client import request_extra_environment
|
|
15
|
+
from ciocore.api_client import _get_compute_usage, get_compute_usage
|
|
16
|
+
from ciocore.api_client import _get_storage_usage, get_storage_usage
|
|
19
17
|
|
|
20
18
|
class ApiClientTest(unittest.TestCase):
|
|
21
19
|
@staticmethod
|
|
@@ -26,7 +24,7 @@ class ApiClientTest(unittest.TestCase):
|
|
|
26
24
|
return True
|
|
27
25
|
|
|
28
26
|
def setUp(self):
|
|
29
|
-
self.env = {"HOME": "/users/joebloggs"}
|
|
27
|
+
self.env = {"USERPROFILE": "/users/joebloggs", "HOME": "/users/joebloggs"}
|
|
30
28
|
|
|
31
29
|
self.api_key_dict = {"api_key": {"client_id": "123", "private_key": "secret123"}}
|
|
32
30
|
|
|
@@ -41,11 +39,298 @@ class ApiClientTest(unittest.TestCase):
|
|
|
41
39
|
|
|
42
40
|
def test_get_standard_creds_path(self):
|
|
43
41
|
with mock.patch.dict("os.environ", self.env):
|
|
44
|
-
fn = api_client.get_creds_path(api_key=False)
|
|
42
|
+
fn = api_client.get_creds_path(api_key=False).replace("\\", "/")
|
|
45
43
|
self.assertEqual(fn, "/users/joebloggs/.config/conductor/credentials")
|
|
46
44
|
|
|
47
45
|
def test_get_api_key_creds_path(self):
|
|
48
46
|
with mock.patch.dict("os.environ", self.env):
|
|
49
|
-
fn = api_client.get_creds_path(api_key=True)
|
|
47
|
+
fn = api_client.get_creds_path(api_key=True).replace("\\", "/")
|
|
50
48
|
self.assertEqual(fn, "/users/joebloggs/.config/conductor/api_key_credentials")
|
|
51
49
|
|
|
50
|
+
|
|
51
|
+
class TestTruncateMiddle(unittest.TestCase):
|
|
52
|
+
|
|
53
|
+
def test_truncation_not_needed(self):
|
|
54
|
+
self.assertEqual(api_client.truncate_middle("short", 10), "short")
|
|
55
|
+
|
|
56
|
+
def test_truncation_with_even_max_length(self):
|
|
57
|
+
self.assertEqual(api_client.truncate_middle("1234567890ABCDEF", 8), "1234~DEF")
|
|
58
|
+
|
|
59
|
+
def test_truncation_with_odd_max_length(self):
|
|
60
|
+
self.assertEqual(api_client.truncate_middle("1234567890ABCDEF", 9), "1234~CDEF")
|
|
61
|
+
|
|
62
|
+
def test_empty_string(self):
|
|
63
|
+
self.assertEqual(api_client.truncate_middle("", 5), "")
|
|
64
|
+
|
|
65
|
+
def test_non_string_input(self):
|
|
66
|
+
with self.assertRaises(TypeError):
|
|
67
|
+
api_client.truncate_middle(12345, 5)
|
|
68
|
+
|
|
69
|
+
class TestRegisterClient(unittest.TestCase):
|
|
70
|
+
USER_AGENT_MAX_PATH_LENGTH = 10 # Example max length for testing
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def setUpClass(cls):
|
|
74
|
+
cls.original_executable = sys.executable
|
|
75
|
+
sys.executable = '/usr/bin/python3' # Example path for testing
|
|
76
|
+
|
|
77
|
+
@classmethod
|
|
78
|
+
def tearDownClass(cls):
|
|
79
|
+
sys.executable = cls.original_executable
|
|
80
|
+
|
|
81
|
+
def test_register_client_with_version(self):
|
|
82
|
+
client_name = 'ApiClient'
|
|
83
|
+
client_version = '1.0'
|
|
84
|
+
|
|
85
|
+
with mock.patch('platform.python_version', return_value='3.8.2'), \
|
|
86
|
+
mock.patch('platform.system', return_value='Linux'), \
|
|
87
|
+
mock.patch('platform.release', return_value='5.4.0-42-generic'):
|
|
88
|
+
user_agent = api_client.ApiClient.register_client(client_name, client_version)
|
|
89
|
+
|
|
90
|
+
expected_user_agent = (
|
|
91
|
+
f"ApiClient/1.0 (python 3.8.2; Linux 5.4.0-42-generic; "
|
|
92
|
+
)
|
|
93
|
+
self.assertTrue(user_agent.startswith(expected_user_agent))
|
|
94
|
+
|
|
95
|
+
def test_register_client_without_version(self):
|
|
96
|
+
client_name = 'ApiClient'
|
|
97
|
+
|
|
98
|
+
with mock.patch('platform.python_version', return_value='3.8.2'), \
|
|
99
|
+
mock.patch('platform.system', return_value='Linux'), \
|
|
100
|
+
mock.patch('platform.release', return_value='5.4.0-42-generic'):
|
|
101
|
+
user_agent = api_client.ApiClient.register_client(client_name)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
expected_user_agent = (
|
|
105
|
+
f"ApiClient/unknown (python 3.8.2; Linux 5.4.0-42-generic; "
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
self.assertTrue(user_agent.startswith(expected_user_agent))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TestRequestExtraEnvironment(unittest.TestCase):
|
|
112
|
+
|
|
113
|
+
def setUp(self):
|
|
114
|
+
self.api_response_data = {
|
|
115
|
+
"data": [
|
|
116
|
+
{"account_id": "123", "env": ["VAR1=value1", "VAR2=value2"]},
|
|
117
|
+
{"account_id": "456", "env": ["VAR3=value3", "VAR4=value4"]}
|
|
118
|
+
]
|
|
119
|
+
}
|
|
120
|
+
self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.api_response_data))
|
|
121
|
+
self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
|
|
122
|
+
|
|
123
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
124
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
125
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
126
|
+
def test_request_extra_environment_success(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
127
|
+
# Set up mocks for successful execution
|
|
128
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
129
|
+
mock_account_id_from_jwt.return_value = "123"
|
|
130
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
131
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
132
|
+
|
|
133
|
+
result = request_extra_environment()
|
|
134
|
+
|
|
135
|
+
self.assertEqual(result, ["VAR1=value1", "VAR2=value2"])
|
|
136
|
+
mock_ApiClient.assert_called_once()
|
|
137
|
+
mock_read_conductor_credentials.assert_called_once_with(True)
|
|
138
|
+
mock_account_id_from_jwt.assert_called_once_with("valid_token")
|
|
139
|
+
|
|
140
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
141
|
+
def test_request_extra_environment_api_failure(self, mock_ApiClient):
|
|
142
|
+
# Set up mock for API failure
|
|
143
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
144
|
+
mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
|
|
145
|
+
|
|
146
|
+
# Assert exception raised when the API call fails
|
|
147
|
+
with self.assertRaises(Exception) as context:
|
|
148
|
+
request_extra_environment()
|
|
149
|
+
|
|
150
|
+
self.assertIn('Failed to get extra environment', str(context.exception))
|
|
151
|
+
mock_ApiClient.assert_called_once()
|
|
152
|
+
|
|
153
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
154
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
155
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
156
|
+
def test_request_extra_environment_no_account_env(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
157
|
+
# Set up mocks to simulate valid token and account ID but no matching environment
|
|
158
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
159
|
+
mock_account_id_from_jwt.return_value = "invalid_id" # This won't match any 'account_id' in response
|
|
160
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
161
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
162
|
+
|
|
163
|
+
with self.assertRaises(Exception) as context:
|
|
164
|
+
request_extra_environment()
|
|
165
|
+
|
|
166
|
+
self.assertEqual("Error: Could not get account environment!", str(context.exception))
|
|
167
|
+
mock_ApiClient.assert_called_once()
|
|
168
|
+
|
|
169
|
+
class TestGetComputeUsage(unittest.TestCase):
|
|
170
|
+
|
|
171
|
+
def setUp(self):
|
|
172
|
+
|
|
173
|
+
# Precison is rounded to 4 places to avoid FP issues.
|
|
174
|
+
self.compute_response_data = {
|
|
175
|
+
"data": [
|
|
176
|
+
{
|
|
177
|
+
"cores": 0.5,
|
|
178
|
+
"instance_cost": 0.0200,
|
|
179
|
+
"license_cost": 0.0200,
|
|
180
|
+
"minutes": 6.9700,
|
|
181
|
+
"self_link": 0,
|
|
182
|
+
"start_time": "Tue, 09 Jan 2024 18:00:00 GMT"
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"cores": 0.4,
|
|
186
|
+
"instance_cost": 0.0200,
|
|
187
|
+
"license_cost": 0.0200,
|
|
188
|
+
"minutes": 6.9600,
|
|
189
|
+
"self_link": 1,
|
|
190
|
+
"start_time": "Tue, 09 Jan 2024 19:00:00 GMT"
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
"cores": 0.9613,
|
|
194
|
+
"instance_cost": 0.0400,
|
|
195
|
+
"license_cost": 0.0800,
|
|
196
|
+
"minutes": 7.2100,
|
|
197
|
+
"self_link": 2,
|
|
198
|
+
"start_time": "Tue, 16 Jan 2024 17:00:00 GMT"
|
|
199
|
+
},
|
|
200
|
+
]
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.compute_response_data))
|
|
204
|
+
self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
|
|
205
|
+
|
|
206
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
207
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
208
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
209
|
+
def test_query_raw_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
210
|
+
# Set up mocks for successful execution
|
|
211
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
212
|
+
mock_account_id_from_jwt.return_value = "123"
|
|
213
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
214
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
215
|
+
|
|
216
|
+
start_time = datetime.datetime(2024, 1, 1)
|
|
217
|
+
end_time = datetime.datetime(2024, 3, 20)
|
|
218
|
+
|
|
219
|
+
result = _get_compute_usage(start_time, end_time)
|
|
220
|
+
|
|
221
|
+
self.assertEqual(result, self.compute_response_data['data'])
|
|
222
|
+
|
|
223
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
224
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
225
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
226
|
+
def test_compute_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
227
|
+
# Set up mocks for successful execution
|
|
228
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
229
|
+
mock_account_id_from_jwt.return_value = "123"
|
|
230
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
231
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
232
|
+
|
|
233
|
+
start_time = datetime.datetime(2024, 1, 1)
|
|
234
|
+
end_time = datetime.datetime(2024, 3, 20)
|
|
235
|
+
|
|
236
|
+
result = get_compute_usage(start_time=start_time, end_time=end_time)
|
|
237
|
+
|
|
238
|
+
self.assertEqual(result, { '2024-01-09': {'cost': 0.08,
|
|
239
|
+
'corehours': 0.9,
|
|
240
|
+
'walltime': 13.93},
|
|
241
|
+
'2024-01-16': { 'cost': 0.12,
|
|
242
|
+
'corehours': 0.9613,
|
|
243
|
+
'walltime': 7.21}})
|
|
244
|
+
|
|
245
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
246
|
+
def test_get_compute_usage_api_failure(self, mock_ApiClient):
|
|
247
|
+
# Set up mock for API failure
|
|
248
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
249
|
+
mock_api_instance.make_request.return_value = (self.response_error.text, self.response_error.status_code)
|
|
250
|
+
|
|
251
|
+
start_time = datetime.datetime(2024, 1, 1)
|
|
252
|
+
end_time = datetime.datetime(2024, 3, 20)
|
|
253
|
+
|
|
254
|
+
# Assert exception raised when the API call fails
|
|
255
|
+
with self.assertRaises(Exception) as context:
|
|
256
|
+
get_compute_usage(start_time, end_time)
|
|
257
|
+
|
|
258
|
+
self.assertIn('Failed to query compute usage', str(context.exception))
|
|
259
|
+
mock_ApiClient.assert_called_once()
|
|
260
|
+
class TestGetStorageUsage(unittest.TestCase):
|
|
261
|
+
|
|
262
|
+
def setUp(self):
|
|
263
|
+
|
|
264
|
+
self.storage_response_data = {
|
|
265
|
+
"data": [
|
|
266
|
+
{
|
|
267
|
+
"cost": "28.96",
|
|
268
|
+
"cost_per_day": [
|
|
269
|
+
4.022,
|
|
270
|
+
4.502,
|
|
271
|
+
4.502,
|
|
272
|
+
5.102,
|
|
273
|
+
5.102,
|
|
274
|
+
5.732
|
|
275
|
+
],
|
|
276
|
+
"currency": "USD",
|
|
277
|
+
"daily_price": "0.006",
|
|
278
|
+
"end_date": "2024-01-07",
|
|
279
|
+
"gibs_per_day": [
|
|
280
|
+
679.714,
|
|
281
|
+
750.34,
|
|
282
|
+
750.34,
|
|
283
|
+
850.36,
|
|
284
|
+
850.35,
|
|
285
|
+
955.32
|
|
286
|
+
],
|
|
287
|
+
"gibs_used": "806.07",
|
|
288
|
+
"monthly_price": "0.18",
|
|
289
|
+
"start_date": "2024-01-01",
|
|
290
|
+
"storage_unit": "GiB"
|
|
291
|
+
}
|
|
292
|
+
]
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
self.response_ok = mock.MagicMock(status_code=200, text=json.dumps(self.storage_response_data))
|
|
296
|
+
self.response_error = mock.MagicMock(status_code=500, text=json.dumps({"error": "Internal Server Error"}))
|
|
297
|
+
|
|
298
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
299
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
300
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
301
|
+
def test_query_raw_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
302
|
+
# Set up mocks for successful execution
|
|
303
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
304
|
+
mock_account_id_from_jwt.return_value = "123"
|
|
305
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
306
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
307
|
+
|
|
308
|
+
start_time = datetime.datetime(2024, 5, 1)
|
|
309
|
+
end_time = datetime.datetime(2024, 5, 17)
|
|
310
|
+
|
|
311
|
+
result = _get_storage_usage(start_time, end_time)
|
|
312
|
+
|
|
313
|
+
self.assertEqual(result, self.storage_response_data['data'][0])
|
|
314
|
+
|
|
315
|
+
@mock.patch("ciocore.api_client.ApiClient")
|
|
316
|
+
@mock.patch("ciocore.api_client.read_conductor_credentials")
|
|
317
|
+
@mock.patch("ciocore.api_client.account_id_from_jwt")
|
|
318
|
+
def test_storage_usage(self, mock_account_id_from_jwt, mock_read_conductor_credentials, mock_ApiClient):
|
|
319
|
+
# Set up mocks for successful execution
|
|
320
|
+
mock_read_conductor_credentials.return_value = "valid_token"
|
|
321
|
+
mock_account_id_from_jwt.return_value = "123"
|
|
322
|
+
mock_api_instance = mock_ApiClient.return_value
|
|
323
|
+
mock_api_instance.make_request.return_value = (self.response_ok.text, self.response_ok.status_code)
|
|
324
|
+
|
|
325
|
+
start_time = datetime.datetime(2024, 1, 1)
|
|
326
|
+
end_time = datetime.datetime(2024, 1, 7)
|
|
327
|
+
|
|
328
|
+
result = get_storage_usage(start_time=start_time, end_time=end_time)
|
|
329
|
+
|
|
330
|
+
self.assertEqual(result, { '2024-01-01': {'cost': 4.022, 'GiB': 679.714},
|
|
331
|
+
'2024-01-02': {'cost': 4.502, 'GiB': 750.34},
|
|
332
|
+
'2024-01-03': {'cost': 4.502, 'GiB': 750.34},
|
|
333
|
+
'2024-01-04': {'cost': 5.102, 'GiB': 850.36},
|
|
334
|
+
'2024-01-05': {'cost': 5.102, 'GiB': 850.35},
|
|
335
|
+
'2024-01-06': {'cost': 5.732, 'GiB': 955.32}
|
|
336
|
+
})
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from ciocore.downloader.base_downloader import (
|
|
2
|
+
BaseDownloader,
|
|
3
|
+
DEFAULT_NUM_THREADS,
|
|
4
|
+
DEFAULT_PROGRESS_INTERVAL,
|
|
5
|
+
DEFAULT_MAX_ATTEMPTS,
|
|
6
|
+
DEFAULT_DELAY,
|
|
7
|
+
DEFAULT_JITTER,
|
|
8
|
+
DEFAULT_PAGE_SIZE,
|
|
9
|
+
)
|
|
10
|
+
import unittest
|
|
11
|
+
from unittest import mock
|
|
12
|
+
|
|
13
|
+
from ciocore import api_client
|
|
14
|
+
|
|
15
|
+
from unittest.mock import patch
|
|
16
|
+
|
|
17
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TestBaseDownloaderInit(unittest.TestCase):
|
|
21
|
+
def test_default_values(self):
|
|
22
|
+
# Create an instance of the class
|
|
23
|
+
downloader = BaseDownloader()
|
|
24
|
+
|
|
25
|
+
# Assertions
|
|
26
|
+
self.assertIsNone(downloader.output_path)
|
|
27
|
+
self.assertFalse(downloader.force)
|
|
28
|
+
self.assertEqual(downloader.num_threads, DEFAULT_NUM_THREADS)
|
|
29
|
+
self.assertEqual(downloader.max_queue_size, DEFAULT_NUM_THREADS * 2)
|
|
30
|
+
self.assertEqual(
|
|
31
|
+
downloader.progress_interval, DEFAULT_PROGRESS_INTERVAL / 1000.0
|
|
32
|
+
)
|
|
33
|
+
self.assertEqual(downloader.page_size, DEFAULT_PAGE_SIZE)
|
|
34
|
+
self.assertIsInstance(downloader.client, api_client.ApiClient)
|
|
35
|
+
self.assertEqual(downloader.max_attempts, DEFAULT_MAX_ATTEMPTS)
|
|
36
|
+
self.assertEqual(downloader.delay, DEFAULT_DELAY)
|
|
37
|
+
self.assertEqual(downloader.jitter, DEFAULT_JITTER)
|
|
38
|
+
self.assertIsNone(downloader.regex)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_custom_values(self):
|
|
42
|
+
output_path = "/path/to/destination"
|
|
43
|
+
num_threads = 4
|
|
44
|
+
progress_interval = 500
|
|
45
|
+
page_size = 10
|
|
46
|
+
force = True
|
|
47
|
+
regex = r"\d+"
|
|
48
|
+
max_attempts = 3
|
|
49
|
+
delay = 2
|
|
50
|
+
jitter = 0.5
|
|
51
|
+
|
|
52
|
+
downloader = BaseDownloader(
|
|
53
|
+
output_path=output_path,
|
|
54
|
+
num_threads=num_threads,
|
|
55
|
+
progress_interval=progress_interval,
|
|
56
|
+
page_size=page_size,
|
|
57
|
+
force=force,
|
|
58
|
+
regex=regex,
|
|
59
|
+
max_attempts=max_attempts,
|
|
60
|
+
delay=delay,
|
|
61
|
+
jitter=jitter,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
# Assertions
|
|
65
|
+
self.assertEqual(downloader.output_path, output_path)
|
|
66
|
+
self.assertTrue(downloader.force)
|
|
67
|
+
self.assertEqual(downloader.num_threads, num_threads)
|
|
68
|
+
self.assertEqual(downloader.max_queue_size, num_threads * 2)
|
|
69
|
+
self.assertAlmostEqual(downloader.progress_interval, progress_interval / 1000.0)
|
|
70
|
+
self.assertEqual(downloader.page_size, page_size)
|
|
71
|
+
self.assertIsInstance(downloader.client, api_client.ApiClient)
|
|
72
|
+
self.assertEqual(downloader.max_attempts, max_attempts)
|
|
73
|
+
self.assertEqual(downloader.delay, delay)
|
|
74
|
+
self.assertEqual(downloader.jitter, jitter)
|
|
75
|
+
self.assertIsNotNone(downloader.regex)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class TestBaseDownloaderRun(unittest.TestCase):
|
|
79
|
+
def setUp(self):
|
|
80
|
+
self.downloader = BaseDownloader()
|
|
81
|
+
|
|
82
|
+
def tearDown(self):
|
|
83
|
+
pass
|
|
84
|
+
|
|
85
|
+
def test_run_method(self):
|
|
86
|
+
with patch(
|
|
87
|
+
"ciocore.downloader.base_downloader.ThreadPoolExecutor"
|
|
88
|
+
) as mock_executor:
|
|
89
|
+
my_mock_executor = mock.MagicMock(spec=ThreadPoolExecutor)
|
|
90
|
+
|
|
91
|
+
mock_executor.return_value.__enter__.return_value = my_mock_executor
|
|
92
|
+
|
|
93
|
+
tasks = [{"id": 1, "name": "task1"}, {"id": 2, "name": "task2"}]
|
|
94
|
+
next_locator = False
|
|
95
|
+
mock_get_some_tasks = mock.MagicMock(return_value=(tasks, next_locator))
|
|
96
|
+
|
|
97
|
+
self.downloader.get_some_tasks = mock_get_some_tasks
|
|
98
|
+
self.downloader.download_tasks = mock.MagicMock()
|
|
99
|
+
self.downloader.event_queue = mock.MagicMock()
|
|
100
|
+
|
|
101
|
+
self.downloader.run()
|
|
102
|
+
|
|
103
|
+
mock_get_some_tasks.assert_called_with(None)
|
|
104
|
+
self.downloader.download_tasks.assert_called_with(tasks, my_mock_executor)
|
tests/test_cli.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from click.testing import CliRunner
|
|
2
|
+
from ciocore.cli import upload as cli_upload
|
|
3
|
+
from ciocore.cli import download as cli_download
|
|
4
|
+
import unittest
|
|
5
|
+
import os
|
|
6
|
+
from unittest import mock
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class CliTestUploaderOptions(unittest.TestCase):
|
|
10
|
+
def setUp(self):
|
|
11
|
+
self.runner = CliRunner()
|
|
12
|
+
self.default_args = {
|
|
13
|
+
"database_filepath": None,
|
|
14
|
+
"location": None,
|
|
15
|
+
"md5_caching": True,
|
|
16
|
+
"thread_count": mock.ANY,
|
|
17
|
+
}
|
|
18
|
+
init_patcher = mock.patch("ciocore.cli.Uploader.__init__", autospec=True)
|
|
19
|
+
self.mock_init = init_patcher.start()
|
|
20
|
+
self.addCleanup(init_patcher.stop)
|
|
21
|
+
|
|
22
|
+
def test_receives_full_args_dict_with_defaults_when_no_args_given(self):
|
|
23
|
+
self.runner.invoke(cli_upload, [])
|
|
24
|
+
self.mock_init.assert_called_once_with(mock.ANY, self.default_args)
|
|
25
|
+
|
|
26
|
+
def test_database_filepath_arg(self):
|
|
27
|
+
self.runner.invoke(cli_upload, ["--database_filepath", "foo"])
|
|
28
|
+
expected = self.default_args
|
|
29
|
+
expected.update({"database_filepath": "foo"})
|
|
30
|
+
self.mock_init.assert_called_once_with(mock.ANY, expected)
|
|
31
|
+
|
|
32
|
+
def test_location_arg(self):
|
|
33
|
+
self.runner.invoke(cli_upload, ["--location", "foo"])
|
|
34
|
+
expected = self.default_args
|
|
35
|
+
expected.update({"location": "foo"})
|
|
36
|
+
self.mock_init.assert_called_once_with(mock.ANY, expected)
|
|
37
|
+
|
|
38
|
+
def test_md5_caching_arg(self):
|
|
39
|
+
self.runner.invoke(cli_upload, ["--md5_caching", False])
|
|
40
|
+
expected = self.default_args
|
|
41
|
+
expected.update({"md5_caching": False})
|
|
42
|
+
self.mock_init.assert_called_once_with(mock.ANY, expected)
|
|
43
|
+
|
|
44
|
+
def test_thread_count_arg(self):
|
|
45
|
+
self.runner.invoke(cli_upload, ["--thread_count", 4])
|
|
46
|
+
expected = self.default_args
|
|
47
|
+
expected.update({"thread_count": 4})
|
|
48
|
+
self.mock_init.assert_called_once_with(mock.ANY, expected)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class CliTestUploaderArguments(unittest.TestCase):
|
|
52
|
+
def setUp(self):
|
|
53
|
+
self.runner = CliRunner()
|
|
54
|
+
|
|
55
|
+
uploader_patcher = mock.patch("ciocore.cli.Uploader", autospec=True)
|
|
56
|
+
self.mock_uploader = uploader_patcher.start()
|
|
57
|
+
self.mock_inst = self.mock_uploader.return_value
|
|
58
|
+
self.addCleanup(uploader_patcher.stop)
|
|
59
|
+
|
|
60
|
+
def test_path_only_branch_if_paths(self):
|
|
61
|
+
with self.runner.isolated_filesystem():
|
|
62
|
+
filenames = ["foo.txt", "bar.txt", "baz.txt", "qux.txt"]
|
|
63
|
+
filenames = [os.path.join(os.getcwd(), filename) for filename in filenames]
|
|
64
|
+
for filename in filenames:
|
|
65
|
+
with open(filename, "w") as f:
|
|
66
|
+
f.write("hello world")
|
|
67
|
+
self.runner.invoke(cli_upload, filenames)
|
|
68
|
+
self.mock_inst.assets_only.assert_called_once_with(
|
|
69
|
+
mock.ANY, mock.ANY, mock.ANY, mock.ANY
|
|
70
|
+
)
|
|
71
|
+
self.mock_inst.main.assert_not_called()
|
|
72
|
+
|
|
73
|
+
def test_main_branch_if_no_paths(self):
|
|
74
|
+
with self.runner.isolated_filesystem():
|
|
75
|
+
self.runner.invoke(cli_upload)
|
|
76
|
+
self.mock_inst.main.assert_called_once()
|
|
77
|
+
self.mock_inst.assets_only.assert_not_called()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CliTestDownloader(unittest.TestCase):
|
|
81
|
+
def setUp(self):
|
|
82
|
+
self.runner = CliRunner()
|
|
83
|
+
|
|
84
|
+
dl_patcher_runner = mock.patch(
|
|
85
|
+
"ciocore.cli.LoggingDownloadRunner.__init__", autospec=True
|
|
86
|
+
)
|
|
87
|
+
self.mock_runner = dl_patcher_runner.start()
|
|
88
|
+
self.addCleanup(dl_patcher_runner.stop)
|
|
89
|
+
|
|
90
|
+
start_daemon_patcher = mock.patch(
|
|
91
|
+
"ciocore.cli.Downloader.start_daemon", autospec=True
|
|
92
|
+
)
|
|
93
|
+
self.mock_start_daemon = start_daemon_patcher.start()
|
|
94
|
+
self.addCleanup(start_daemon_patcher.stop)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_jobid_branch_if_job_id(self):
|
|
98
|
+
jid = "00000"
|
|
99
|
+
# self.assertTrue(True)
|
|
100
|
+
self.runner.invoke(cli_download, [jid])
|
|
101
|
+
self.mock_runner.assert_called_once( )
|
|
102
|
+
self.mock_start_daemon.assert_not_called()
|
|
103
|
+
|
|
104
|
+
# def test_job_id_only(self):
|
|
105
|
+
# jid = "00000"
|
|
106
|
+
# self.runner.invoke(cli_download, [jid])
|
|
107
|
+
# self.mock_dljobs.assert_called_once_with(
|
|
108
|
+
# (jid,), thread_count=mock.ANY, output_dir=mock.ANY
|
|
109
|
+
# )
|
|
110
|
+
|
|
111
|
+
# def test_several_job_ids(self):
|
|
112
|
+
# jid1 = "00000"
|
|
113
|
+
# jid2 = "11111"
|
|
114
|
+
# self.runner.invoke(cli_download, [jid1, jid2])
|
|
115
|
+
# self.mock_dljobs.assert_called_once_with(
|
|
116
|
+
# (jid1,jid2), thread_count=mock.ANY, output_dir=mock.ANY
|
|
117
|
+
# )
|
|
118
|
+
|
|
119
|
+
# def test_job_ids_and_others(self):
|
|
120
|
+
# jid1 = "00000"
|
|
121
|
+
# jid2 = "11111"
|
|
122
|
+
# tc = 4
|
|
123
|
+
# od = "foo"
|
|
124
|
+
# self.runner.invoke(
|
|
125
|
+
# cli_download,
|
|
126
|
+
# ["--thread_count", tc, "--destination", od, jid1, jid2 ],
|
|
127
|
+
# )
|
|
128
|
+
# self.mock_dljobs.assert_called_once_with(
|
|
129
|
+
# (jid1,jid2), thread_count=tc, output_dir=od
|
|
130
|
+
# )
|
|
131
|
+
|
|
132
|
+
# def test_daemon_branch_if_no_job_id(self):
|
|
133
|
+
# self.runner.invoke(cli_download, [])
|
|
134
|
+
# self.mock_start_daemon.assert_called_once()
|
|
135
|
+
# self.mock_dljobs.assert_not_called()
|
|
136
|
+
|
|
137
|
+
# def test_daemon_branch_args_present(self):
|
|
138
|
+
# tc = 4
|
|
139
|
+
# od = "foo"
|
|
140
|
+
# self.runner.invoke(cli_download)
|
|
141
|
+
# self.mock_start_daemon.assert_called_once_with(thread_count=mock.ANY, location=mock.ANY, output_dir=mock.ANY)
|
|
142
|
+
|
|
143
|
+
# def test_daemon_branch_args(self):
|
|
144
|
+
# tc = 4
|
|
145
|
+
# od = "/foo"
|
|
146
|
+
# loc = "bar"
|
|
147
|
+
# self.runner.invoke(cli_download, ["--thread_count", tc, "--location", loc])
|
|
148
|
+
# self.mock_start_daemon.assert_called_once_with(thread_count=tc, location="bar", output_dir=None)
|
|
149
|
+
|
tests/test_common.py
CHANGED
|
@@ -9,13 +9,7 @@ FILES_PATH = os.path.join(os.path.dirname(__file__), "files")
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class TestMd5(unittest.TestCase):
|
|
12
|
-
|
|
13
|
-
def test_get_base64_md5_is_correct_md5(self):
|
|
14
|
-
from ciocore import common
|
|
15
|
-
fn1 = os.path.join(FILES_PATH, "one")
|
|
16
|
-
md5=common.get_base64_md5(fn1)
|
|
17
|
-
self.assertEqual(md5, "9iVbsBxkj+lncU1SqJ6OnA==")
|
|
18
|
-
|
|
12
|
+
|
|
19
13
|
def test_get_base64_md5_is_correct_type(self):
|
|
20
14
|
from ciocore import common
|
|
21
15
|
from builtins import str
|