ciocore 8.0.0rc1__py2.py3-none-any.whl → 8.1.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ciocore might be problematic. Click here for more details.
- ciocore/VERSION +1 -1
- ciocore/api_client.py +223 -1
- ciocore/cli.py +8 -10
- ciocore/docsite/404.html +1 -1
- ciocore/docsite/apidoc/api_client/index.html +267 -1
- ciocore/docsite/apidoc/apidoc/index.html +1 -1
- ciocore/docsite/apidoc/config/index.html +1 -1
- ciocore/docsite/apidoc/data/index.html +1 -1
- ciocore/docsite/apidoc/hardware_set/index.html +1 -1
- ciocore/docsite/apidoc/package_environment/index.html +1 -1
- ciocore/docsite/apidoc/package_tree/index.html +1 -1
- ciocore/docsite/cmdline/docs/index.html +1 -1
- ciocore/docsite/cmdline/downloader/index.html +1 -1
- ciocore/docsite/cmdline/packages/index.html +1 -1
- ciocore/docsite/cmdline/uploader/index.html +1 -1
- ciocore/docsite/how-to-guides/index.html +1 -1
- ciocore/docsite/index.html +1 -1
- ciocore/docsite/objects.inv +0 -0
- ciocore/docsite/search/search_index.json +1 -1
- ciocore/docsite/sitemap.xml.gz +0 -0
- ciocore/uploader/_uploader.py +8 -7
- ciocore/worker.py +10 -5
- {ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/METADATA +11 -3
- {ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/RECORD +29 -29
- tests/test_api_client.py +174 -1
- tests/test_cli.py +0 -14
- {ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/WHEEL +0 -0
- {ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/entry_points.txt +0 -0
- {ciocore-8.0.0rc1.dist-info → ciocore-8.1.0.dist-info}/top_level.txt +0 -0
ciocore/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
8.
|
|
1
|
+
8.1.0
|
ciocore/api_client.py
CHANGED
|
@@ -3,6 +3,9 @@ The api_client module is used to make requests to the Conductor API.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import base64
|
|
6
|
+
import collections
|
|
7
|
+
import datetime
|
|
8
|
+
import hashlib
|
|
6
9
|
import importlib
|
|
7
10
|
import json
|
|
8
11
|
import jwt
|
|
@@ -14,7 +17,6 @@ import socket
|
|
|
14
17
|
import time
|
|
15
18
|
import sys
|
|
16
19
|
import platform
|
|
17
|
-
import hashlib
|
|
18
20
|
|
|
19
21
|
from urllib import parse
|
|
20
22
|
|
|
@@ -786,3 +788,223 @@ def kill_tasks(job_id, *task_ids):
|
|
|
786
788
|
raise Exception(msg)
|
|
787
789
|
|
|
788
790
|
return json.loads(response)
|
|
791
|
+
|
|
792
|
+
def _get_compute_usage(start_time, end_time):
|
|
793
|
+
"""
|
|
794
|
+
Query the account usage to get the raw compute data. Private method.
|
|
795
|
+
|
|
796
|
+
Compute includes licenses, instances and Conductor cost. Everything involved
|
|
797
|
+
with running a job.
|
|
798
|
+
|
|
799
|
+
Please use the public method api_client.get_compute_usage() instead.
|
|
800
|
+
|
|
801
|
+
Args:
|
|
802
|
+
start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
803
|
+
end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
804
|
+
|
|
805
|
+
Returns:
|
|
806
|
+
list: A list of billing entries
|
|
807
|
+
|
|
808
|
+
Examples:
|
|
809
|
+
>>> from ciocore import api_client
|
|
810
|
+
>>> api_client._get_compute_usage(start_time, end_time)
|
|
811
|
+
[
|
|
812
|
+
{
|
|
813
|
+
"cores": 0.5,
|
|
814
|
+
"instance_cost": 0.019999999552965164,
|
|
815
|
+
"license_cost": 0.019999999552965164,
|
|
816
|
+
"minutes": 6.9700000286102295,
|
|
817
|
+
"self_link": 0,
|
|
818
|
+
"start_time": "Tue, 09 Jan 2024 18:00:00 GMT"
|
|
819
|
+
},
|
|
820
|
+
{
|
|
821
|
+
"cores": 0.4,
|
|
822
|
+
"instance_cost": 0.019999999552965164,
|
|
823
|
+
"license_cost": 0.019999999552965164,
|
|
824
|
+
"minutes": 6.960000038146973,
|
|
825
|
+
"self_link": 1,
|
|
826
|
+
"start_time": "Tue, 09 Jan 2024 19:00:00 GMT"
|
|
827
|
+
}]
|
|
828
|
+
"""
|
|
829
|
+
|
|
830
|
+
api = ApiClient()
|
|
831
|
+
|
|
832
|
+
payload = {
|
|
833
|
+
"filter": json.dumps(
|
|
834
|
+
[{"field": "start_time", "op": ">=", "value": start_time.date().isoformat()},
|
|
835
|
+
{"field": "start_time", "op": "<", "value": end_time.date().isoformat()}]),
|
|
836
|
+
"group_by": json.dumps(["start_time"]),
|
|
837
|
+
"order_by": "start_time"
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
response, response_code = api.make_request(
|
|
841
|
+
uri_path="billing/get_usage",
|
|
842
|
+
verb="GET",
|
|
843
|
+
raise_on_error=False,
|
|
844
|
+
use_api_key=True,
|
|
845
|
+
params=payload
|
|
846
|
+
)
|
|
847
|
+
|
|
848
|
+
if response_code not in [200]:
|
|
849
|
+
msg = f"Failed to query compute usage"
|
|
850
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
851
|
+
raise Exception(msg)
|
|
852
|
+
|
|
853
|
+
return json.loads(response)['data']
|
|
854
|
+
|
|
855
|
+
def _get_storage_usage(start_time, end_time):
|
|
856
|
+
"""
|
|
857
|
+
Query the account usage to get the raw storage data. Private method.
|
|
858
|
+
|
|
859
|
+
Please use the public method api_client.get_storage_usage() instead.
|
|
860
|
+
|
|
861
|
+
Args:
|
|
862
|
+
start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
863
|
+
end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
864
|
+
|
|
865
|
+
Returns:
|
|
866
|
+
dict: A dict of billing details related to storage
|
|
867
|
+
|
|
868
|
+
Examples:
|
|
869
|
+
>>> from ciocore import api_client
|
|
870
|
+
>>> api_client._get_storage_usage(start_time, end_time)
|
|
871
|
+
{
|
|
872
|
+
"cost": "28.96",
|
|
873
|
+
"cost_per_day": [
|
|
874
|
+
4.022,
|
|
875
|
+
4.502,
|
|
876
|
+
4.502,
|
|
877
|
+
5.102,
|
|
878
|
+
5.102,
|
|
879
|
+
5.732
|
|
880
|
+
],
|
|
881
|
+
"currency": "USD",
|
|
882
|
+
"daily_price": "0.006",
|
|
883
|
+
"end_date": "2024-01-07",
|
|
884
|
+
"gibs_per_day": [
|
|
885
|
+
679.714,
|
|
886
|
+
750.34,
|
|
887
|
+
750.34,
|
|
888
|
+
850.36,
|
|
889
|
+
850.35,
|
|
890
|
+
955.32
|
|
891
|
+
],
|
|
892
|
+
"gibs_used": "806.07",
|
|
893
|
+
"monthly_price": "0.18",
|
|
894
|
+
"start_date": "2024-01-01",
|
|
895
|
+
"storage_unit": "GiB"
|
|
896
|
+
}
|
|
897
|
+
]
|
|
898
|
+
}
|
|
899
|
+
"""
|
|
900
|
+
|
|
901
|
+
api = ApiClient()
|
|
902
|
+
|
|
903
|
+
# Add one day to the end time as the query is exclusive of the last day but
|
|
904
|
+
# we want consistency with _get_compute_usage()
|
|
905
|
+
payload = {
|
|
906
|
+
"start": start_time.date().isoformat(),
|
|
907
|
+
"end": (end_time.date() + datetime.timedelta(days=1)).isoformat()
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
response, response_code = api.make_request(
|
|
911
|
+
uri_path="billing/get_storage_usage",
|
|
912
|
+
verb="GET",
|
|
913
|
+
raise_on_error=False,
|
|
914
|
+
use_api_key=True,
|
|
915
|
+
params=payload
|
|
916
|
+
)
|
|
917
|
+
|
|
918
|
+
if response_code not in [200]:
|
|
919
|
+
msg = f"Failed to query storage usage"
|
|
920
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
921
|
+
raise Exception(msg)
|
|
922
|
+
|
|
923
|
+
return json.loads(response)['data'][0]
|
|
924
|
+
|
|
925
|
+
def get_compute_usage(start_time, end_time):
|
|
926
|
+
'''
|
|
927
|
+
Query the compute usage for an account.
|
|
928
|
+
|
|
929
|
+
Compute includes licenses, instances and Conductor cost. Everything involved
|
|
930
|
+
with running a job.
|
|
931
|
+
|
|
932
|
+
Args:
|
|
933
|
+
start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
934
|
+
end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
935
|
+
|
|
936
|
+
Returns:
|
|
937
|
+
dict: Each key is a date (UTC). The value is a dict with values for:
|
|
938
|
+
- cost: The total accumulated compute cost for the day
|
|
939
|
+
- corehours: The total accumulated core hours for the day
|
|
940
|
+
- walltime: The number of minutes that instances (regardless of type) were running
|
|
941
|
+
|
|
942
|
+
Examples:
|
|
943
|
+
>>> from ciocore import api_client
|
|
944
|
+
>>> api_client.get_compute_usage(start_time, end_time)
|
|
945
|
+
{ '2024-01-09': { 'cost': 0.08,
|
|
946
|
+
'corehours': 0.9,
|
|
947
|
+
'walltime': 13.93},
|
|
948
|
+
'2024-01-16': { 'cost': 0.12,
|
|
949
|
+
'corehours': 0.9613,
|
|
950
|
+
'walltime': 7.21}}
|
|
951
|
+
'''
|
|
952
|
+
date_format = "%a, %d %b %Y %H:%M:%S %Z"
|
|
953
|
+
data = _get_compute_usage(start_time, end_time)
|
|
954
|
+
|
|
955
|
+
# Create a nested default dictionary with initial float values of 0.0
|
|
956
|
+
results = collections.defaultdict(lambda: collections.defaultdict(float))
|
|
957
|
+
|
|
958
|
+
for entry in data:
|
|
959
|
+
entry_start_date = datetime.datetime.strptime(entry['start_time'], date_format).date().isoformat()
|
|
960
|
+
|
|
961
|
+
results[entry_start_date]['walltime'] += entry['minutes']
|
|
962
|
+
results[entry_start_date]['corehours'] += entry['cores']
|
|
963
|
+
results[entry_start_date]['cost'] += entry['license_cost'] + entry['instance_cost']
|
|
964
|
+
|
|
965
|
+
# Round the data to avoid FP errors
|
|
966
|
+
results[entry_start_date]['walltime'] = round(results[entry_start_date]['walltime'], 4)
|
|
967
|
+
results[entry_start_date]['corehours'] = round(results[entry_start_date]['corehours'], 4)
|
|
968
|
+
results[entry_start_date]['cost'] = round(results[entry_start_date]['cost'], 4)
|
|
969
|
+
|
|
970
|
+
return results
|
|
971
|
+
|
|
972
|
+
def get_storage_usage(start_time, end_time):
|
|
973
|
+
'''
|
|
974
|
+
Query the storage usage for an account.
|
|
975
|
+
|
|
976
|
+
Storage is calculated twice a day (UTC) and the average is used.
|
|
977
|
+
|
|
978
|
+
Args:
|
|
979
|
+
start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
980
|
+
end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.
|
|
981
|
+
|
|
982
|
+
Returns:
|
|
983
|
+
dict: Each key is a date (UTC). The value is a dict with values for:
|
|
984
|
+
- cost: The cost of accumulated storage for that one day
|
|
985
|
+
- GiB: The total amount of storage used on that day
|
|
986
|
+
|
|
987
|
+
Examples:
|
|
988
|
+
>>> from ciocore import api_client
|
|
989
|
+
>>> api_client.get_storage_usage(start_time, end_time)
|
|
990
|
+
{ '2024-01-01': {'cost': 4.022, 'GiB': 679.714},
|
|
991
|
+
'2024-01-02': {'cost': 4.502, 'GiB': 750.34},
|
|
992
|
+
'2024-01-03': {'cost': 4.502, 'GiB': 750.34}}
|
|
993
|
+
'''
|
|
994
|
+
one_day = datetime.timedelta(days=1)
|
|
995
|
+
|
|
996
|
+
data = _get_storage_usage(start_time, end_time)
|
|
997
|
+
|
|
998
|
+
results = {}
|
|
999
|
+
|
|
1000
|
+
entry_date = datetime.date.fromisoformat(data['start_date'])
|
|
1001
|
+
|
|
1002
|
+
for cnt, entry in enumerate(data["cost_per_day"]):
|
|
1003
|
+
|
|
1004
|
+
entry_start_date = entry_date.isoformat()
|
|
1005
|
+
results[entry_start_date] = {'cost': float(entry), 'GiB': float(data['gibs_per_day'][cnt])}
|
|
1006
|
+
entry_date += one_day
|
|
1007
|
+
|
|
1008
|
+
return results
|
|
1009
|
+
|
|
1010
|
+
|
ciocore/cli.py
CHANGED
|
@@ -118,9 +118,11 @@ def _set_logging(log_level, log_to_console=True, log_dir=None, log_filename=None
|
|
|
118
118
|
log_filename=log_filename,
|
|
119
119
|
console_formatter=loggeria.FORMATTER_VERBOSE,
|
|
120
120
|
disable_console_logging= not log_to_console,
|
|
121
|
-
use_system_log=
|
|
121
|
+
use_system_log=True,
|
|
122
122
|
)
|
|
123
123
|
|
|
124
|
+
print("Logging to %s" % loggeria.LOG_PATH)
|
|
125
|
+
|
|
124
126
|
def _register(client):
|
|
125
127
|
api_client.ApiClient.register_client(
|
|
126
128
|
client_name=client.CLIENT_NAME, client_version=VERSION
|
|
@@ -166,7 +168,7 @@ def main(ctx, version):
|
|
|
166
168
|
default=DEFAULT_CONFIG_LOG_LEVEL,
|
|
167
169
|
)
|
|
168
170
|
@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
|
|
169
|
-
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=
|
|
171
|
+
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=False)
|
|
170
172
|
@click.option(
|
|
171
173
|
"-tc",
|
|
172
174
|
"--thread_count",
|
|
@@ -189,15 +191,13 @@ def upload(
|
|
|
189
191
|
Example:
|
|
190
192
|
conductor upload file1 file2 file3
|
|
191
193
|
"""
|
|
192
|
-
|
|
193
|
-
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir)
|
|
194
|
+
|
|
195
|
+
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir, log_filename="conductor_uploader.log")
|
|
194
196
|
|
|
195
197
|
args_dict = {
|
|
196
198
|
"database_filepath": database_filepath,
|
|
197
199
|
"location": location,
|
|
198
200
|
"md5_caching": md5_caching,
|
|
199
|
-
"log_level": log_level,
|
|
200
|
-
"log_dir": log_dir,
|
|
201
201
|
"thread_count": thread_count,
|
|
202
202
|
}
|
|
203
203
|
|
|
@@ -284,7 +284,7 @@ DEPRECATED_PATHS_HELP = "Specify a list of paths to upload."
|
|
|
284
284
|
help=THREADS_HELP,
|
|
285
285
|
default=DEFAULT_CONFIG_THREAD_COUNT,
|
|
286
286
|
)
|
|
287
|
-
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=
|
|
287
|
+
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=False)
|
|
288
288
|
@click.option("-lc", "--location", help=LOCATION_HELP)
|
|
289
289
|
@click.option(
|
|
290
290
|
"-p",
|
|
@@ -306,14 +306,12 @@ def uploader(
|
|
|
306
306
|
Example:
|
|
307
307
|
conductor upload file1 file2 file3
|
|
308
308
|
"""
|
|
309
|
-
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir)
|
|
309
|
+
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir, log_filename="conductor_uploader.log")
|
|
310
310
|
|
|
311
311
|
args_dict = {
|
|
312
312
|
"database_filepath": database_filepath,
|
|
313
313
|
"location": location,
|
|
314
314
|
"md5_caching": md5_caching,
|
|
315
|
-
"log_level": log_level,
|
|
316
|
-
"log_dir": log_dir,
|
|
317
315
|
"thread_count": thread_count,
|
|
318
316
|
}
|
|
319
317
|
|
ciocore/docsite/404.html
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
<link rel="icon" href="../../assets/images/favicon.png">
|
|
21
|
-
<meta name="generator" content="mkdocs-1.
|
|
21
|
+
<meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
|
|
@@ -709,6 +709,20 @@
|
|
|
709
709
|
kill_tasks()
|
|
710
710
|
</a>
|
|
711
711
|
|
|
712
|
+
</li>
|
|
713
|
+
|
|
714
|
+
<li class="md-nav__item">
|
|
715
|
+
<a href="#ciocore.api_client.get_compute_usage" class="md-nav__link">
|
|
716
|
+
get_compute_usage()
|
|
717
|
+
</a>
|
|
718
|
+
|
|
719
|
+
</li>
|
|
720
|
+
|
|
721
|
+
<li class="md-nav__item">
|
|
722
|
+
<a href="#ciocore.api_client.get_storage_usage" class="md-nav__link">
|
|
723
|
+
get_storage_usage()
|
|
724
|
+
</a>
|
|
725
|
+
|
|
712
726
|
</li>
|
|
713
727
|
|
|
714
728
|
</ul>
|
|
@@ -1023,6 +1037,20 @@
|
|
|
1023
1037
|
kill_tasks()
|
|
1024
1038
|
</a>
|
|
1025
1039
|
|
|
1040
|
+
</li>
|
|
1041
|
+
|
|
1042
|
+
<li class="md-nav__item">
|
|
1043
|
+
<a href="#ciocore.api_client.get_compute_usage" class="md-nav__link">
|
|
1044
|
+
get_compute_usage()
|
|
1045
|
+
</a>
|
|
1046
|
+
|
|
1047
|
+
</li>
|
|
1048
|
+
|
|
1049
|
+
<li class="md-nav__item">
|
|
1050
|
+
<a href="#ciocore.api_client.get_storage_usage" class="md-nav__link">
|
|
1051
|
+
get_storage_usage()
|
|
1052
|
+
</a>
|
|
1053
|
+
|
|
1026
1054
|
</li>
|
|
1027
1055
|
|
|
1028
1056
|
</ul>
|
|
@@ -3138,6 +3166,244 @@ prompt the user to log in. </p>
|
|
|
3138
3166
|
|
|
3139
3167
|
|
|
3140
3168
|
|
|
3169
|
+
<div class="doc doc-object doc-function">
|
|
3170
|
+
|
|
3171
|
+
|
|
3172
|
+
|
|
3173
|
+
<h2 id="ciocore.api_client.get_compute_usage" class="doc doc-heading">
|
|
3174
|
+
<code class="highlight language-python"><span class="n">get_compute_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span></code>
|
|
3175
|
+
|
|
3176
|
+
|
|
3177
|
+
</h2>
|
|
3178
|
+
|
|
3179
|
+
<div class="doc doc-contents ">
|
|
3180
|
+
|
|
3181
|
+
<p>Query the compute usage for an account.</p>
|
|
3182
|
+
<p>Compute includes licenses, instances and Conductor cost. Everything involved
|
|
3183
|
+
with running a job. </p>
|
|
3184
|
+
|
|
3185
|
+
<p><strong>Parameters:</strong></p>
|
|
3186
|
+
<table>
|
|
3187
|
+
<thead>
|
|
3188
|
+
<tr>
|
|
3189
|
+
<th>Name</th>
|
|
3190
|
+
<th>Type</th>
|
|
3191
|
+
<th>Description</th>
|
|
3192
|
+
<th>Default</th>
|
|
3193
|
+
</tr>
|
|
3194
|
+
</thead>
|
|
3195
|
+
<tbody>
|
|
3196
|
+
<tr>
|
|
3197
|
+
<td><code>start_time</code></td>
|
|
3198
|
+
<td><code>datetime.datetime</code></td>
|
|
3199
|
+
<td><p>The first day to include in the report. Only the date is considered and it's assumed to be in UTC.</p></td>
|
|
3200
|
+
<td><em>required</em></td>
|
|
3201
|
+
</tr>
|
|
3202
|
+
<tr>
|
|
3203
|
+
<td><code>end_time</code></td>
|
|
3204
|
+
<td><code>datetime.datetime</code></td>
|
|
3205
|
+
<td><p>The last day to include in the report. Only the date is considered and it's assumed to be in UTC.</p></td>
|
|
3206
|
+
<td><em>required</em></td>
|
|
3207
|
+
</tr>
|
|
3208
|
+
</tbody>
|
|
3209
|
+
</table>
|
|
3210
|
+
<p><strong>Returns:</strong></p>
|
|
3211
|
+
<table>
|
|
3212
|
+
<thead>
|
|
3213
|
+
<tr>
|
|
3214
|
+
<th>Type</th>
|
|
3215
|
+
<th>Description</th>
|
|
3216
|
+
</tr>
|
|
3217
|
+
</thead>
|
|
3218
|
+
<tbody>
|
|
3219
|
+
<tr>
|
|
3220
|
+
<td><code>dict</code></td>
|
|
3221
|
+
<td><p>Each key is a date (UTC). The value is a dict with values for:
|
|
3222
|
+
- cost: The total accumulated compute cost for the day
|
|
3223
|
+
- corehours: The total accumulated core hours for the day
|
|
3224
|
+
- walltime: The number of minutes that instances (regardless of type) were running</p></td>
|
|
3225
|
+
</tr>
|
|
3226
|
+
</tbody>
|
|
3227
|
+
</table>
|
|
3228
|
+
<p><strong>Examples:</strong></p>
|
|
3229
|
+
<div class="highlight"><pre><span></span><code><span class="o">>>></span> <span class="kn">from</span> <span class="nn">ciocore</span> <span class="kn">import</span> <span class="n">api_client</span>
|
|
3230
|
+
<span class="o">>>></span> <span class="n">api_client</span><span class="o">.</span><span class="n">get_compute_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span>
|
|
3231
|
+
<span class="p">{</span> <span class="s1">'2024-01-09'</span><span class="p">:</span> <span class="p">{</span> <span class="s1">'cost'</span><span class="p">:</span> <span class="mf">0.08</span><span class="p">,</span>
|
|
3232
|
+
<span class="s1">'corehours'</span><span class="p">:</span> <span class="mf">0.9</span><span class="p">,</span>
|
|
3233
|
+
<span class="s1">'walltime'</span><span class="p">:</span> <span class="mf">13.93</span><span class="p">},</span>
|
|
3234
|
+
<span class="s1">'2024-01-16'</span><span class="p">:</span> <span class="p">{</span> <span class="s1">'cost'</span><span class="p">:</span> <span class="mf">0.12</span><span class="p">,</span>
|
|
3235
|
+
<span class="s1">'corehours'</span><span class="p">:</span> <span class="mf">0.9613</span><span class="p">,</span>
|
|
3236
|
+
<span class="s1">'walltime'</span><span class="p">:</span> <span class="mf">7.21</span><span class="p">}}</span>
|
|
3237
|
+
</code></pre></div>
|
|
3238
|
+
|
|
3239
|
+
<details class="quote">
|
|
3240
|
+
<summary>Source code in <code>ciocore/api_client.py</code></summary>
|
|
3241
|
+
<div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">get_compute_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">):</span>
|
|
3242
|
+
<span class="w"> </span><span class="sd">'''</span>
|
|
3243
|
+
<span class="sd"> Query the compute usage for an account.</span>
|
|
3244
|
+
|
|
3245
|
+
<span class="sd"> Compute includes licenses, instances and Conductor cost. Everything involved</span>
|
|
3246
|
+
<span class="sd"> with running a job. </span>
|
|
3247
|
+
|
|
3248
|
+
<span class="sd"> Args:</span>
|
|
3249
|
+
<span class="sd"> start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.</span>
|
|
3250
|
+
<span class="sd"> end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.</span>
|
|
3251
|
+
|
|
3252
|
+
<span class="sd"> Returns:</span>
|
|
3253
|
+
<span class="sd"> dict: Each key is a date (UTC). The value is a dict with values for:</span>
|
|
3254
|
+
<span class="sd"> - cost: The total accumulated compute cost for the day</span>
|
|
3255
|
+
<span class="sd"> - corehours: The total accumulated core hours for the day</span>
|
|
3256
|
+
<span class="sd"> - walltime: The number of minutes that instances (regardless of type) were running</span>
|
|
3257
|
+
|
|
3258
|
+
<span class="sd"> Examples:</span>
|
|
3259
|
+
<span class="sd"> >>> from ciocore import api_client</span>
|
|
3260
|
+
<span class="sd"> >>> api_client.get_compute_usage(start_time, end_time)</span>
|
|
3261
|
+
<span class="sd"> { '2024-01-09': { 'cost': 0.08,</span>
|
|
3262
|
+
<span class="sd"> 'corehours': 0.9, </span>
|
|
3263
|
+
<span class="sd"> 'walltime': 13.93},</span>
|
|
3264
|
+
<span class="sd"> '2024-01-16': { 'cost': 0.12,</span>
|
|
3265
|
+
<span class="sd"> 'corehours': 0.9613, </span>
|
|
3266
|
+
<span class="sd"> 'walltime': 7.21}}</span>
|
|
3267
|
+
<span class="sd"> '''</span>
|
|
3268
|
+
<span class="n">date_format</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%a</span><span class="s2">, </span><span class="si">%d</span><span class="s2"> %b %Y %H:%M:%S %Z"</span>
|
|
3269
|
+
<span class="n">data</span> <span class="o">=</span> <span class="n">_get_compute_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span>
|
|
3270
|
+
|
|
3271
|
+
<span class="c1"># Create a nested default dictionary with initial float values of 0.0</span>
|
|
3272
|
+
<span class="n">results</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">defaultdict</span><span class="p">(</span><span class="k">lambda</span><span class="p">:</span> <span class="n">collections</span><span class="o">.</span><span class="n">defaultdict</span><span class="p">(</span><span class="nb">float</span><span class="p">))</span>
|
|
3273
|
+
|
|
3274
|
+
<span class="k">for</span> <span class="n">entry</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
|
3275
|
+
<span class="n">entry_start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">entry</span><span class="p">[</span><span class="s1">'start_time'</span><span class="p">],</span> <span class="n">date_format</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
3276
|
+
|
|
3277
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'walltime'</span><span class="p">]</span> <span class="o">+=</span> <span class="n">entry</span><span class="p">[</span><span class="s1">'minutes'</span><span class="p">]</span>
|
|
3278
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'corehours'</span><span class="p">]</span> <span class="o">+=</span> <span class="n">entry</span><span class="p">[</span><span class="s1">'cores'</span><span class="p">]</span>
|
|
3279
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'cost'</span><span class="p">]</span> <span class="o">+=</span> <span class="n">entry</span><span class="p">[</span><span class="s1">'license_cost'</span><span class="p">]</span> <span class="o">+</span> <span class="n">entry</span><span class="p">[</span><span class="s1">'instance_cost'</span><span class="p">]</span>
|
|
3280
|
+
|
|
3281
|
+
<span class="c1"># Round the data to avoid FP errors</span>
|
|
3282
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'walltime'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">round</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'walltime'</span><span class="p">],</span> <span class="mi">4</span><span class="p">)</span>
|
|
3283
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'corehours'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">round</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'corehours'</span><span class="p">],</span> <span class="mi">4</span><span class="p">)</span>
|
|
3284
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'cost'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">round</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">][</span><span class="s1">'cost'</span><span class="p">],</span> <span class="mi">4</span><span class="p">)</span>
|
|
3285
|
+
|
|
3286
|
+
<span class="k">return</span> <span class="n">results</span>
|
|
3287
|
+
</code></pre></div>
|
|
3288
|
+
</details>
|
|
3289
|
+
</div>
|
|
3290
|
+
|
|
3291
|
+
</div>
|
|
3292
|
+
|
|
3293
|
+
|
|
3294
|
+
|
|
3295
|
+
<div class="doc doc-object doc-function">
|
|
3296
|
+
|
|
3297
|
+
|
|
3298
|
+
|
|
3299
|
+
<h2 id="ciocore.api_client.get_storage_usage" class="doc doc-heading">
|
|
3300
|
+
<code class="highlight language-python"><span class="n">get_storage_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span></code>
|
|
3301
|
+
|
|
3302
|
+
|
|
3303
|
+
</h2>
|
|
3304
|
+
|
|
3305
|
+
<div class="doc doc-contents ">
|
|
3306
|
+
|
|
3307
|
+
<p>Query the storage usage for an account.</p>
|
|
3308
|
+
<p>Storage is calculated twice a day (UTC) and the average is used.</p>
|
|
3309
|
+
|
|
3310
|
+
<p><strong>Parameters:</strong></p>
|
|
3311
|
+
<table>
|
|
3312
|
+
<thead>
|
|
3313
|
+
<tr>
|
|
3314
|
+
<th>Name</th>
|
|
3315
|
+
<th>Type</th>
|
|
3316
|
+
<th>Description</th>
|
|
3317
|
+
<th>Default</th>
|
|
3318
|
+
</tr>
|
|
3319
|
+
</thead>
|
|
3320
|
+
<tbody>
|
|
3321
|
+
<tr>
|
|
3322
|
+
<td><code>start_time</code></td>
|
|
3323
|
+
<td><code>datetime.datetime</code></td>
|
|
3324
|
+
<td><p>The first day to include in the report. Only the date is considered and it's assumed to be in UTC.</p></td>
|
|
3325
|
+
<td><em>required</em></td>
|
|
3326
|
+
</tr>
|
|
3327
|
+
<tr>
|
|
3328
|
+
<td><code>end_time</code></td>
|
|
3329
|
+
<td><code>datetime.datetime</code></td>
|
|
3330
|
+
<td><p>The last day to include in the report. Only the date is considered and it's assumed to be in UTC.</p></td>
|
|
3331
|
+
<td><em>required</em></td>
|
|
3332
|
+
</tr>
|
|
3333
|
+
</tbody>
|
|
3334
|
+
</table>
|
|
3335
|
+
<p><strong>Returns:</strong></p>
|
|
3336
|
+
<table>
|
|
3337
|
+
<thead>
|
|
3338
|
+
<tr>
|
|
3339
|
+
<th>Type</th>
|
|
3340
|
+
<th>Description</th>
|
|
3341
|
+
</tr>
|
|
3342
|
+
</thead>
|
|
3343
|
+
<tbody>
|
|
3344
|
+
<tr>
|
|
3345
|
+
<td><code>dict</code></td>
|
|
3346
|
+
<td><p>Each key is a date (UTC). The value is a dict with values for:
|
|
3347
|
+
- cost: The cost of accumulated storage for that one day
|
|
3348
|
+
- GiB: The total amount of storage used on that day</p></td>
|
|
3349
|
+
</tr>
|
|
3350
|
+
</tbody>
|
|
3351
|
+
</table>
|
|
3352
|
+
<p><strong>Examples:</strong></p>
|
|
3353
|
+
<div class="highlight"><pre><span></span><code><span class="o">>>></span> <span class="kn">from</span> <span class="nn">ciocore</span> <span class="kn">import</span> <span class="n">api_client</span>
|
|
3354
|
+
<span class="o">>>></span> <span class="n">api_client</span><span class="o">.</span><span class="n">get_storage_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span>
|
|
3355
|
+
<span class="p">{</span> <span class="s1">'2024-01-01'</span><span class="p">:</span> <span class="p">{</span><span class="s1">'cost'</span><span class="p">:</span> <span class="mf">4.022</span><span class="p">,</span> <span class="s1">'GiB'</span><span class="p">:</span> <span class="mf">679.714</span><span class="p">},</span>
|
|
3356
|
+
<span class="s1">'2024-01-02'</span><span class="p">:</span> <span class="p">{</span><span class="s1">'cost'</span><span class="p">:</span> <span class="mf">4.502</span><span class="p">,</span> <span class="s1">'GiB'</span><span class="p">:</span> <span class="mf">750.34</span><span class="p">},</span>
|
|
3357
|
+
<span class="s1">'2024-01-03'</span><span class="p">:</span> <span class="p">{</span><span class="s1">'cost'</span><span class="p">:</span> <span class="mf">4.502</span><span class="p">,</span> <span class="s1">'GiB'</span><span class="p">:</span> <span class="mf">750.34</span><span class="p">}}</span>
|
|
3358
|
+
</code></pre></div>
|
|
3359
|
+
|
|
3360
|
+
<details class="quote">
|
|
3361
|
+
<summary>Source code in <code>ciocore/api_client.py</code></summary>
|
|
3362
|
+
<div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">get_storage_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">):</span>
|
|
3363
|
+
<span class="w"> </span><span class="sd">'''</span>
|
|
3364
|
+
<span class="sd"> Query the storage usage for an account.</span>
|
|
3365
|
+
|
|
3366
|
+
<span class="sd"> Storage is calculated twice a day (UTC) and the average is used.</span>
|
|
3367
|
+
|
|
3368
|
+
<span class="sd"> Args:</span>
|
|
3369
|
+
<span class="sd"> start_time (datetime.datetime): The first day to include in the report. Only the date is considered and it's assumed to be in UTC.</span>
|
|
3370
|
+
<span class="sd"> end_time (datetime.datetime): The last day to include in the report. Only the date is considered and it's assumed to be in UTC.</span>
|
|
3371
|
+
|
|
3372
|
+
<span class="sd"> Returns:</span>
|
|
3373
|
+
<span class="sd"> dict: Each key is a date (UTC). The value is a dict with values for:</span>
|
|
3374
|
+
<span class="sd"> - cost: The cost of accumulated storage for that one day</span>
|
|
3375
|
+
<span class="sd"> - GiB: The total amount of storage used on that day</span>
|
|
3376
|
+
|
|
3377
|
+
<span class="sd"> Examples:</span>
|
|
3378
|
+
<span class="sd"> >>> from ciocore import api_client</span>
|
|
3379
|
+
<span class="sd"> >>> api_client.get_storage_usage(start_time, end_time)</span>
|
|
3380
|
+
<span class="sd"> { '2024-01-01': {'cost': 4.022, 'GiB': 679.714},</span>
|
|
3381
|
+
<span class="sd"> '2024-01-02': {'cost': 4.502, 'GiB': 750.34},</span>
|
|
3382
|
+
<span class="sd"> '2024-01-03': {'cost': 4.502, 'GiB': 750.34}}</span>
|
|
3383
|
+
<span class="sd"> '''</span>
|
|
3384
|
+
<span class="n">one_day</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
3385
|
+
|
|
3386
|
+
<span class="n">data</span> <span class="o">=</span> <span class="n">_get_storage_usage</span><span class="p">(</span><span class="n">start_time</span><span class="p">,</span> <span class="n">end_time</span><span class="p">)</span>
|
|
3387
|
+
|
|
3388
|
+
<span class="n">results</span> <span class="o">=</span> <span class="p">{}</span>
|
|
3389
|
+
|
|
3390
|
+
<span class="n">entry_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">])</span>
|
|
3391
|
+
|
|
3392
|
+
<span class="k">for</span> <span class="n">cnt</span><span class="p">,</span> <span class="n">entry</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="s2">"cost_per_day"</span><span class="p">]):</span>
|
|
3393
|
+
|
|
3394
|
+
<span class="n">entry_start_date</span> <span class="o">=</span> <span class="n">entry_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
|
3395
|
+
<span class="n">results</span><span class="p">[</span><span class="n">entry_start_date</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'cost'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">entry</span><span class="p">),</span> <span class="s1">'GiB'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="s1">'gibs_per_day'</span><span class="p">][</span><span class="n">cnt</span><span class="p">])}</span>
|
|
3396
|
+
<span class="n">entry_date</span> <span class="o">+=</span> <span class="n">one_day</span>
|
|
3397
|
+
|
|
3398
|
+
<span class="k">return</span> <span class="n">results</span>
|
|
3399
|
+
</code></pre></div>
|
|
3400
|
+
</details>
|
|
3401
|
+
</div>
|
|
3402
|
+
|
|
3403
|
+
</div>
|
|
3404
|
+
|
|
3405
|
+
|
|
3406
|
+
|
|
3141
3407
|
|
|
3142
3408
|
|
|
3143
3409
|
|