ciocore 8.0.0b11__py2.py3-none-any.whl → 8.0.0b14__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ciocore might be problematic. Click here for more details.
- ciocore/VERSION +1 -1
- ciocore/api_client.py +40 -81
- ciocore/cli.py +12 -14
- ciocore/config.py +13 -1
- ciocore/docsite/apidoc/api_client/index.html +146 -77
- ciocore/docsite/apidoc/config/index.html +8 -2
- ciocore/docsite/objects.inv +0 -0
- ciocore/docsite/search/search_index.json +1 -1
- ciocore/docsite/sitemap.xml.gz +0 -0
- ciocore/downloader/base_downloader.py +3 -1
- ciocore/downloader/job_downloader.py +7 -7
- ciocore/downloader/logging_download_runner.py +1 -2
- {ciocore-8.0.0b11.dist-info → ciocore-8.0.0b14.dist-info}/METADATA +4 -1
- {ciocore-8.0.0b11.dist-info → ciocore-8.0.0b14.dist-info}/RECORD +21 -21
- tests/test_api_client.py +61 -1
- tests/test_common.py +8 -8
- tests/test_downloader.py +1 -0
- tests/test_hardware_set.py +33 -30
- {ciocore-8.0.0b11.dist-info → ciocore-8.0.0b14.dist-info}/WHEEL +0 -0
- {ciocore-8.0.0b11.dist-info → ciocore-8.0.0b14.dist-info}/entry_points.txt +0 -0
- {ciocore-8.0.0b11.dist-info → ciocore-8.0.0b14.dist-info}/top_level.txt +0 -0
ciocore/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
8.0.0-beta.
|
|
1
|
+
8.0.0-beta.14
|
ciocore/api_client.py
CHANGED
|
@@ -13,7 +13,8 @@ import requests
|
|
|
13
13
|
import socket
|
|
14
14
|
import time
|
|
15
15
|
import sys
|
|
16
|
-
|
|
16
|
+
import platform
|
|
17
|
+
import hashlib
|
|
17
18
|
|
|
18
19
|
from urllib import parse
|
|
19
20
|
|
|
@@ -39,11 +40,33 @@ except AttributeError:
|
|
|
39
40
|
)
|
|
40
41
|
|
|
41
42
|
|
|
43
|
+
def truncate_middle(s, max_length):
|
|
44
|
+
"""
|
|
45
|
+
Truncate the string `s` to `max_length` by removing characters from the middle.
|
|
46
|
+
|
|
47
|
+
:param s: The original string to be truncated.
|
|
48
|
+
:type s: str
|
|
49
|
+
:param max_length: The maximum allowed length of the string after truncation.
|
|
50
|
+
:type max_length: int
|
|
51
|
+
:return: The truncated string.
|
|
52
|
+
:rtype: str
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
if len(s) <= max_length:
|
|
56
|
+
# String is already at or below the maximum length, return it as is
|
|
57
|
+
return s
|
|
58
|
+
|
|
59
|
+
# Calculate the number of characters to keep from the start and end of the string
|
|
60
|
+
num_keep_front = (max_length // 2)
|
|
61
|
+
num_keep_end = max_length - num_keep_front - 1 # -1 for the ellipsis
|
|
62
|
+
|
|
63
|
+
# Construct the truncated string
|
|
64
|
+
return s[:num_keep_front] + '~' + s[-num_keep_end:]
|
|
65
|
+
|
|
42
66
|
|
|
43
67
|
# TODO: appspot_dot_com_cert = os.path.join(common.base_dir(),'auth','appspot_dot_com_cert2') load
|
|
44
68
|
# appspot.com cert into requests lib verify = appspot_dot_com_cert
|
|
45
69
|
|
|
46
|
-
|
|
47
70
|
class ApiClient:
|
|
48
71
|
"""
|
|
49
72
|
The ApiClient class is a wrapper around the requests library that handles authentication and retries.
|
|
@@ -239,96 +262,32 @@ class ApiClient:
|
|
|
239
262
|
return response.text, response.status_code
|
|
240
263
|
|
|
241
264
|
@classmethod
|
|
242
|
-
def
|
|
265
|
+
def register_client(cls, client_name, client_version=None):
|
|
243
266
|
"""
|
|
244
267
|
Generates the http User Agent header that includes helpful debug info.
|
|
245
|
-
|
|
246
|
-
The final component is the path to the python executable (MD5 hex).
|
|
247
|
-
|
|
248
|
-
Example:
|
|
249
|
-
'ciomaya/0.3.7 (ciocore 4.3.2; python 3.9.5; linux 3.10.0-1160.53.1.el7.x86_64; 0ee7123c2365d7a0d126de5a70f19727)'
|
|
250
|
-
|
|
251
|
-
:param client_name: The name of the client to be used in the header. If it's importable it
|
|
252
|
-
will be queried for its __version__ (unless client_version is supplied)
|
|
253
|
-
:type client_name: str
|
|
254
|
-
|
|
255
|
-
:param client_version: The version to use in the header if client_name can't be queried for
|
|
256
|
-
__version__ (or it needs to be overridden)
|
|
257
|
-
:type client_version: str [default = None]
|
|
258
|
-
|
|
259
|
-
:return: The value for the User Agent header
|
|
260
|
-
:rtype: str
|
|
261
268
|
"""
|
|
262
269
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
except ImportError:
|
|
266
|
-
logger.warning(f"Unable to import module '{client_name}'. Won't query for version details.")
|
|
270
|
+
# Use the provided client_version.
|
|
271
|
+
if not client_version:
|
|
267
272
|
client_version = 'unknown'
|
|
268
273
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
except AttributeError:
|
|
273
|
-
try:
|
|
274
|
-
client_version = client_module.version
|
|
275
|
-
except AttributeError:
|
|
276
|
-
logger.warning(f"Module '{client_name}' has no __version__ or version attribute. Setting version to 'N/A' in the user agent")
|
|
277
|
-
client_version = 'unknown'
|
|
274
|
+
python_version = platform.python_version()
|
|
275
|
+
system_info = platform.system()
|
|
276
|
+
release_info = platform.release()
|
|
278
277
|
|
|
279
278
|
|
|
279
|
+
# Get the MD5 hex digest of the path to the python executable
|
|
280
|
+
python_executable_path = truncate_middle(sys.executable.encode('utf-8'), cls.USER_AGENT_MAX_PATH_LENGTH)
|
|
281
|
+
md5_hash = hashlib.md5(python_executable_path).hexdigest()
|
|
280
282
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
if len(python_path) > cls.USER_AGENT_MAX_PATH_LENGTH:
|
|
285
|
-
first_half = int(cls.USER_AGENT_MAX_PATH_LENGTH / 2)
|
|
286
|
-
second_half = len(python_path) - int(cls.USER_AGENT_MAX_PATH_LENGTH / 2)
|
|
287
|
-
|
|
288
|
-
print(first_half, second_half)
|
|
289
|
-
python_path = "{}...{}".format(
|
|
290
|
-
python_path[0:first_half], python_path[second_half:-1]
|
|
291
|
-
)
|
|
292
|
-
|
|
293
|
-
python_path_encoded = base64.b64encode(python_path.encode("utf-8")).decode(
|
|
294
|
-
"utf-8"
|
|
283
|
+
user_agent = (
|
|
284
|
+
f"{client_name}/{client_version} "
|
|
285
|
+
f"(python {python_version}; {system_info} {release_info}; {md5_hash})"
|
|
295
286
|
)
|
|
296
|
-
|
|
297
|
-
if platform.system() == "Linux":
|
|
298
|
-
platform_details = platform.release()
|
|
299
|
-
|
|
300
|
-
elif platform.system() == "Windows":
|
|
301
|
-
platform_details = platform.version()
|
|
302
|
-
|
|
303
|
-
elif platform.system() == "Darwin":
|
|
304
|
-
platform_details = platform.mac_ver()[0]
|
|
305
|
-
|
|
306
|
-
else:
|
|
307
|
-
raise ValueError("Unrecognized platform '{}'".format(platform.release()))
|
|
308
|
-
|
|
309
|
-
pid = base64.b64encode(str(os.getpid()).encode("utf-8")).decode("utf-8")
|
|
310
|
-
hostname = base64.b64encode(socket.gethostname().encode("utf-8")).decode(
|
|
311
|
-
"utf-8"
|
|
312
|
-
)
|
|
313
|
-
|
|
314
|
-
return cls.USER_AGENT_TEMPLATE.format(
|
|
315
|
-
client_name=client_name,
|
|
316
|
-
client_version=client_version,
|
|
317
|
-
ciocore_version=ciocore.version,
|
|
318
|
-
runtime="python",
|
|
319
|
-
runtime_version=platform.python_version(),
|
|
320
|
-
platform=sys.platform,
|
|
321
|
-
platform_details=platform_details,
|
|
322
|
-
pid=pid,
|
|
323
|
-
hostname=hostname,
|
|
324
|
-
python_path=python_path_encoded,
|
|
325
|
-
)
|
|
326
|
-
|
|
327
|
-
@classmethod
|
|
328
|
-
def register_client(cls, client_name, client_version=None):
|
|
329
|
-
cls.user_agent_header = cls._get_user_agent_header(client_name, client_version)
|
|
330
|
-
|
|
287
|
+
cls.user_agent_header = user_agent
|
|
331
288
|
|
|
289
|
+
return user_agent
|
|
290
|
+
|
|
332
291
|
|
|
333
292
|
def read_conductor_credentials(use_api_key=False):
|
|
334
293
|
"""
|
ciocore/cli.py
CHANGED
|
@@ -97,14 +97,12 @@ LOG_FORMATTER = logging.Formatter(
|
|
|
97
97
|
"%(asctime)s %(name)s%(levelname)9s %(threadName)s: %(message)s"
|
|
98
98
|
)
|
|
99
99
|
|
|
100
|
-
|
|
101
100
|
cfg = config.get()
|
|
102
101
|
DEFAULT_CONFIG_MD5_CACHING = cfg["md5_caching"]
|
|
103
102
|
DEFAULT_CONFIG_THREAD_COUNT = cfg["thread_count"]
|
|
104
103
|
DEFAULT_CONFIG_LOG_LEVEL = cfg["log_level"]
|
|
105
104
|
DEFAULT_DOWNLOADER_PAGE_SIZE = cfg["downloader_page_size"]
|
|
106
105
|
|
|
107
|
-
|
|
108
106
|
def _set_logging(log_level, log_to_console=True, log_dir=None, log_filename=None):
|
|
109
107
|
level = loggeria.LEVEL_MAP.get(log_level)
|
|
110
108
|
loggeria.setup_conductor_logging(
|
|
@@ -115,13 +113,11 @@ def _set_logging(log_level, log_to_console=True, log_dir=None, log_filename=None
|
|
|
115
113
|
use_system_log=False,
|
|
116
114
|
)
|
|
117
115
|
|
|
118
|
-
|
|
119
116
|
def _register(client):
|
|
120
117
|
api_client.ApiClient.register_client(
|
|
121
118
|
client_name=client.CLIENT_NAME, client_version=VERSION
|
|
122
119
|
)
|
|
123
120
|
|
|
124
|
-
|
|
125
121
|
########################### MAIN #################################
|
|
126
122
|
@click.group(invoke_without_command=True)
|
|
127
123
|
@click.pass_context
|
|
@@ -135,8 +131,7 @@ def main(ctx, version):
|
|
|
135
131
|
click.echo(ctx.get_help())
|
|
136
132
|
ctx.exit()
|
|
137
133
|
|
|
138
|
-
|
|
139
|
-
############################# UPLOADER #############################
|
|
134
|
+
############################# UPLOAD #############################
|
|
140
135
|
@main.command()
|
|
141
136
|
@click.option("-db", "--database_filepath", help=UPLOADER_DATABASE_FILEPATH_HELP)
|
|
142
137
|
@click.option(
|
|
@@ -155,6 +150,7 @@ def main(ctx, version):
|
|
|
155
150
|
default=DEFAULT_CONFIG_LOG_LEVEL,
|
|
156
151
|
)
|
|
157
152
|
@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
|
|
153
|
+
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=True)
|
|
158
154
|
@click.option(
|
|
159
155
|
"-tc",
|
|
160
156
|
"--thread_count",
|
|
@@ -165,7 +161,7 @@ def main(ctx, version):
|
|
|
165
161
|
@click.option("-lc", "--location", help=LOCATION_HELP)
|
|
166
162
|
@click.argument("paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
|
|
167
163
|
def upload(
|
|
168
|
-
database_filepath, location, md5_caching, log_level, log_dir, thread_count, paths
|
|
164
|
+
database_filepath, location, md5_caching, log_level, log_to_console, log_dir, thread_count, paths
|
|
169
165
|
):
|
|
170
166
|
"""Upload files to Conductor.
|
|
171
167
|
|
|
@@ -178,7 +174,7 @@ def upload(
|
|
|
178
174
|
conductor upload file1 file2 file3
|
|
179
175
|
"""
|
|
180
176
|
logfile = log_dir and os.path.join(log_dir, "conductor_ul.log")
|
|
181
|
-
_set_logging(log_level,
|
|
177
|
+
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir)
|
|
182
178
|
|
|
183
179
|
args_dict = {
|
|
184
180
|
"database_filepath": database_filepath,
|
|
@@ -272,6 +268,7 @@ DEPRECATED_PATHS_HELP = "Specify a list of paths to upload."
|
|
|
272
268
|
help=THREADS_HELP,
|
|
273
269
|
default=DEFAULT_CONFIG_THREAD_COUNT,
|
|
274
270
|
)
|
|
271
|
+
@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=True)
|
|
275
272
|
@click.option("-lc", "--location", help=LOCATION_HELP)
|
|
276
273
|
@click.option(
|
|
277
274
|
"-p",
|
|
@@ -293,7 +290,7 @@ def uploader(
|
|
|
293
290
|
Example:
|
|
294
291
|
conductor upload file1 file2 file3
|
|
295
292
|
"""
|
|
296
|
-
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir
|
|
293
|
+
_set_logging(log_level, log_to_console=log_to_console, log_dir=log_dir)
|
|
297
294
|
|
|
298
295
|
args_dict = {
|
|
299
296
|
"database_filepath": database_filepath,
|
|
@@ -418,7 +415,6 @@ def downloader(
|
|
|
418
415
|
default=DEFAULT_CONFIG_THREAD_COUNT,
|
|
419
416
|
)
|
|
420
417
|
@click.option("-r", "--regex", help=DOWNLOAD_REGEX_HELP)
|
|
421
|
-
@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
|
|
422
418
|
@click.option("-lc", "--location", help=LOCATION_HELP)
|
|
423
419
|
@click.argument("jobids", nargs=-1)
|
|
424
420
|
def download(
|
|
@@ -428,7 +424,6 @@ def download(
|
|
|
428
424
|
location,
|
|
429
425
|
progress_interval,
|
|
430
426
|
log_level,
|
|
431
|
-
log_dir,
|
|
432
427
|
thread_count,
|
|
433
428
|
force,
|
|
434
429
|
regex,
|
|
@@ -454,9 +449,13 @@ def download(
|
|
|
454
449
|
conductor download 1234:1-5 1235:5-10
|
|
455
450
|
"""
|
|
456
451
|
|
|
457
|
-
|
|
458
|
-
_set_logging(
|
|
452
|
+
|
|
453
|
+
_set_logging(
|
|
454
|
+
log_level=log_level,
|
|
455
|
+
log_to_console=True)
|
|
456
|
+
|
|
459
457
|
_register(LoggingDownloadRunner)
|
|
458
|
+
|
|
460
459
|
kwargs = {
|
|
461
460
|
"destination": destination,
|
|
462
461
|
"num_threads": thread_count,
|
|
@@ -473,7 +472,6 @@ def download(
|
|
|
473
472
|
)
|
|
474
473
|
|
|
475
474
|
dl.run()
|
|
476
|
-
|
|
477
475
|
|
|
478
476
|
|
|
479
477
|
########################### SEQUENCE CHECKER #############################
|
ciocore/config.py
CHANGED
|
@@ -10,6 +10,7 @@ import os
|
|
|
10
10
|
import base64
|
|
11
11
|
import json
|
|
12
12
|
import re
|
|
13
|
+
import platform
|
|
13
14
|
|
|
14
15
|
from ciocore.common import CONDUCTOR_LOGGER_NAME
|
|
15
16
|
|
|
@@ -17,6 +18,15 @@ logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
|
|
|
17
18
|
|
|
18
19
|
# https://stackoverflow.com/a/3809435/179412
|
|
19
20
|
|
|
21
|
+
|
|
22
|
+
USER_DIRS = {
|
|
23
|
+
"Linux": os.path.expanduser(os.path.join("~", ".conductor")),
|
|
24
|
+
"Darwin": os.path.expanduser(os.path.join("~",".conductor")),
|
|
25
|
+
"Windows": os.path.expanduser(os.path.join("~", "AppData", "Local", "Conductor")),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
DEFAULT_USER_DIR = USER_DIRS.get(platform.system(), USER_DIRS["Linux"])
|
|
29
|
+
|
|
20
30
|
URL_REGEX = re.compile(
|
|
21
31
|
r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"
|
|
22
32
|
)
|
|
@@ -104,6 +114,8 @@ class Config(object):
|
|
|
104
114
|
"""
|
|
105
115
|
default_downloader_page_size = 50
|
|
106
116
|
|
|
117
|
+
|
|
118
|
+
|
|
107
119
|
try:
|
|
108
120
|
default_thread_count = min( os.cpu_count() - 1, 15)
|
|
109
121
|
except NotImplementedError:
|
|
@@ -145,7 +157,7 @@ class Config(object):
|
|
|
145
157
|
"auth_url": url,
|
|
146
158
|
"api_url": api_url,
|
|
147
159
|
"api_key": self.get_api_key_from_variable() or self.get_api_key_from_file(),
|
|
148
|
-
"user_dir": os.environ.get('CONDUCTOR_USER_DIR',
|
|
160
|
+
"user_dir": os.environ.get('CONDUCTOR_USER_DIR', DEFAULT_USER_DIR)
|
|
149
161
|
}
|
|
150
162
|
|
|
151
163
|
@staticmethod
|
|
@@ -585,11 +585,25 @@
|
|
|
585
585
|
make_request()
|
|
586
586
|
</a>
|
|
587
587
|
|
|
588
|
+
</li>
|
|
589
|
+
|
|
590
|
+
<li class="md-nav__item">
|
|
591
|
+
<a href="#ciocore.api_client.ApiClient.register_client" class="md-nav__link">
|
|
592
|
+
register_client()
|
|
593
|
+
</a>
|
|
594
|
+
|
|
588
595
|
</li>
|
|
589
596
|
|
|
590
597
|
</ul>
|
|
591
598
|
</nav>
|
|
592
599
|
|
|
600
|
+
</li>
|
|
601
|
+
|
|
602
|
+
<li class="md-nav__item">
|
|
603
|
+
<a href="#ciocore.api_client.truncate_middle" class="md-nav__link">
|
|
604
|
+
truncate_middle()
|
|
605
|
+
</a>
|
|
606
|
+
|
|
593
607
|
</li>
|
|
594
608
|
|
|
595
609
|
<li class="md-nav__item">
|
|
@@ -878,11 +892,25 @@
|
|
|
878
892
|
make_request()
|
|
879
893
|
</a>
|
|
880
894
|
|
|
895
|
+
</li>
|
|
896
|
+
|
|
897
|
+
<li class="md-nav__item">
|
|
898
|
+
<a href="#ciocore.api_client.ApiClient.register_client" class="md-nav__link">
|
|
899
|
+
register_client()
|
|
900
|
+
</a>
|
|
901
|
+
|
|
881
902
|
</li>
|
|
882
903
|
|
|
883
904
|
</ul>
|
|
884
905
|
</nav>
|
|
885
906
|
|
|
907
|
+
</li>
|
|
908
|
+
|
|
909
|
+
<li class="md-nav__item">
|
|
910
|
+
<a href="#ciocore.api_client.truncate_middle" class="md-nav__link">
|
|
911
|
+
truncate_middle()
|
|
912
|
+
</a>
|
|
913
|
+
|
|
886
914
|
</li>
|
|
887
915
|
|
|
888
916
|
<li class="md-nav__item">
|
|
@@ -1232,94 +1260,31 @@ ApiClient </code>
|
|
|
1232
1260
|
<span class="k">return</span> <span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span>
|
|
1233
1261
|
|
|
1234
1262
|
<span class="nd">@classmethod</span>
|
|
1235
|
-
<span class="k">def</span> <span class="nf">
|
|
1263
|
+
<span class="k">def</span> <span class="nf">register_client</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">client_name</span><span class="p">,</span> <span class="n">client_version</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
1236
1264
|
<span class="w"> </span><span class="sd">"""</span>
|
|
1237
1265
|
<span class="sd"> Generates the http User Agent header that includes helpful debug info.</span>
|
|
1238
|
-
|
|
1239
|
-
<span class="sd"> The final component is the path to the python executable (MD5 hex).</span>
|
|
1240
|
-
|
|
1241
|
-
<span class="sd"> Example:</span>
|
|
1242
|
-
<span class="sd"> 'ciomaya/0.3.7 (ciocore 4.3.2; python 3.9.5; linux 3.10.0-1160.53.1.el7.x86_64; 0ee7123c2365d7a0d126de5a70f19727)'</span>
|
|
1243
|
-
|
|
1244
|
-
<span class="sd"> :param client_name: The name of the client to be used in the header. If it's importable it</span>
|
|
1245
|
-
<span class="sd"> will be queried for its __version__ (unless client_version is supplied)</span>
|
|
1246
|
-
<span class="sd"> :type client_name: str</span>
|
|
1247
|
-
|
|
1248
|
-
<span class="sd"> :param client_version: The version to use in the header if client_name can't be queried for</span>
|
|
1249
|
-
<span class="sd"> __version__ (or it needs to be overridden)</span>
|
|
1250
|
-
<span class="sd"> :type client_version: str [default = None]</span>
|
|
1251
|
-
|
|
1252
|
-
<span class="sd"> :return: The value for the User Agent header</span>
|
|
1253
|
-
<span class="sd"> :rtype: str</span>
|
|
1254
1266
|
<span class="sd"> """</span>
|
|
1255
1267
|
|
|
1256
|
-
<span class="
|
|
1257
|
-
<span class="n">client_module</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">client_name</span><span class="p">)</span>
|
|
1258
|
-
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
|
1259
|
-
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unable to import module '</span><span class="si">{</span><span class="n">client_name</span><span class="si">}</span><span class="s2">'. Won't query for version details."</span><span class="p">)</span>
|
|
1260
|
-
<span class="n">client_version</span> <span class="o">=</span> <span class="s1">'unknown'</span>
|
|
1261
|
-
|
|
1268
|
+
<span class="c1"># Use the provided client_version.</span>
|
|
1262
1269
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">client_version</span><span class="p">:</span>
|
|
1263
|
-
<span class="
|
|
1264
|
-
<span class="n">client_version</span> <span class="o">=</span> <span class="n">client_module</span><span class="o">.</span><span class="n">__version__</span>
|
|
1265
|
-
<span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
|
|
1266
|
-
<span class="k">try</span><span class="p">:</span>
|
|
1267
|
-
<span class="n">client_version</span> <span class="o">=</span> <span class="n">client_module</span><span class="o">.</span><span class="n">version</span>
|
|
1268
|
-
<span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
|
|
1269
|
-
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Module '</span><span class="si">{</span><span class="n">client_name</span><span class="si">}</span><span class="s2">' has no __version__ or version attribute. Setting version to 'N/A' in the user agent"</span><span class="p">)</span>
|
|
1270
|
-
<span class="n">client_version</span> <span class="o">=</span> <span class="s1">'unknown'</span>
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
<span class="n">python_path</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">executable</span>
|
|
1275
|
-
|
|
1276
|
-
<span class="c1"># If the length of the path is longer than allowed, truncate the middle</span>
|
|
1277
|
-
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">python_path</span><span class="p">)</span> <span class="o">></span> <span class="bp">cls</span><span class="o">.</span><span class="n">USER_AGENT_MAX_PATH_LENGTH</span><span class="p">:</span>
|
|
1278
|
-
<span class="n">first_half</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">USER_AGENT_MAX_PATH_LENGTH</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span>
|
|
1279
|
-
<span class="n">second_half</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">python_path</span><span class="p">)</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">USER_AGENT_MAX_PATH_LENGTH</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span>
|
|
1280
|
-
|
|
1281
|
-
<span class="nb">print</span><span class="p">(</span><span class="n">first_half</span><span class="p">,</span> <span class="n">second_half</span><span class="p">)</span>
|
|
1282
|
-
<span class="n">python_path</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{}</span><span class="s2">...</span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
|
1283
|
-
<span class="n">python_path</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">first_half</span><span class="p">],</span> <span class="n">python_path</span><span class="p">[</span><span class="n">second_half</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
|
1284
|
-
<span class="p">)</span>
|
|
1285
|
-
|
|
1286
|
-
<span class="n">python_path_encoded</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">python_path</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span>
|
|
1287
|
-
<span class="s2">"utf-8"</span>
|
|
1288
|
-
<span class="p">)</span>
|
|
1289
|
-
|
|
1290
|
-
<span class="k">if</span> <span class="n">platform</span><span class="o">.</span><span class="n">system</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"Linux"</span><span class="p">:</span>
|
|
1291
|
-
<span class="n">platform_details</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
|
|
1292
|
-
|
|
1293
|
-
<span class="k">elif</span> <span class="n">platform</span><span class="o">.</span><span class="n">system</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"Windows"</span><span class="p">:</span>
|
|
1294
|
-
<span class="n">platform_details</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">version</span><span class="p">()</span>
|
|
1270
|
+
<span class="n">client_version</span> <span class="o">=</span> <span class="s1">'unknown'</span>
|
|
1295
1271
|
|
|
1296
|
-
<span class="
|
|
1297
|
-
|
|
1272
|
+
<span class="n">python_version</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
|
|
1273
|
+
<span class="n">system_info</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">system</span><span class="p">()</span>
|
|
1274
|
+
<span class="n">release_info</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
|
|
1298
1275
|
|
|
1299
|
-
<span class="k">else</span><span class="p">:</span>
|
|
1300
|
-
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unrecognized platform '</span><span class="si">{}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">()))</span>
|
|
1301
1276
|
|
|
1302
|
-
<span class="
|
|
1303
|
-
<span class="n">
|
|
1304
|
-
|
|
1305
|
-
<span class="p">)</span>
|
|
1277
|
+
<span class="c1"># Get the MD5 hex digest of the path to the python executable</span>
|
|
1278
|
+
<span class="n">python_executable_path</span> <span class="o">=</span> <span class="n">truncate_middle</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">executable</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="bp">cls</span><span class="o">.</span><span class="n">USER_AGENT_MAX_PATH_LENGTH</span><span class="p">)</span>
|
|
1279
|
+
<span class="n">md5_hash</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">md5</span><span class="p">(</span><span class="n">python_executable_path</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
|
1306
1280
|
|
|
1307
|
-
<span class="
|
|
1308
|
-
<span class="n">client_name</span><span class="
|
|
1309
|
-
<span class="n">
|
|
1310
|
-
<span class="n">ciocore_version</span><span class="o">=</span><span class="n">ciocore</span><span class="o">.</span><span class="n">version</span><span class="p">,</span>
|
|
1311
|
-
<span class="n">runtime</span><span class="o">=</span><span class="s2">"python"</span><span class="p">,</span>
|
|
1312
|
-
<span class="n">runtime_version</span><span class="o">=</span><span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">(),</span>
|
|
1313
|
-
<span class="n">platform</span><span class="o">=</span><span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="p">,</span>
|
|
1314
|
-
<span class="n">platform_details</span><span class="o">=</span><span class="n">platform_details</span><span class="p">,</span>
|
|
1315
|
-
<span class="n">pid</span><span class="o">=</span><span class="n">pid</span><span class="p">,</span>
|
|
1316
|
-
<span class="n">hostname</span><span class="o">=</span><span class="n">hostname</span><span class="p">,</span>
|
|
1317
|
-
<span class="n">python_path</span><span class="o">=</span><span class="n">python_path_encoded</span><span class="p">,</span>
|
|
1281
|
+
<span class="n">user_agent</span> <span class="o">=</span> <span class="p">(</span>
|
|
1282
|
+
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">client_name</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">client_version</span><span class="si">}</span><span class="s2"> "</span>
|
|
1283
|
+
<span class="sa">f</span><span class="s2">"(python </span><span class="si">{</span><span class="n">python_version</span><span class="si">}</span><span class="s2">; </span><span class="si">{</span><span class="n">system_info</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">release_info</span><span class="si">}</span><span class="s2">; </span><span class="si">{</span><span class="n">md5_hash</span><span class="si">}</span><span class="s2">)"</span>
|
|
1318
1284
|
<span class="p">)</span>
|
|
1285
|
+
<span class="bp">cls</span><span class="o">.</span><span class="n">user_agent_header</span> <span class="o">=</span> <span class="n">user_agent</span>
|
|
1319
1286
|
|
|
1320
|
-
|
|
1321
|
-
<span class="k">def</span> <span class="nf">register_client</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">client_name</span><span class="p">,</span> <span class="n">client_version</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
1322
|
-
<span class="bp">cls</span><span class="o">.</span><span class="n">user_agent_header</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_get_user_agent_header</span><span class="p">(</span><span class="n">client_name</span><span class="p">,</span> <span class="n">client_version</span><span class="p">)</span>
|
|
1287
|
+
<span class="k">return</span> <span class="n">user_agent</span>
|
|
1323
1288
|
</code></pre></div>
|
|
1324
1289
|
</details>
|
|
1325
1290
|
|
|
@@ -1715,6 +1680,58 @@ client is built to use Requests 3.x this function can be removed.</p>
|
|
|
1715
1680
|
|
|
1716
1681
|
|
|
1717
1682
|
|
|
1683
|
+
<div class="doc doc-object doc-method">
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
<h3 id="ciocore.api_client.ApiClient.register_client" class="doc doc-heading">
|
|
1688
|
+
<code class="highlight language-python"><span class="n">register_client</span><span class="p">(</span><span class="n">client_name</span><span class="p">,</span> <span class="n">client_version</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></code>
|
|
1689
|
+
|
|
1690
|
+
<span class="doc doc-properties">
|
|
1691
|
+
<small class="doc doc-property doc-property-classmethod"><code>classmethod</code></small>
|
|
1692
|
+
</span>
|
|
1693
|
+
|
|
1694
|
+
</h3>
|
|
1695
|
+
|
|
1696
|
+
<div class="doc doc-contents ">
|
|
1697
|
+
|
|
1698
|
+
<p>Generates the http User Agent header that includes helpful debug info.</p>
|
|
1699
|
+
|
|
1700
|
+
<details class="quote">
|
|
1701
|
+
<summary>Source code in <code>ciocore/api_client.py</code></summary>
|
|
1702
|
+
<div class="highlight"><pre><span></span><code><span class="nd">@classmethod</span>
|
|
1703
|
+
<span class="k">def</span> <span class="nf">register_client</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">client_name</span><span class="p">,</span> <span class="n">client_version</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
1704
|
+
<span class="w"> </span><span class="sd">"""</span>
|
|
1705
|
+
<span class="sd"> Generates the http User Agent header that includes helpful debug info.</span>
|
|
1706
|
+
<span class="sd"> """</span>
|
|
1707
|
+
|
|
1708
|
+
<span class="c1"># Use the provided client_version.</span>
|
|
1709
|
+
<span class="k">if</span> <span class="ow">not</span> <span class="n">client_version</span><span class="p">:</span>
|
|
1710
|
+
<span class="n">client_version</span> <span class="o">=</span> <span class="s1">'unknown'</span>
|
|
1711
|
+
|
|
1712
|
+
<span class="n">python_version</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">python_version</span><span class="p">()</span>
|
|
1713
|
+
<span class="n">system_info</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">system</span><span class="p">()</span>
|
|
1714
|
+
<span class="n">release_info</span> <span class="o">=</span> <span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
|
|
1715
|
+
|
|
1716
|
+
|
|
1717
|
+
<span class="c1"># Get the MD5 hex digest of the path to the python executable</span>
|
|
1718
|
+
<span class="n">python_executable_path</span> <span class="o">=</span> <span class="n">truncate_middle</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">executable</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="bp">cls</span><span class="o">.</span><span class="n">USER_AGENT_MAX_PATH_LENGTH</span><span class="p">)</span>
|
|
1719
|
+
<span class="n">md5_hash</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">md5</span><span class="p">(</span><span class="n">python_executable_path</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
|
1720
|
+
|
|
1721
|
+
<span class="n">user_agent</span> <span class="o">=</span> <span class="p">(</span>
|
|
1722
|
+
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">client_name</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">client_version</span><span class="si">}</span><span class="s2"> "</span>
|
|
1723
|
+
<span class="sa">f</span><span class="s2">"(python </span><span class="si">{</span><span class="n">python_version</span><span class="si">}</span><span class="s2">; </span><span class="si">{</span><span class="n">system_info</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">release_info</span><span class="si">}</span><span class="s2">; </span><span class="si">{</span><span class="n">md5_hash</span><span class="si">}</span><span class="s2">)"</span>
|
|
1724
|
+
<span class="p">)</span>
|
|
1725
|
+
<span class="bp">cls</span><span class="o">.</span><span class="n">user_agent_header</span> <span class="o">=</span> <span class="n">user_agent</span>
|
|
1726
|
+
|
|
1727
|
+
<span class="k">return</span> <span class="n">user_agent</span>
|
|
1728
|
+
</code></pre></div>
|
|
1729
|
+
</details>
|
|
1730
|
+
</div>
|
|
1731
|
+
|
|
1732
|
+
</div>
|
|
1733
|
+
|
|
1734
|
+
|
|
1718
1735
|
|
|
1719
1736
|
|
|
1720
1737
|
|
|
@@ -1727,6 +1744,58 @@ client is built to use Requests 3.x this function can be removed.</p>
|
|
|
1727
1744
|
|
|
1728
1745
|
|
|
1729
1746
|
|
|
1747
|
+
<div class="doc doc-object doc-function">
|
|
1748
|
+
|
|
1749
|
+
|
|
1750
|
+
|
|
1751
|
+
<h2 id="ciocore.api_client.truncate_middle" class="doc doc-heading">
|
|
1752
|
+
<code class="highlight language-python"><span class="n">truncate_middle</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">max_length</span><span class="p">)</span></code>
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
</h2>
|
|
1756
|
+
|
|
1757
|
+
<div class="doc doc-contents ">
|
|
1758
|
+
|
|
1759
|
+
<p>Truncate the string <code>s</code> to <code>max_length</code> by removing characters from the middle.</p>
|
|
1760
|
+
<p>:param s: The original string to be truncated.
|
|
1761
|
+
:type s: str
|
|
1762
|
+
:param max_length: The maximum allowed length of the string after truncation.
|
|
1763
|
+
:type max_length: int
|
|
1764
|
+
:return: The truncated string.
|
|
1765
|
+
:rtype: str</p>
|
|
1766
|
+
|
|
1767
|
+
<details class="quote">
|
|
1768
|
+
<summary>Source code in <code>ciocore/api_client.py</code></summary>
|
|
1769
|
+
<div class="highlight"><pre><span></span><code><span class="k">def</span> <span class="nf">truncate_middle</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">max_length</span><span class="p">):</span>
|
|
1770
|
+
<span class="w"> </span><span class="sd">"""</span>
|
|
1771
|
+
<span class="sd"> Truncate the string `s` to `max_length` by removing characters from the middle.</span>
|
|
1772
|
+
|
|
1773
|
+
<span class="sd"> :param s: The original string to be truncated.</span>
|
|
1774
|
+
<span class="sd"> :type s: str</span>
|
|
1775
|
+
<span class="sd"> :param max_length: The maximum allowed length of the string after truncation.</span>
|
|
1776
|
+
<span class="sd"> :type max_length: int</span>
|
|
1777
|
+
<span class="sd"> :return: The truncated string.</span>
|
|
1778
|
+
<span class="sd"> :rtype: str</span>
|
|
1779
|
+
<span class="sd"> """</span>
|
|
1780
|
+
|
|
1781
|
+
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o"><=</span> <span class="n">max_length</span><span class="p">:</span>
|
|
1782
|
+
<span class="c1"># String is already at or below the maximum length, return it as is</span>
|
|
1783
|
+
<span class="k">return</span> <span class="n">s</span>
|
|
1784
|
+
|
|
1785
|
+
<span class="c1"># Calculate the number of characters to keep from the start and end of the string</span>
|
|
1786
|
+
<span class="n">num_keep_front</span> <span class="o">=</span> <span class="p">(</span><span class="n">max_length</span> <span class="o">//</span> <span class="mi">2</span><span class="p">)</span>
|
|
1787
|
+
<span class="n">num_keep_end</span> <span class="o">=</span> <span class="n">max_length</span> <span class="o">-</span> <span class="n">num_keep_front</span> <span class="o">-</span> <span class="mi">1</span> <span class="c1"># -1 for the ellipsis</span>
|
|
1788
|
+
|
|
1789
|
+
<span class="c1"># Construct the truncated string</span>
|
|
1790
|
+
<span class="k">return</span> <span class="n">s</span><span class="p">[:</span><span class="n">num_keep_front</span><span class="p">]</span> <span class="o">+</span> <span class="s1">'~'</span> <span class="o">+</span> <span class="n">s</span><span class="p">[</span><span class="o">-</span><span class="n">num_keep_end</span><span class="p">:]</span>
|
|
1791
|
+
</code></pre></div>
|
|
1792
|
+
</details>
|
|
1793
|
+
</div>
|
|
1794
|
+
|
|
1795
|
+
</div>
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
|
|
1730
1799
|
<div class="doc doc-object doc-function">
|
|
1731
1800
|
|
|
1732
1801
|
|