PyS3Uploader 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of PyS3Uploader might be problematic. Click here for more details.
- {pys3uploader-0.2.1.dist-info → pys3uploader-0.2.2.dist-info}/METADATA +1 -1
- pys3uploader-0.2.2.dist-info/RECORD +11 -0
- s3/__init__.py +1 -1
- s3/uploader.py +2 -1
- s3/utils.py +75 -12
- pys3uploader-0.2.1.dist-info/RECORD +0 -11
- {pys3uploader-0.2.1.dist-info → pys3uploader-0.2.2.dist-info}/LICENSE +0 -0
- {pys3uploader-0.2.1.dist-info → pys3uploader-0.2.2.dist-info}/WHEEL +0 -0
- {pys3uploader-0.2.1.dist-info → pys3uploader-0.2.2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
s3/__init__.py,sha256=L12aFBb0plj8WISe0_He1vvQ55aOKd8VCyMlj_2LqrQ,66
|
|
2
|
+
s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
|
|
3
|
+
s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
|
|
4
|
+
s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
|
|
5
|
+
s3/uploader.py,sha256=I2An6Ix0rFMlvDLtLaDQ6F-YrN70IDCNFgh9E32cXHA,11489
|
|
6
|
+
s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
|
|
7
|
+
pys3uploader-0.2.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
|
|
8
|
+
pys3uploader-0.2.2.dist-info/METADATA,sha256=fIlyxO6dFHYH3uhFg4yZa7RtGGBKqaOynAHlrbuffoY,7449
|
|
9
|
+
pys3uploader-0.2.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
10
|
+
pys3uploader-0.2.2.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
|
|
11
|
+
pys3uploader-0.2.2.dist-info/RECORD,,
|
s3/__init__.py
CHANGED
s3/uploader.py
CHANGED
|
@@ -14,6 +14,7 @@ from s3.logger import default_logger
|
|
|
14
14
|
from s3.utils import (
|
|
15
15
|
RETRY_CONFIG,
|
|
16
16
|
UploadResults,
|
|
17
|
+
convert_seconds,
|
|
17
18
|
convert_to_folder_structure,
|
|
18
19
|
getenv,
|
|
19
20
|
urljoin,
|
|
@@ -134,7 +135,7 @@ class Uploader:
|
|
|
134
135
|
self.logger.info(
|
|
135
136
|
"Total number of uploads: %d, success: %d, failed: %d", total, self.results.success, self.results.failed
|
|
136
137
|
)
|
|
137
|
-
self.logger.info("Run
|
|
138
|
+
self.logger.info("Run time: %s", convert_seconds(time.time() - self.start))
|
|
138
139
|
|
|
139
140
|
def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
|
|
140
141
|
"""Compares file size if the object already exists in S3.
|
s3/utils.py
CHANGED
|
@@ -3,18 +3,6 @@ from typing import Dict, Set
|
|
|
3
3
|
|
|
4
4
|
from botocore.config import Config
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
class UploadResults(dict):
|
|
8
|
-
"""Object to store results of S3 upload.
|
|
9
|
-
|
|
10
|
-
>>> UploadResults
|
|
11
|
-
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
success: int = 0
|
|
15
|
-
failed: int = 0
|
|
16
|
-
|
|
17
|
-
|
|
18
6
|
RETRY_CONFIG: Config = Config(
|
|
19
7
|
retries={
|
|
20
8
|
"max_attempts": 10,
|
|
@@ -27,6 +15,17 @@ RETRY_CONFIG: Config = Config(
|
|
|
27
15
|
)
|
|
28
16
|
|
|
29
17
|
|
|
18
|
+
class UploadResults(dict):
|
|
19
|
+
"""Object to store results of S3 upload.
|
|
20
|
+
|
|
21
|
+
>>> UploadResults
|
|
22
|
+
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
success: int = 0
|
|
26
|
+
failed: int = 0
|
|
27
|
+
|
|
28
|
+
|
|
30
29
|
def getenv(*args, default: str = None) -> str:
|
|
31
30
|
"""Returns the key-ed environment variable or the default value."""
|
|
32
31
|
for key in args:
|
|
@@ -82,3 +81,67 @@ def convert_to_folder_structure(sequence: Set[str]) -> str:
|
|
|
82
81
|
return result
|
|
83
82
|
|
|
84
83
|
return generate_folder_structure(folder_structure)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def convert_seconds(seconds: int | float, n_elem: int = 2) -> str:
|
|
87
|
+
"""Calculate years, months, days, hours, minutes, seconds, and milliseconds from given input.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
seconds: Number of seconds to convert (supports float values).
|
|
91
|
+
n_elem: Number of elements required from the converted list.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
str:
|
|
95
|
+
Returns a humanized string notion of the number of seconds.
|
|
96
|
+
"""
|
|
97
|
+
if not seconds:
|
|
98
|
+
return "0s"
|
|
99
|
+
elif seconds < 1:
|
|
100
|
+
return f"{seconds * 1000:.0f}ms"
|
|
101
|
+
|
|
102
|
+
seconds_in_year = 365 * 24 * 3600
|
|
103
|
+
seconds_in_month = 30 * 24 * 3600
|
|
104
|
+
|
|
105
|
+
years = seconds // seconds_in_year
|
|
106
|
+
seconds %= seconds_in_year
|
|
107
|
+
|
|
108
|
+
months = seconds // seconds_in_month
|
|
109
|
+
seconds %= seconds_in_month
|
|
110
|
+
|
|
111
|
+
days = seconds // (24 * 3600)
|
|
112
|
+
seconds %= 24 * 3600
|
|
113
|
+
|
|
114
|
+
hours = seconds // 3600
|
|
115
|
+
seconds %= 3600
|
|
116
|
+
|
|
117
|
+
minutes = seconds // 60
|
|
118
|
+
seconds %= 60
|
|
119
|
+
|
|
120
|
+
milliseconds = round((seconds % 1) * 1000)
|
|
121
|
+
seconds = int(seconds) # Convert remaining seconds to int for display
|
|
122
|
+
|
|
123
|
+
time_parts = []
|
|
124
|
+
|
|
125
|
+
if years > 0:
|
|
126
|
+
time_parts.append(f"{int(years)} year{'s' if years > 1 else ''}")
|
|
127
|
+
if months > 0:
|
|
128
|
+
time_parts.append(f"{int(months)} month{'s' if months > 1 else ''}")
|
|
129
|
+
if days > 0:
|
|
130
|
+
time_parts.append(f"{int(days)} day{'s' if days > 1 else ''}")
|
|
131
|
+
if hours > 0:
|
|
132
|
+
time_parts.append(f"{int(hours)} hour{'s' if hours > 1 else ''}")
|
|
133
|
+
if minutes > 0:
|
|
134
|
+
time_parts.append(f"{int(minutes)} minute{'s' if minutes > 1 else ''}")
|
|
135
|
+
if seconds > 0 or milliseconds > 0:
|
|
136
|
+
if seconds > 0 and milliseconds > 0:
|
|
137
|
+
time_parts.append(f"{seconds + milliseconds / 1000:.1f}s")
|
|
138
|
+
elif seconds > 0:
|
|
139
|
+
time_parts.append(f"{seconds}s")
|
|
140
|
+
else:
|
|
141
|
+
time_parts.append(f"{milliseconds}ms")
|
|
142
|
+
|
|
143
|
+
if len(time_parts) == 1:
|
|
144
|
+
return time_parts[0]
|
|
145
|
+
|
|
146
|
+
list_ = time_parts[:n_elem]
|
|
147
|
+
return ", and ".join([", ".join(list_[:-1]), list_[-1]] if len(list_) > 2 else list_)
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
s3/__init__.py,sha256=IqcPR9iWMw0GDBEmKvLzW7P-AhInTkwRklkvYgiT1Xc,66
|
|
2
|
-
s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
|
|
3
|
-
s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
|
|
4
|
-
s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
|
|
5
|
-
s3/uploader.py,sha256=v6TGdm8EcAldoIw3GOAwkuoUzi9b9IRj8b94hl1Pkyw,11454
|
|
6
|
-
s3/utils.py,sha256=dd1OeLbswLzFVyjYiXixkJlFsoGWRtRCOHha6wLG5zQ,2485
|
|
7
|
-
pys3uploader-0.2.1.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
|
|
8
|
-
pys3uploader-0.2.1.dist-info/METADATA,sha256=NA7x6YqpWEKvn3XEYzVug7XTl1vPcwiphUzWtMvlzHE,7449
|
|
9
|
-
pys3uploader-0.2.1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
10
|
-
pys3uploader-0.2.1.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
|
|
11
|
-
pys3uploader-0.2.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|