bluer-objects 6.223.1__py3-none-any.whl → 6.231.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of bluer-objects might be problematic. Click here for more details.
- bluer_objects/README/functions.py +107 -218
- bluer_objects/README/utils.py +235 -0
- bluer_objects/__init__.py +1 -1
- bluer_objects/storage/WebDAV.py +3 -0
- bluer_objects/storage/WebDAVrequest.py +4 -0
- bluer_objects/storage/WebDAVzip.py +5 -2
- bluer_objects/storage/__init__.py +3 -0
- bluer_objects/storage/base.py +8 -1
- bluer_objects/storage/policies.py +7 -0
- bluer_objects/storage/s3.py +9 -0
- {bluer_objects-6.223.1.dist-info → bluer_objects-6.231.1.dist-info}/METADATA +2 -2
- {bluer_objects-6.223.1.dist-info → bluer_objects-6.231.1.dist-info}/RECORD +15 -13
- {bluer_objects-6.223.1.dist-info → bluer_objects-6.231.1.dist-info}/WHEEL +0 -0
- {bluer_objects-6.223.1.dist-info → bluer_objects-6.231.1.dist-info}/licenses/LICENSE +0 -0
- {bluer_objects-6.223.1.dist-info → bluer_objects-6.231.1.dist-info}/top_level.txt +0 -0
|
@@ -3,12 +3,24 @@ import os
|
|
|
3
3
|
import yaml
|
|
4
4
|
|
|
5
5
|
from blueness import module
|
|
6
|
-
from bluer_options.env import get_env
|
|
7
6
|
|
|
8
|
-
from bluer_objects import NAME as MY_NAME
|
|
7
|
+
from bluer_objects import NAME as MY_NAME
|
|
9
8
|
from bluer_objects.metadata import get_from_object
|
|
10
9
|
from bluer_objects import file, env
|
|
11
10
|
from bluer_objects import markdown
|
|
11
|
+
from bluer_objects.README.utils import (
|
|
12
|
+
apply_legacy,
|
|
13
|
+
process_assets,
|
|
14
|
+
process_details,
|
|
15
|
+
process_envs,
|
|
16
|
+
process_help,
|
|
17
|
+
process_include,
|
|
18
|
+
process_mermaid,
|
|
19
|
+
process_objects,
|
|
20
|
+
process_variable,
|
|
21
|
+
signature,
|
|
22
|
+
variables,
|
|
23
|
+
)
|
|
12
24
|
from bluer_objects.logger import logger
|
|
13
25
|
|
|
14
26
|
MY_NAME = module.name(__file__, MY_NAME)
|
|
@@ -56,156 +68,47 @@ def build(
|
|
|
56
68
|
|
|
57
69
|
table_of_items = markdown.generate_table(items, cols=cols) if cols > 0 else items
|
|
58
70
|
|
|
59
|
-
signature = [
|
|
60
|
-
"",
|
|
61
|
-
" ".join(
|
|
62
|
-
[
|
|
63
|
-
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml)",
|
|
64
|
-
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml)",
|
|
65
|
-
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml)",
|
|
66
|
-
f"[](https://pypi.org/project/{MODULE_NAME}/)",
|
|
67
|
-
f"[](https://pypistats.org/packages/{MODULE_NAME})",
|
|
68
|
-
]
|
|
69
|
-
),
|
|
70
|
-
"",
|
|
71
|
-
"built by {} [`{}`]({}), based on {}[`{}-{}`]({}).".format(
|
|
72
|
-
MY_ICON,
|
|
73
|
-
"bluer README",
|
|
74
|
-
"https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README",
|
|
75
|
-
f"{ICON} " if ICON else "",
|
|
76
|
-
NAME,
|
|
77
|
-
VERSION,
|
|
78
|
-
f"https://github.com/kamangir/{REPO_NAME}",
|
|
79
|
-
),
|
|
80
|
-
]
|
|
81
|
-
|
|
82
71
|
success, template = file.load_text(template_filename)
|
|
83
72
|
if not success:
|
|
84
73
|
return success
|
|
85
74
|
|
|
86
|
-
def apply_legacy(line: str) -> str:
|
|
87
|
-
for before, after in {
|
|
88
|
-
"yaml:::": "metadata:::",
|
|
89
|
-
"--help--": "help:::",
|
|
90
|
-
"--include": "include:::",
|
|
91
|
-
"--table--": "items:::",
|
|
92
|
-
"--signature--": "signature:::",
|
|
93
|
-
}.items():
|
|
94
|
-
line = line.replace(before, after)
|
|
95
|
-
return line
|
|
96
|
-
|
|
97
75
|
if legacy_mode:
|
|
98
|
-
|
|
99
|
-
template = [apply_legacy(line) for line in template]
|
|
76
|
+
template = apply_legacy(template)
|
|
100
77
|
|
|
101
78
|
content: List[str] = []
|
|
102
79
|
mermaid_started: bool = False
|
|
103
|
-
variables: Dict[str, str] = {}
|
|
104
80
|
for template_line in template:
|
|
105
81
|
if template_line.startswith("ignore:::"):
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
(
|
|
137
|
-
(
|
|
138
|
-
"".format(
|
|
139
|
-
assets_repo,
|
|
140
|
-
token.split(":::")[1].strip(),
|
|
141
|
-
)
|
|
142
|
-
if any(
|
|
143
|
-
token.endswith(extension)
|
|
144
|
-
for extension in ["png", "jpg", "jpeg", "gif"]
|
|
145
|
-
)
|
|
146
|
-
else "[{}](https://github.com/{}/blob/main/{})".format(
|
|
147
|
-
file.name_and_extension(
|
|
148
|
-
token.split(":::")[1].strip()
|
|
149
|
-
),
|
|
150
|
-
assets_repo,
|
|
151
|
-
token.split(":::")[1].strip(),
|
|
152
|
-
)
|
|
153
|
-
)
|
|
154
|
-
if token.startswith("assets:::")
|
|
155
|
-
else token
|
|
156
|
-
)
|
|
157
|
-
for token in template_line.split(" ")
|
|
158
|
-
]
|
|
159
|
-
)
|
|
160
|
-
|
|
161
|
-
if "object:::" in template_line:
|
|
162
|
-
template_line = " ".join(
|
|
163
|
-
[
|
|
164
|
-
(
|
|
165
|
-
"[{}](https://{}.{}/{}.tar.gz)".format(
|
|
166
|
-
token.split(":::")[1].strip(),
|
|
167
|
-
env.S3_PUBLIC_STORAGE_BUCKET,
|
|
168
|
-
env.S3_STORAGE_ENDPOINT_URL.split("https://", 1)[1],
|
|
169
|
-
token.split(":::")[1].strip(),
|
|
170
|
-
)
|
|
171
|
-
if token.startswith("object:::")
|
|
172
|
-
else token
|
|
173
|
-
)
|
|
174
|
-
for token in template_line.split(" ")
|
|
175
|
-
]
|
|
176
|
-
)
|
|
177
|
-
|
|
178
|
-
content_section: List[str] = [template_line]
|
|
179
|
-
|
|
180
|
-
if template_line.startswith("details:::"):
|
|
181
|
-
suffix = template_line.split(":::", 1)[1]
|
|
182
|
-
if suffix:
|
|
183
|
-
content_section = [
|
|
184
|
-
"",
|
|
185
|
-
"<details>",
|
|
186
|
-
f"<summary>{suffix}</summary>",
|
|
187
|
-
"",
|
|
188
|
-
]
|
|
189
|
-
else:
|
|
190
|
-
content_section = [
|
|
191
|
-
"",
|
|
192
|
-
"</details>",
|
|
193
|
-
"",
|
|
194
|
-
]
|
|
195
|
-
elif template_line.startswith("metadata:::"):
|
|
196
|
-
object_name_and_key = template_line.split(":::", 1)[1]
|
|
197
|
-
if ":::" not in object_name_and_key:
|
|
198
|
-
object_name_and_key += ":::"
|
|
199
|
-
object_name, key = object_name_and_key.split(":::", 1)
|
|
200
|
-
|
|
201
|
-
value = get_from_object(
|
|
202
|
-
object_name,
|
|
203
|
-
key,
|
|
204
|
-
{},
|
|
205
|
-
download=True,
|
|
206
|
-
)
|
|
207
|
-
|
|
208
|
-
content_section = (
|
|
82
|
+
content += [template_line.split(":::", 1)[1].strip()]
|
|
83
|
+
continue
|
|
84
|
+
|
|
85
|
+
template_line = process_envs(template_line)
|
|
86
|
+
|
|
87
|
+
for key, value in variables.items():
|
|
88
|
+
template_line = template_line.replace(
|
|
89
|
+
f"get:::{key}",
|
|
90
|
+
value,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
if "metadata:::" in template_line:
|
|
94
|
+
object_name_and_key = template_line.split("metadata:::", 1)[1]
|
|
95
|
+
if " " in object_name_and_key:
|
|
96
|
+
object_name_and_key = object_name_and_key.split(" ", 1)[0]
|
|
97
|
+
if ":::" not in object_name_and_key:
|
|
98
|
+
object_name_and_key += ":::"
|
|
99
|
+
object_name, key = object_name_and_key.split(":::", 1)
|
|
100
|
+
|
|
101
|
+
value = get_from_object(
|
|
102
|
+
object_name,
|
|
103
|
+
key,
|
|
104
|
+
{},
|
|
105
|
+
download=True,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
logger.info(f"metadata[{object_name_and_key}] = {value}")
|
|
109
|
+
|
|
110
|
+
if template_line.startswith("metadata:::"):
|
|
111
|
+
content += (
|
|
209
112
|
["```yaml"]
|
|
210
113
|
+ yaml.dump(
|
|
211
114
|
value,
|
|
@@ -213,84 +116,70 @@ def build(
|
|
|
213
116
|
).split("\n")
|
|
214
117
|
+ ["```"]
|
|
215
118
|
)
|
|
216
|
-
|
|
217
|
-
mermaid_started = True
|
|
218
|
-
logger.info("🧜🏽♀️ detected ...")
|
|
219
|
-
elif mermaid_started and template_line.startswith("```"):
|
|
220
|
-
mermaid_started = False
|
|
221
|
-
elif mermaid_started:
|
|
222
|
-
if '"' in template_line and ":::folder" not in template_line:
|
|
223
|
-
template_line_pieces = template_line.split('"')
|
|
224
|
-
if len(template_line_pieces) != 3:
|
|
225
|
-
logger.error(
|
|
226
|
-
f"🧜🏽♀️ mermaid line not in expected format: {template_line}."
|
|
227
|
-
)
|
|
228
|
-
return False
|
|
229
|
-
|
|
230
|
-
template_line_pieces[1] = (
|
|
231
|
-
template_line_pieces[1]
|
|
232
|
-
.replace("<", "<")
|
|
233
|
-
.replace(">", ">")
|
|
234
|
-
.replace(" ", "<br>")
|
|
235
|
-
.replace("~~", " ")
|
|
236
|
-
)
|
|
237
|
-
|
|
238
|
-
content_section = ['"'.join(template_line_pieces)]
|
|
239
|
-
elif "items:::" in template_line:
|
|
240
|
-
content_section = table_of_items
|
|
241
|
-
elif "signature:::" in template_line:
|
|
242
|
-
content_section = signature
|
|
243
|
-
elif "include:::" in template_line:
|
|
244
|
-
include_filename_relative = template_line.split(" ")[1].strip()
|
|
245
|
-
include_filename = file.absolute(
|
|
246
|
-
include_filename_relative,
|
|
247
|
-
file.path(template_filename),
|
|
248
|
-
)
|
|
249
|
-
|
|
250
|
-
success, content_section = file.load_text(include_filename)
|
|
251
|
-
if not success:
|
|
252
|
-
return success
|
|
253
|
-
|
|
254
|
-
content_section = [
|
|
255
|
-
line for line in content_section if not line.startswith("used by:")
|
|
256
|
-
]
|
|
257
|
-
|
|
258
|
-
include_title = (template_line.split(" ", 2) + ["", "", ""])[2]
|
|
259
|
-
if include_title:
|
|
260
|
-
content_section = [f"## {include_title}"] + content_section[1:]
|
|
261
|
-
|
|
262
|
-
if "include:::noref" not in template_line:
|
|
263
|
-
content_section += [
|
|
264
|
-
"using [{}]({}).".format(
|
|
265
|
-
file.name(include_filename),
|
|
266
|
-
include_filename_relative,
|
|
267
|
-
)
|
|
268
|
-
]
|
|
269
|
-
|
|
270
|
-
logger.info(f"{MY_NAME}.build: including {include_filename} ...")
|
|
271
|
-
elif "help:::" in template_line:
|
|
272
|
-
if help_function is not None:
|
|
273
|
-
help_command = template_line.split("help:::")[1].strip()
|
|
274
|
-
|
|
275
|
-
tokens = help_command.strip().split(" ")[1:]
|
|
276
|
-
|
|
277
|
-
help_content = help_function(tokens)
|
|
278
|
-
if not help_content:
|
|
279
|
-
logger.warning(f"help not found: {help_command}: {tokens}")
|
|
280
|
-
return False
|
|
119
|
+
continue
|
|
281
120
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
121
|
+
template_line = template_line.replace(
|
|
122
|
+
f"metadata:::{object_name}:::{key}",
|
|
123
|
+
str(value),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
if template_line.startswith("set:::"):
|
|
127
|
+
process_variable(template_line)
|
|
128
|
+
continue
|
|
129
|
+
|
|
130
|
+
template_line = process_assets(template_line, assets_repo)
|
|
131
|
+
|
|
132
|
+
template_line = process_objects(template_line)
|
|
133
|
+
|
|
134
|
+
if template_line.startswith("details:::"):
|
|
135
|
+
content += process_details(template_line)
|
|
136
|
+
continue
|
|
137
|
+
|
|
138
|
+
if "items:::" in template_line:
|
|
139
|
+
content += table_of_items
|
|
140
|
+
continue
|
|
141
|
+
|
|
142
|
+
if "include:::" in template_line:
|
|
143
|
+
content += process_include(
|
|
144
|
+
template_line,
|
|
145
|
+
file.path(template_filename),
|
|
146
|
+
)
|
|
147
|
+
continue
|
|
148
|
+
|
|
149
|
+
if "signature:::" in template_line:
|
|
150
|
+
content += signature(
|
|
151
|
+
REPO_NAME,
|
|
152
|
+
NAME,
|
|
153
|
+
ICON,
|
|
154
|
+
MODULE_NAME,
|
|
155
|
+
VERSION,
|
|
156
|
+
)
|
|
157
|
+
continue
|
|
158
|
+
|
|
159
|
+
if "help:::" in template_line:
|
|
160
|
+
if help_function is None:
|
|
161
|
+
logger.warning("help_function not found.")
|
|
289
162
|
else:
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
163
|
+
content += process_help(
|
|
164
|
+
template_line,
|
|
165
|
+
help_function,
|
|
166
|
+
)
|
|
167
|
+
continue
|
|
168
|
+
|
|
169
|
+
content_section = [template_line]
|
|
170
|
+
if template_line.startswith("```mermaid"):
|
|
171
|
+
mermaid_started = True
|
|
172
|
+
logger.info("🧜🏽♀️ detected ...")
|
|
173
|
+
elif mermaid_started and template_line.startswith("```"):
|
|
174
|
+
mermaid_started = False
|
|
175
|
+
elif mermaid_started:
|
|
176
|
+
if '"' in template_line and ":::folder" not in template_line:
|
|
177
|
+
content_section = process_mermaid(template_line)
|
|
178
|
+
else:
|
|
179
|
+
for macro, macro_value in macros.items():
|
|
180
|
+
if macro in template_line:
|
|
181
|
+
content_section = macro_value
|
|
182
|
+
break
|
|
294
183
|
|
|
295
184
|
content += content_section
|
|
296
185
|
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
from typing import List, Tuple, Dict, Union, Callable
|
|
2
|
+
|
|
3
|
+
from bluer_options.env import get_env
|
|
4
|
+
from bluer_objects import file
|
|
5
|
+
from bluer_objects import env
|
|
6
|
+
from bluer_objects import NAME as MY_NAME, ICON as MY_ICON
|
|
7
|
+
|
|
8
|
+
from bluer_objects.logger import logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
variables: Dict[str, str] = {}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def apply_legacy_on_line(line: str) -> str:
|
|
15
|
+
for before, after in {
|
|
16
|
+
"yaml:::": "metadata:::",
|
|
17
|
+
"--help--": "help:::",
|
|
18
|
+
"--include": "include:::",
|
|
19
|
+
"--table--": "items:::",
|
|
20
|
+
"--signature--": "signature:::",
|
|
21
|
+
}.items():
|
|
22
|
+
line = line.replace(before, after)
|
|
23
|
+
return line
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def apply_legacy(template: List[str]) -> List[str]:
|
|
27
|
+
logger.info("applying legacy conversions...")
|
|
28
|
+
template = [apply_legacy_on_line(line) for line in template]
|
|
29
|
+
return template
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def process_assets(
|
|
33
|
+
template_line: str,
|
|
34
|
+
assets_repo: str,
|
|
35
|
+
) -> str:
|
|
36
|
+
if "assets:::" in template_line:
|
|
37
|
+
template_line = " ".join(
|
|
38
|
+
[
|
|
39
|
+
(
|
|
40
|
+
(
|
|
41
|
+
"".format(
|
|
42
|
+
assets_repo,
|
|
43
|
+
token.split(":::")[1].strip(),
|
|
44
|
+
)
|
|
45
|
+
if any(
|
|
46
|
+
token.endswith(extension)
|
|
47
|
+
for extension in ["png", "jpg", "jpeg", "gif"]
|
|
48
|
+
)
|
|
49
|
+
else "[{}](https://github.com/{}/blob/main/{})".format(
|
|
50
|
+
file.name_and_extension(token.split(":::")[1].strip()),
|
|
51
|
+
assets_repo,
|
|
52
|
+
token.split(":::")[1].strip(),
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
if token.startswith("assets:::")
|
|
56
|
+
else token
|
|
57
|
+
)
|
|
58
|
+
for token in template_line.split(" ")
|
|
59
|
+
]
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
return template_line
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def process_details(template_line: str) -> List[str]:
|
|
66
|
+
suffix = template_line.split(":::", 1)[1]
|
|
67
|
+
if suffix:
|
|
68
|
+
content_section = [
|
|
69
|
+
"",
|
|
70
|
+
"<details>",
|
|
71
|
+
f"<summary>{suffix}</summary>",
|
|
72
|
+
"",
|
|
73
|
+
]
|
|
74
|
+
else:
|
|
75
|
+
content_section = [
|
|
76
|
+
"",
|
|
77
|
+
"</details>",
|
|
78
|
+
"",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
return content_section
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def process_envs(template_line: str) -> str:
|
|
85
|
+
while "env:::" in template_line:
|
|
86
|
+
env_name = template_line.split("env:::", 1)[1]
|
|
87
|
+
if " " in env_name:
|
|
88
|
+
env_name = env_name.split(" ", 1)[0]
|
|
89
|
+
else:
|
|
90
|
+
if ":::" in env_name:
|
|
91
|
+
env_name = env_name.split(":::", 1)[0]
|
|
92
|
+
|
|
93
|
+
env_value = get_env(env_name)
|
|
94
|
+
|
|
95
|
+
template_line = template_line.replace(
|
|
96
|
+
f"env:::{env_name}",
|
|
97
|
+
env_value,
|
|
98
|
+
)
|
|
99
|
+
logger.info(f"{env_name} -> {env_value}")
|
|
100
|
+
|
|
101
|
+
return template_line
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def process_help(
|
|
105
|
+
template_line: str,
|
|
106
|
+
help_function: Union[Callable[[List[str]], str], None] = None,
|
|
107
|
+
) -> List[str]:
|
|
108
|
+
help_command = template_line.split("help:::")[1].strip()
|
|
109
|
+
|
|
110
|
+
tokens = help_command.strip().split(" ")[1:]
|
|
111
|
+
|
|
112
|
+
help_content = help_function(tokens)
|
|
113
|
+
if not help_content:
|
|
114
|
+
logger.warning(f"help not found: {help_command}: {tokens}")
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
logger.info(f"+= help: {help_command}")
|
|
118
|
+
print(help_content)
|
|
119
|
+
content_section = [
|
|
120
|
+
"```bash",
|
|
121
|
+
help_content,
|
|
122
|
+
"```",
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
return content_section
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def process_include(
|
|
129
|
+
template_line: str,
|
|
130
|
+
template_path: str,
|
|
131
|
+
) -> List[str]:
|
|
132
|
+
include_filename_relative = template_line.split(" ")[1].strip()
|
|
133
|
+
include_filename = file.absolute(
|
|
134
|
+
include_filename_relative,
|
|
135
|
+
template_path,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
success, content_section = file.load_text(include_filename)
|
|
139
|
+
if not success:
|
|
140
|
+
return success
|
|
141
|
+
|
|
142
|
+
content_section = [
|
|
143
|
+
line for line in content_section if not line.startswith("used by:")
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
include_title = (template_line.split(" ", 2) + ["", "", ""])[2]
|
|
147
|
+
if include_title:
|
|
148
|
+
content_section = [f"## {include_title}"] + content_section[1:]
|
|
149
|
+
|
|
150
|
+
if "include:::noref" not in template_line:
|
|
151
|
+
content_section += [
|
|
152
|
+
"using [{}]({}).".format(
|
|
153
|
+
file.name(include_filename),
|
|
154
|
+
include_filename_relative,
|
|
155
|
+
)
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
logger.info(f"{MY_NAME}.build: including {include_filename} ...")
|
|
159
|
+
|
|
160
|
+
return content_section
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def process_mermaid(template_line: str) -> List[str]:
|
|
164
|
+
template_line_pieces = template_line.split('"')
|
|
165
|
+
if len(template_line_pieces) != 3:
|
|
166
|
+
logger.error(f"🧜🏽♀️ mermaid line not in expected format: {template_line}.")
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
template_line_pieces[1] = (
|
|
170
|
+
template_line_pieces[1]
|
|
171
|
+
.replace("<", "<")
|
|
172
|
+
.replace(">", ">")
|
|
173
|
+
.replace(" ", "<br>")
|
|
174
|
+
.replace("~~", " ")
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
return ['"'.join(template_line_pieces)]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def process_objects(template_line: str) -> str:
|
|
181
|
+
if "object:::" in template_line:
|
|
182
|
+
template_line = " ".join(
|
|
183
|
+
[
|
|
184
|
+
(
|
|
185
|
+
"[{}](https://{}.{}/{}.tar.gz)".format(
|
|
186
|
+
token.split(":::")[1].strip(),
|
|
187
|
+
env.S3_PUBLIC_STORAGE_BUCKET,
|
|
188
|
+
env.S3_STORAGE_ENDPOINT_URL.split("https://", 1)[1],
|
|
189
|
+
token.split(":::")[1].strip(),
|
|
190
|
+
)
|
|
191
|
+
if token.startswith("object:::")
|
|
192
|
+
else token
|
|
193
|
+
)
|
|
194
|
+
for token in template_line.split(" ")
|
|
195
|
+
]
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
return template_line
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def process_variable(template_line: str):
|
|
202
|
+
key, value = template_line.split("set:::", 1)[1].split(" ", 1)
|
|
203
|
+
variables[key] = value
|
|
204
|
+
logger.info(f"{key} = {value}")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def signature(
|
|
208
|
+
REPO_NAME: str,
|
|
209
|
+
NAME: str,
|
|
210
|
+
ICON: str,
|
|
211
|
+
MODULE_NAME: str,
|
|
212
|
+
VERSION: str,
|
|
213
|
+
) -> List[str]:
|
|
214
|
+
return [
|
|
215
|
+
"",
|
|
216
|
+
" ".join(
|
|
217
|
+
[
|
|
218
|
+
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml)",
|
|
219
|
+
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml)",
|
|
220
|
+
f"[](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml)",
|
|
221
|
+
f"[](https://pypi.org/project/{MODULE_NAME}/)",
|
|
222
|
+
f"[](https://pypistats.org/packages/{MODULE_NAME})",
|
|
223
|
+
]
|
|
224
|
+
),
|
|
225
|
+
"",
|
|
226
|
+
"built by {} [`{}`]({}), based on {}[`{}-{}`]({}).".format(
|
|
227
|
+
MY_ICON,
|
|
228
|
+
"bluer README",
|
|
229
|
+
"https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README",
|
|
230
|
+
f"{ICON} " if ICON else "",
|
|
231
|
+
NAME,
|
|
232
|
+
VERSION,
|
|
233
|
+
f"https://github.com/kamangir/{REPO_NAME}",
|
|
234
|
+
),
|
|
235
|
+
]
|
bluer_objects/__init__.py
CHANGED
bluer_objects/storage/WebDAV.py
CHANGED
|
@@ -4,6 +4,7 @@ from webdav3.client import Client
|
|
|
4
4
|
from bluer_objects.storage.base import StorageInterface
|
|
5
5
|
from bluer_objects import env, file, path
|
|
6
6
|
from bluer_objects import objects
|
|
7
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
7
8
|
from bluer_objects.logger import logger
|
|
8
9
|
|
|
9
10
|
|
|
@@ -56,6 +57,7 @@ class WebDAVInterface(StorageInterface):
|
|
|
56
57
|
object_name: str,
|
|
57
58
|
filename: str = "",
|
|
58
59
|
log: bool = True,
|
|
60
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
59
61
|
) -> bool:
|
|
60
62
|
local_path = objects.path_of(
|
|
61
63
|
object_name=object_name,
|
|
@@ -80,6 +82,7 @@ class WebDAVInterface(StorageInterface):
|
|
|
80
82
|
object_name=object_name,
|
|
81
83
|
filename=filename,
|
|
82
84
|
log=log,
|
|
85
|
+
policy=policy,
|
|
83
86
|
)
|
|
84
87
|
|
|
85
88
|
def upload(
|
|
@@ -8,6 +8,7 @@ from tqdm import tqdm
|
|
|
8
8
|
from bluer_objects.storage.base import StorageInterface
|
|
9
9
|
from bluer_objects import env, file, path
|
|
10
10
|
from bluer_objects import objects
|
|
11
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
11
12
|
from bluer_objects.logger import logger
|
|
12
13
|
|
|
13
14
|
|
|
@@ -136,6 +137,7 @@ class WebDAVRequestInterface(StorageInterface):
|
|
|
136
137
|
object_name: str,
|
|
137
138
|
filename: str = "",
|
|
138
139
|
log: bool = True,
|
|
140
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
139
141
|
) -> bool:
|
|
140
142
|
if filename:
|
|
141
143
|
local_path = objects.path_of(
|
|
@@ -176,6 +178,7 @@ class WebDAVRequestInterface(StorageInterface):
|
|
|
176
178
|
object_name=object_name,
|
|
177
179
|
filename=filename,
|
|
178
180
|
log=log,
|
|
181
|
+
policy=policy,
|
|
179
182
|
)
|
|
180
183
|
|
|
181
184
|
logger.error(f"failed to download: {response.status_code}")
|
|
@@ -193,6 +196,7 @@ class WebDAVRequestInterface(StorageInterface):
|
|
|
193
196
|
object_name=object_name,
|
|
194
197
|
filename=filename_,
|
|
195
198
|
log=log,
|
|
199
|
+
policy=policy,
|
|
196
200
|
):
|
|
197
201
|
return False
|
|
198
202
|
|
|
@@ -5,9 +5,10 @@ from webdav3.client import Client
|
|
|
5
5
|
from tqdm import tqdm
|
|
6
6
|
|
|
7
7
|
from bluer_objects.storage.base import StorageInterface
|
|
8
|
-
from bluer_objects import env
|
|
8
|
+
from bluer_objects import env
|
|
9
9
|
from bluer_objects import objects
|
|
10
|
-
from bluer_objects.host import
|
|
10
|
+
from bluer_objects.host import unzip
|
|
11
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
11
12
|
from bluer_objects.logger import logger
|
|
12
13
|
|
|
13
14
|
|
|
@@ -65,6 +66,7 @@ class WebDAVzipInterface(StorageInterface):
|
|
|
65
66
|
object_name: str,
|
|
66
67
|
filename: str = "",
|
|
67
68
|
log: bool = True,
|
|
69
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
68
70
|
) -> bool:
|
|
69
71
|
object_path = objects.object_path(
|
|
70
72
|
object_name=object_name,
|
|
@@ -99,6 +101,7 @@ class WebDAVzipInterface(StorageInterface):
|
|
|
99
101
|
return super().download(
|
|
100
102
|
object_name=object_name,
|
|
101
103
|
log=log,
|
|
104
|
+
policy=policy,
|
|
102
105
|
)
|
|
103
106
|
|
|
104
107
|
def ls(
|
|
@@ -5,6 +5,7 @@ from bluer_objects.storage.base import StorageInterface
|
|
|
5
5
|
from bluer_objects.storage.WebDAV import WebDAVInterface
|
|
6
6
|
from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
|
|
7
7
|
from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
|
|
8
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
8
9
|
from bluer_objects import env
|
|
9
10
|
from bluer_objects.logger import logger
|
|
10
11
|
|
|
@@ -35,11 +36,13 @@ def download(
|
|
|
35
36
|
object_name: str,
|
|
36
37
|
filename: str = "",
|
|
37
38
|
log: bool = True,
|
|
39
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
38
40
|
) -> bool:
|
|
39
41
|
return interface.download(
|
|
40
42
|
object_name=object_name,
|
|
41
43
|
filename=filename,
|
|
42
44
|
log=log,
|
|
45
|
+
policy=policy,
|
|
43
46
|
)
|
|
44
47
|
|
|
45
48
|
|
bluer_objects/storage/base.py
CHANGED
|
@@ -3,6 +3,7 @@ import glob
|
|
|
3
3
|
from typing import Tuple, List
|
|
4
4
|
|
|
5
5
|
from bluer_objects import objects
|
|
6
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
6
7
|
from bluer_objects.logger import logger
|
|
7
8
|
|
|
8
9
|
|
|
@@ -18,13 +19,19 @@ class StorageInterface:
|
|
|
18
19
|
object_name: str,
|
|
19
20
|
filename: str = "",
|
|
20
21
|
log: bool = True,
|
|
22
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
21
23
|
) -> bool:
|
|
22
24
|
if log:
|
|
23
25
|
logger.info(
|
|
24
|
-
"{}.download {}{}".format(
|
|
26
|
+
"{}.download {}{}{}".format(
|
|
25
27
|
self.__class__.__name__,
|
|
26
28
|
object_name,
|
|
27
29
|
f"/{filename}" if filename else "",
|
|
30
|
+
(
|
|
31
|
+
""
|
|
32
|
+
if policy == DownloadPolicy.NONE
|
|
33
|
+
else " - policy:{}".format(policy.name.lower())
|
|
34
|
+
),
|
|
28
35
|
)
|
|
29
36
|
)
|
|
30
37
|
|
bluer_objects/storage/s3.py
CHANGED
|
@@ -11,6 +11,7 @@ from bluer_objects.storage.base import StorageInterface
|
|
|
11
11
|
from bluer_objects.env import ABCLI_OBJECT_ROOT
|
|
12
12
|
from bluer_objects import env, file, path
|
|
13
13
|
from bluer_objects import objects
|
|
14
|
+
from bluer_objects.storage.policies import DownloadPolicy
|
|
14
15
|
from bluer_objects.logger import logger
|
|
15
16
|
|
|
16
17
|
|
|
@@ -121,6 +122,7 @@ class S3Interface(StorageInterface):
|
|
|
121
122
|
object_name: str,
|
|
122
123
|
filename: str = "",
|
|
123
124
|
log: bool = True,
|
|
125
|
+
policy: DownloadPolicy = DownloadPolicy.NONE,
|
|
124
126
|
) -> bool:
|
|
125
127
|
if filename:
|
|
126
128
|
local_path = objects.path_of(
|
|
@@ -129,6 +131,11 @@ class S3Interface(StorageInterface):
|
|
|
129
131
|
create=True,
|
|
130
132
|
)
|
|
131
133
|
|
|
134
|
+
if policy == DownloadPolicy.DOESNT_EXIST and file.exists(local_path):
|
|
135
|
+
if log:
|
|
136
|
+
logger.info(f"✅ {filename}")
|
|
137
|
+
return True
|
|
138
|
+
|
|
132
139
|
if not path.create(file.path(local_path)):
|
|
133
140
|
return False
|
|
134
141
|
|
|
@@ -160,6 +167,7 @@ class S3Interface(StorageInterface):
|
|
|
160
167
|
object_name=object_name,
|
|
161
168
|
filename=filename,
|
|
162
169
|
log=log,
|
|
170
|
+
policy=policy,
|
|
163
171
|
)
|
|
164
172
|
|
|
165
173
|
success, list_of_files = self.ls(
|
|
@@ -174,6 +182,7 @@ class S3Interface(StorageInterface):
|
|
|
174
182
|
object_name=object_name,
|
|
175
183
|
filename=filename_,
|
|
176
184
|
log=log,
|
|
185
|
+
policy=policy,
|
|
177
186
|
):
|
|
178
187
|
return False
|
|
179
188
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bluer_objects
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.231.1
|
|
4
4
|
Summary: 🌀 Object management in Bash.
|
|
5
5
|
Home-page: https://github.com/kamangir/bluer-objects
|
|
6
6
|
Author: Arash Abadpour (Kamangir)
|
|
@@ -64,6 +64,6 @@ pip install bluer-objects
|
|
|
64
64
|
|
|
65
65
|
[](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [](https://pypi.org/project/bluer-objects/) [](https://pypistats.org/packages/bluer-objects)
|
|
66
66
|
|
|
67
|
-
built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.
|
|
67
|
+
built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.231.1`](https://github.com/kamangir/bluer-objects).
|
|
68
68
|
|
|
69
69
|
built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
bluer_objects/__init__.py,sha256=
|
|
1
|
+
bluer_objects/__init__.py,sha256=ObgK0QNoNgfJkVpVSqCPbWvtul6Np2g-HnsCFjqIArY,315
|
|
2
2
|
bluer_objects/__main__.py,sha256=Yqfov833_hJuRne19WrGhT5DWAPtdffpoMxeSXS7EGw,359
|
|
3
3
|
bluer_objects/config.env,sha256=RjcpnbKfRqNyGLRB4z7M_OG9z2pOM032ck__53JqXqo,216
|
|
4
4
|
bluer_objects/env.py,sha256=iw4QvaImqnavlsHwfkUScNHc7afDEJQKJSsHTtVJE78,2019
|
|
@@ -72,8 +72,9 @@ bluer_objects/.abcli/tests/web_where_am_ai.sh,sha256=BJ9G_m4id8cx_UB_l_jV2xY6AfQ
|
|
|
72
72
|
bluer_objects/.abcli/web/is_accessible.sh,sha256=Luv_6IvpscRYx7f39V0RnkkNEWTRfVGyQVUeij3iqa0,262
|
|
73
73
|
bluer_objects/.abcli/web/where_am_i.sh,sha256=QPBXFo6Ni4pZEoOx0rtuJUxk6tOlp0ESMyAc9YPy9zg,92
|
|
74
74
|
bluer_objects/README/__init__.py,sha256=JwxdTVAK3LeUaw7rMJujOFIXZA59HaLCtxpsR1C-vpo,1311
|
|
75
|
-
bluer_objects/README/functions.py,sha256=
|
|
75
|
+
bluer_objects/README/functions.py,sha256=JPr7kkBTwC8xbZptxDSUeey8PoHYCfssu3wuaff_S2k,5390
|
|
76
76
|
bluer_objects/README/items.py,sha256=rTiOVd9fCoKUbFofBWjRV6oNDaH3XXdpL0XIPZqpy_w,951
|
|
77
|
+
bluer_objects/README/utils.py,sha256=oZysOe-184SXz0yX_C8M_roTJ4wC8VRdyLpAezM0qb4,7139
|
|
77
78
|
bluer_objects/file/__init__.py,sha256=c_79ipBkKl6OFDimOev0vnaVdpUk-Bl3oJUapOreMXc,681
|
|
78
79
|
bluer_objects/file/__main__.py,sha256=v2IXWvZeh_B2sGYWzv1CiUY-7HWHXXghZM5M4IPjbu4,1277
|
|
79
80
|
bluer_objects/file/classes.py,sha256=TRgeRP2yxInPkBnywhuB4BsoBcBRA3UmQzX1dI43UxU,872
|
|
@@ -126,13 +127,14 @@ bluer_objects/mlflow/testing.py,sha256=cJH5Ki02fJN_Xos1j9yvwQChXvMkOa9i12vtDKmkb
|
|
|
126
127
|
bluer_objects/mlflow/lock/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
127
128
|
bluer_objects/mlflow/lock/__main__.py,sha256=xF_xq2UqAsEohSOHjxaFXaw9KopOEDg6LRDM5a4VAPQ,1138
|
|
128
129
|
bluer_objects/mlflow/lock/functions.py,sha256=MOslqblNAOsRvILzLF4q6m2EAwCk4f4zEWQpsy8lVnM,3045
|
|
129
|
-
bluer_objects/storage/WebDAV.py,sha256=
|
|
130
|
-
bluer_objects/storage/WebDAVrequest.py,sha256=
|
|
131
|
-
bluer_objects/storage/WebDAVzip.py,sha256=
|
|
132
|
-
bluer_objects/storage/__init__.py,sha256=
|
|
130
|
+
bluer_objects/storage/WebDAV.py,sha256=UftrpSDzJAtTeMEORNpOTbDhmRYAzgdTNbmrAkgzPTU,3279
|
|
131
|
+
bluer_objects/storage/WebDAVrequest.py,sha256=Wp4rtlWif7sdu4VohxenT8YVmf-tmV9gbBYxuKFDR8M,10279
|
|
132
|
+
bluer_objects/storage/WebDAVzip.py,sha256=EHbERaxnLUQqWSM12dwJYybvXCIqqo66uThvNBE8BHI,4212
|
|
133
|
+
bluer_objects/storage/__init__.py,sha256=LY3pxxflaZ0KUSJJzlh0GEFH-HpshBntn5qs62uk0WQ,1866
|
|
133
134
|
bluer_objects/storage/__main__.py,sha256=6T4gltrJ-3lihce2fm43UeWB1y09VInXHrVBnAAa_R4,2005
|
|
134
|
-
bluer_objects/storage/base.py,sha256=
|
|
135
|
-
bluer_objects/storage/
|
|
135
|
+
bluer_objects/storage/base.py,sha256=35G51iTsU99N_JQwUS2oatbACudAsiOrQ0E6fIz2cic,2396
|
|
136
|
+
bluer_objects/storage/policies.py,sha256=xIuIhFjcWH7ptg_jUgfYTNnXqs9xaX4yU5qSOhmmxhI,125
|
|
137
|
+
bluer_objects/storage/s3.py,sha256=30i9bqj5gGZQZ9YRAqcF5piThlFilsmYqI0eFXnJ3M8,9755
|
|
136
138
|
bluer_objects/testing/__init__.py,sha256=DWY5ZtvCnHG_t9BDiqy_ArLOZi-nlyAtPVMLA1PPAMU,62
|
|
137
139
|
bluer_objects/testing/__main__.py,sha256=hhJV9qn0V_8FxzNDcoHCHr4A7zf9UudnNGJCAPkTBGU,750
|
|
138
140
|
bluer_objects/testing/functions.py,sha256=AXAfzWLcEPkbSYTehdahshjKJ45C4IJkRs_TgrHOntc,1355
|
|
@@ -169,8 +171,8 @@ bluer_objects/tests/test_web_is_accessible.py,sha256=2Y20NAEDMblg0MKnhnqcfw3XVKE
|
|
|
169
171
|
bluer_objects/web/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
170
172
|
bluer_objects/web/__main__.py,sha256=xf2Ob54FI8JEokfGhFmiyOBdD9nBactwqmZvsKsdioU,624
|
|
171
173
|
bluer_objects/web/functions.py,sha256=KNufAFOc6N3BYf83lN2rUpKUdsnzb2anWyp9koFRVUo,172
|
|
172
|
-
bluer_objects-6.
|
|
173
|
-
bluer_objects-6.
|
|
174
|
-
bluer_objects-6.
|
|
175
|
-
bluer_objects-6.
|
|
176
|
-
bluer_objects-6.
|
|
174
|
+
bluer_objects-6.231.1.dist-info/licenses/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
|
|
175
|
+
bluer_objects-6.231.1.dist-info/METADATA,sha256=K9N197qOMIX9IXRIdgkJFkIjDATJBrqB_ex64KFfiJ4,3678
|
|
176
|
+
bluer_objects-6.231.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
177
|
+
bluer_objects-6.231.1.dist-info/top_level.txt,sha256=RX2TpddbnRkurda3G_pAdyeTztP2IhhRPx949GlEvQo,14
|
|
178
|
+
bluer_objects-6.231.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|