fosslight-util 1.4.34__py3-none-any.whl → 2.1.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_util/_get_downloadable_url.py +466 -36
- fosslight_util/compare_yaml.py +20 -11
- fosslight_util/constant.py +35 -0
- fosslight_util/correct.py +46 -78
- fosslight_util/cover.py +60 -0
- fosslight_util/download.py +302 -95
- fosslight_util/exclude.py +65 -0
- fosslight_util/help.py +20 -8
- fosslight_util/oss_item.py +171 -110
- fosslight_util/output_format.py +147 -19
- fosslight_util/parsing_yaml.py +45 -23
- fosslight_util/read_excel.py +40 -39
- fosslight_util/set_log.py +30 -5
- fosslight_util/spdx_licenses.py +2 -1
- fosslight_util/write_cyclonedx.py +210 -0
- fosslight_util/write_excel.py +141 -133
- fosslight_util/write_opossum.py +14 -20
- fosslight_util/write_scancodejson.py +51 -32
- fosslight_util/write_spdx.py +162 -115
- fosslight_util/write_txt.py +2 -1
- fosslight_util/write_yaml.py +43 -49
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/METADATA +32 -24
- fosslight_util-2.1.28.dist-info/RECORD +32 -0
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/WHEEL +1 -1
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/entry_points.txt +0 -1
- fosslight_util/convert_excel_to_yaml.py +0 -69
- fosslight_util-1.4.34.dist-info/RECORD +0 -30
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info/licenses}/LICENSE +0 -0
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/top_level.txt +0 -0
fosslight_util/download.py
CHANGED
|
@@ -10,7 +10,7 @@ import zipfile
|
|
|
10
10
|
import logging
|
|
11
11
|
import argparse
|
|
12
12
|
import shutil
|
|
13
|
-
import
|
|
13
|
+
from git import Repo, GitCommandError, Git
|
|
14
14
|
import bz2
|
|
15
15
|
import contextlib
|
|
16
16
|
from datetime import datetime
|
|
@@ -24,9 +24,14 @@ import time
|
|
|
24
24
|
import threading
|
|
25
25
|
import platform
|
|
26
26
|
import subprocess
|
|
27
|
+
import re
|
|
28
|
+
from typing import Tuple
|
|
29
|
+
import urllib.parse
|
|
30
|
+
import json
|
|
27
31
|
|
|
28
32
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
29
|
-
compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
|
|
33
|
+
compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2", ".whl"}
|
|
34
|
+
prefix_refs = ["refs/remotes/origin/", "refs/tags/"]
|
|
30
35
|
SIGNAL_TIMEOUT = 600
|
|
31
36
|
|
|
32
37
|
|
|
@@ -43,12 +48,14 @@ class Alarm(threading.Thread):
|
|
|
43
48
|
|
|
44
49
|
|
|
45
50
|
class TimeOutException(Exception):
|
|
46
|
-
|
|
51
|
+
def __init__(self, message, error_code):
|
|
52
|
+
super().__init__(message)
|
|
53
|
+
self.error_code = error_code
|
|
47
54
|
|
|
48
55
|
|
|
49
56
|
def alarm_handler(signum, frame):
|
|
50
57
|
logger.warning("download timeout! (%d sec)", SIGNAL_TIMEOUT)
|
|
51
|
-
raise TimeOutException()
|
|
58
|
+
raise TimeOutException(f'Timeout ({SIGNAL_TIMEOUT} sec)', 1)
|
|
52
59
|
|
|
53
60
|
|
|
54
61
|
def change_src_link_to_https(src_link):
|
|
@@ -58,13 +65,21 @@ def change_src_link_to_https(src_link):
|
|
|
58
65
|
return src_link
|
|
59
66
|
|
|
60
67
|
|
|
68
|
+
def change_ssh_link_to_https(src_link):
|
|
69
|
+
src_link = src_link.replace("git@github.com:", "https://github.com/")
|
|
70
|
+
return src_link
|
|
71
|
+
|
|
72
|
+
|
|
61
73
|
def parse_src_link(src_link):
|
|
62
|
-
src_info = {}
|
|
74
|
+
src_info = {"url": src_link}
|
|
63
75
|
src_link_changed = ""
|
|
64
|
-
if src_link.startswith("git://") or src_link.startswith("
|
|
76
|
+
if src_link.startswith("git://") or src_link.startswith("git@") \
|
|
77
|
+
or src_link.startswith("https://") or src_link.startswith("http://"):
|
|
65
78
|
src_link_split = src_link.split(';')
|
|
66
79
|
if src_link.startswith("git://github.com/"):
|
|
67
80
|
src_link_changed = change_src_link_to_https(src_link_split[0])
|
|
81
|
+
elif src_link.startswith("git@github.com:"):
|
|
82
|
+
src_link_changed = src_link_split[0]
|
|
68
83
|
else:
|
|
69
84
|
if "rubygems.org" in src_link:
|
|
70
85
|
src_info["rubygems"] = True
|
|
@@ -76,49 +91,26 @@ def parse_src_link(src_link):
|
|
|
76
91
|
src_info["url"] = src_link_changed
|
|
77
92
|
src_info["branch"] = branch_info
|
|
78
93
|
src_info["tag"] = tag_info
|
|
79
|
-
|
|
94
|
+
return src_info
|
|
80
95
|
|
|
81
96
|
|
|
82
|
-
def
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
|
|
88
|
-
|
|
89
|
-
src_link = ""
|
|
90
|
-
target_dir = os.getcwd()
|
|
91
|
-
log_dir = os.getcwd()
|
|
92
|
-
|
|
93
|
-
try:
|
|
94
|
-
args = parser.parse_args()
|
|
95
|
-
except SystemExit:
|
|
96
|
-
sys.exit(0)
|
|
97
|
-
|
|
98
|
-
if args.help:
|
|
99
|
-
print_help_msg_download()
|
|
100
|
-
if args.source:
|
|
101
|
-
src_link = args.source
|
|
102
|
-
if args.target_dir:
|
|
103
|
-
target_dir = args.target_dir
|
|
104
|
-
if args.log_dir:
|
|
105
|
-
log_dir = args.log_dir
|
|
106
|
-
|
|
107
|
-
if not src_link:
|
|
108
|
-
print_help_msg_download()
|
|
109
|
-
else:
|
|
110
|
-
cli_download_and_extract(src_link, target_dir, log_dir)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compressed_only=False):
|
|
97
|
+
def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_to: str = "",
|
|
98
|
+
compressed_only: bool = False, ssh_key: str = "",
|
|
99
|
+
id: str = "", git_token: str = "",
|
|
100
|
+
called_cli: bool = True,
|
|
101
|
+
output: bool = False) -> Tuple[bool, str, str, str]:
|
|
114
102
|
global logger
|
|
115
103
|
|
|
116
104
|
success = True
|
|
117
105
|
msg = ""
|
|
106
|
+
msg_wget = ""
|
|
107
|
+
oss_name = ""
|
|
108
|
+
oss_version = ""
|
|
118
109
|
log_file_name = "fosslight_download_" + \
|
|
119
110
|
datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
|
|
120
111
|
logger, log_item = init_log(os.path.join(log_dir, log_file_name))
|
|
121
112
|
link = link.strip()
|
|
113
|
+
is_rubygems = False
|
|
122
114
|
|
|
123
115
|
try:
|
|
124
116
|
if not link:
|
|
@@ -127,6 +119,9 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
|
|
|
127
119
|
elif os.path.isfile(target_dir):
|
|
128
120
|
success = False
|
|
129
121
|
msg = f"The target directory exists as a file.: {target_dir}"
|
|
122
|
+
elif os.path.exists(link) or os.path.isdir(link) or os.path.isfile(link):
|
|
123
|
+
success = False
|
|
124
|
+
msg = f"You cannot enter a path instead of a link.: {link}"
|
|
130
125
|
else:
|
|
131
126
|
src_info = parse_src_link(link)
|
|
132
127
|
link = src_info.get("url", "")
|
|
@@ -135,22 +130,50 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
|
|
|
135
130
|
is_rubygems = src_info.get("rubygems", False)
|
|
136
131
|
|
|
137
132
|
# General download (git clone, wget)
|
|
138
|
-
|
|
133
|
+
success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir,
|
|
134
|
+
checkout_to,
|
|
135
|
+
tag, branch,
|
|
136
|
+
ssh_key, id, git_token,
|
|
137
|
+
called_cli)
|
|
138
|
+
link = change_ssh_link_to_https(link)
|
|
139
|
+
if (not is_rubygems) and (not success_git):
|
|
139
140
|
if os.path.isfile(target_dir):
|
|
140
141
|
shutil.rmtree(target_dir)
|
|
141
142
|
|
|
142
|
-
success, downloaded_file = download_wget(link, target_dir,
|
|
143
|
+
success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir,
|
|
144
|
+
compressed_only, checkout_to)
|
|
143
145
|
if success:
|
|
144
|
-
success = extract_compressed_file(downloaded_file, target_dir, True)
|
|
146
|
+
success = extract_compressed_file(downloaded_file, target_dir, True, compressed_only)
|
|
145
147
|
# Download from rubygems.org
|
|
146
148
|
elif is_rubygems and shutil.which("gem"):
|
|
147
149
|
success = gem_download(link, target_dir, checkout_to)
|
|
150
|
+
if msg:
|
|
151
|
+
msg = f'git fail: {msg}'
|
|
152
|
+
if is_rubygems:
|
|
153
|
+
msg = f'gem download: {success}'
|
|
154
|
+
else:
|
|
155
|
+
if msg_wget:
|
|
156
|
+
msg = f'{msg}, wget fail: {msg_wget}'
|
|
157
|
+
else:
|
|
158
|
+
msg = f'{msg}, wget success'
|
|
159
|
+
|
|
148
160
|
except Exception as error:
|
|
149
161
|
success = False
|
|
150
162
|
msg = str(error)
|
|
151
163
|
|
|
152
|
-
|
|
153
|
-
|
|
164
|
+
if output:
|
|
165
|
+
output_result = {
|
|
166
|
+
"success": success,
|
|
167
|
+
"message": msg,
|
|
168
|
+
"oss_name": oss_name,
|
|
169
|
+
"oss_version": oss_version
|
|
170
|
+
}
|
|
171
|
+
output_json = os.path.join(log_dir, "fosslight_download_output.json")
|
|
172
|
+
with open(output_json, 'w') as f:
|
|
173
|
+
json.dump(output_result, f, indent=4)
|
|
174
|
+
|
|
175
|
+
logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
|
|
176
|
+
return success, msg, oss_name, oss_version
|
|
154
177
|
|
|
155
178
|
|
|
156
179
|
def get_ref_to_checkout(checkout_to, ref_list):
|
|
@@ -160,7 +183,6 @@ def get_ref_to_checkout(checkout_to, ref_list):
|
|
|
160
183
|
if checkout_to in ref_list:
|
|
161
184
|
return checkout_to
|
|
162
185
|
|
|
163
|
-
prefix_refs = ["refs/remotes/origin/", "refs/tags/"]
|
|
164
186
|
for prefix in prefix_refs:
|
|
165
187
|
ref_to_checkout = prefix+checkout_to
|
|
166
188
|
if ref_to_checkout in ref_list:
|
|
@@ -173,62 +195,188 @@ def get_ref_to_checkout(checkout_to, ref_list):
|
|
|
173
195
|
return ref_to_checkout
|
|
174
196
|
|
|
175
197
|
|
|
176
|
-
def
|
|
177
|
-
if
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
198
|
+
def get_remote_refs(git_url: str):
|
|
199
|
+
if not git_url:
|
|
200
|
+
return {"tags": [], "branches": []}
|
|
201
|
+
tags = []
|
|
202
|
+
branches = []
|
|
203
|
+
try:
|
|
204
|
+
cp = subprocess.run(["git", "ls-remote", "--tags", "--heads", git_url], capture_output=True, text=True, timeout=30)
|
|
205
|
+
if cp.returncode == 0:
|
|
206
|
+
for line in cp.stdout.splitlines():
|
|
207
|
+
parts = line.split('\t')
|
|
208
|
+
if len(parts) != 2:
|
|
209
|
+
continue
|
|
210
|
+
ref = parts[1]
|
|
211
|
+
if ref.startswith('refs/tags/'):
|
|
212
|
+
tags.append(ref[len('refs/tags/'):])
|
|
213
|
+
elif ref.startswith('refs/heads/'):
|
|
214
|
+
branches.append(ref[len('refs/heads/'):])
|
|
215
|
+
except Exception as e:
|
|
216
|
+
logger.debug(f"get_remote_refs - failed: {e}")
|
|
217
|
+
return {"tags": tags, "branches": branches}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def decide_checkout(checkout_to="", tag="", branch="", git_url=""):
|
|
221
|
+
base = checkout_to or tag or branch
|
|
222
|
+
if not base:
|
|
223
|
+
return ""
|
|
224
|
+
|
|
225
|
+
ref_dict = get_remote_refs(git_url)
|
|
226
|
+
tag_set = set(ref_dict.get("tags", []))
|
|
227
|
+
branch_set = set(ref_dict.get("branches", []))
|
|
228
|
+
|
|
229
|
+
ver_re = re.compile(r'^(?:v\.? ?)?' + re.escape(base) + r'$', re.IGNORECASE)
|
|
230
|
+
|
|
231
|
+
# tag: exact -> prefix variant -> endswith
|
|
232
|
+
if base in tag_set:
|
|
233
|
+
return base
|
|
234
|
+
tag_candidates = [c for c in tag_set if ver_re.match(c)]
|
|
235
|
+
if tag_candidates:
|
|
236
|
+
return min(tag_candidates, key=lambda x: (len(x), x.lower()))
|
|
237
|
+
tag_ends = [n for n in tag_set if n.endswith(base)]
|
|
238
|
+
if tag_ends:
|
|
239
|
+
return min(tag_ends, key=len)
|
|
240
|
+
|
|
241
|
+
# branch: exact -> prefix variant -> endswith
|
|
242
|
+
if base in branch_set:
|
|
243
|
+
return base
|
|
244
|
+
branch_candidates = [c for c in branch_set if ver_re.match(c)]
|
|
245
|
+
if branch_candidates:
|
|
246
|
+
return min(branch_candidates, key=lambda x: (len(x), x.lower()))
|
|
247
|
+
branch_ends = [n for n in branch_set if n.endswith(base)]
|
|
248
|
+
if branch_ends:
|
|
249
|
+
return min(branch_ends, key=len)
|
|
250
|
+
|
|
251
|
+
return base
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def get_github_ossname(link):
|
|
255
|
+
oss_name = ""
|
|
256
|
+
p = re.compile(r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?')
|
|
257
|
+
match = p.match(link)
|
|
258
|
+
if match:
|
|
259
|
+
oss_name = f"{match.group(1)}-{match.group(2)}"
|
|
260
|
+
return oss_name
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def get_github_token(git_url):
|
|
264
|
+
github_token = ""
|
|
265
|
+
pattern = r'https://(.*?)@'
|
|
266
|
+
search = re.search(pattern, git_url)
|
|
267
|
+
if search:
|
|
268
|
+
github_token = search.group(1)
|
|
269
|
+
return github_token
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli=True):
|
|
273
|
+
success = False
|
|
274
|
+
oss_version = ""
|
|
275
|
+
|
|
276
|
+
logger.info(f"Download git url :{git_url}")
|
|
277
|
+
env = os.environ.copy()
|
|
278
|
+
if not called_cli:
|
|
279
|
+
env["GIT_TERMINAL_PROMPT"] = "0"
|
|
280
|
+
if refs_to_checkout:
|
|
281
|
+
try:
|
|
282
|
+
# gitPython uses the branch argument the same whether you check out to a branch or a tag.
|
|
283
|
+
Repo.clone_from(git_url, target_dir, branch=refs_to_checkout, env=env)
|
|
284
|
+
if any(Path(target_dir).iterdir()):
|
|
285
|
+
success = True
|
|
286
|
+
oss_version = refs_to_checkout
|
|
287
|
+
logger.info(f"Files found in {target_dir} after clone.")
|
|
288
|
+
else:
|
|
289
|
+
logger.info(f"No files found in {target_dir} after clone.")
|
|
290
|
+
success = False
|
|
291
|
+
except GitCommandError as error:
|
|
292
|
+
logger.info(f"Git checkout error:{error}")
|
|
293
|
+
success = False
|
|
294
|
+
except Exception as e:
|
|
295
|
+
logger.info(f"Repo.clone_from error:{e}")
|
|
296
|
+
success = False
|
|
297
|
+
|
|
298
|
+
if not success:
|
|
299
|
+
Repo.clone_from(git_url, target_dir, env=env)
|
|
300
|
+
if any(Path(target_dir).iterdir()):
|
|
301
|
+
success = True
|
|
182
302
|
else:
|
|
183
|
-
|
|
184
|
-
|
|
303
|
+
logger.info(f"No files found in {target_dir} after clone.")
|
|
304
|
+
success = False
|
|
305
|
+
return success, oss_version
|
|
185
306
|
|
|
186
307
|
|
|
187
|
-
def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""
|
|
188
|
-
|
|
308
|
+
def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
|
|
309
|
+
ssh_key="", id="", git_token="", called_cli=True):
|
|
310
|
+
oss_name = get_github_ossname(git_url)
|
|
311
|
+
refs_to_checkout = decide_checkout(checkout_to, tag, branch, git_url)
|
|
312
|
+
msg = ""
|
|
313
|
+
success = True
|
|
189
314
|
|
|
190
|
-
if platform.system() != "Windows":
|
|
191
|
-
signal.signal(signal.SIGALRM, alarm_handler)
|
|
192
|
-
signal.alarm(SIGNAL_TIMEOUT)
|
|
193
|
-
else:
|
|
194
|
-
alarm = Alarm(SIGNAL_TIMEOUT)
|
|
195
|
-
alarm.start()
|
|
196
315
|
try:
|
|
197
|
-
Path(target_dir).mkdir(parents=True, exist_ok=True)
|
|
198
|
-
repo = git.clone_repository(git_url, target_dir,
|
|
199
|
-
bare=False, repository=None,
|
|
200
|
-
remote=None, callbacks=None)
|
|
201
316
|
if platform.system() != "Windows":
|
|
202
|
-
signal.
|
|
317
|
+
signal.signal(signal.SIGALRM, alarm_handler)
|
|
318
|
+
signal.alarm(SIGNAL_TIMEOUT)
|
|
203
319
|
else:
|
|
204
|
-
|
|
320
|
+
alarm = Alarm(SIGNAL_TIMEOUT)
|
|
321
|
+
alarm.start()
|
|
322
|
+
|
|
323
|
+
Path(target_dir).mkdir(parents=True, exist_ok=True)
|
|
324
|
+
|
|
325
|
+
if git_url.startswith("ssh:") and not ssh_key:
|
|
326
|
+
msg = "Private git needs ssh_key"
|
|
327
|
+
success = False
|
|
328
|
+
else:
|
|
329
|
+
if ssh_key:
|
|
330
|
+
logger.info(f"Download git with ssh_key:{git_url}")
|
|
331
|
+
git_ssh_cmd = f'ssh -i {ssh_key}'
|
|
332
|
+
with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
|
|
333
|
+
success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
|
|
334
|
+
else:
|
|
335
|
+
if id and git_token:
|
|
336
|
+
try:
|
|
337
|
+
m = re.match(r"^(ht|f)tp(s?)\:\/\/", git_url)
|
|
338
|
+
protocol = m.group()
|
|
339
|
+
if protocol:
|
|
340
|
+
encoded_git_token = urllib.parse.quote(git_token, safe='')
|
|
341
|
+
encoded_id = urllib.parse.quote(id, safe='')
|
|
342
|
+
git_url = git_url.replace(protocol, f"{protocol}{encoded_id}:{encoded_git_token}@")
|
|
343
|
+
except Exception as error:
|
|
344
|
+
logger.info(f"Failed to insert id, token to git url:{error}")
|
|
345
|
+
success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
|
|
346
|
+
|
|
347
|
+
logger.info(f"git checkout: {oss_version}")
|
|
348
|
+
refs_to_checkout = oss_version
|
|
349
|
+
|
|
350
|
+
if platform.system() != "Windows":
|
|
351
|
+
signal.alarm(0)
|
|
352
|
+
else:
|
|
353
|
+
del alarm
|
|
205
354
|
except Exception as error:
|
|
355
|
+
success = False
|
|
206
356
|
logger.warning(f"git clone - failed: {error}")
|
|
207
|
-
|
|
208
|
-
try:
|
|
209
|
-
if ref_to_checkout != "":
|
|
210
|
-
ref_list = [x for x in repo.references]
|
|
211
|
-
ref_to_checkout = get_ref_to_checkout(ref_to_checkout, ref_list)
|
|
212
|
-
logger.info(f"git checkout: {ref_to_checkout}")
|
|
213
|
-
repo.checkout(ref_to_checkout)
|
|
214
|
-
except Exception as error:
|
|
215
|
-
logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
|
|
216
|
-
return True
|
|
357
|
+
msg = str(error)
|
|
217
358
|
|
|
359
|
+
return success, msg, oss_name, refs_to_checkout
|
|
218
360
|
|
|
219
|
-
|
|
361
|
+
|
|
362
|
+
def download_wget(link, target_dir, compressed_only, checkout_to):
|
|
220
363
|
success = False
|
|
364
|
+
msg = ""
|
|
365
|
+
oss_name = ""
|
|
366
|
+
oss_version = ""
|
|
221
367
|
downloaded_file = ""
|
|
222
|
-
|
|
223
|
-
signal.signal(signal.SIGALRM, alarm_handler)
|
|
224
|
-
signal.alarm(SIGNAL_TIMEOUT)
|
|
225
|
-
else:
|
|
226
|
-
alarm = Alarm(SIGNAL_TIMEOUT)
|
|
227
|
-
alarm.start()
|
|
368
|
+
|
|
228
369
|
try:
|
|
370
|
+
if platform.system() != "Windows":
|
|
371
|
+
signal.signal(signal.SIGALRM, alarm_handler)
|
|
372
|
+
signal.alarm(SIGNAL_TIMEOUT)
|
|
373
|
+
else:
|
|
374
|
+
alarm = Alarm(SIGNAL_TIMEOUT)
|
|
375
|
+
alarm.start()
|
|
376
|
+
|
|
229
377
|
Path(target_dir).mkdir(parents=True, exist_ok=True)
|
|
230
378
|
|
|
231
|
-
ret, new_link = get_downloadable_url(link)
|
|
379
|
+
ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link, checkout_to)
|
|
232
380
|
if ret and new_link:
|
|
233
381
|
link = new_link
|
|
234
382
|
|
|
@@ -237,6 +385,9 @@ def download_wget(link, target_dir, compressed_only):
|
|
|
237
385
|
if link.endswith(ext):
|
|
238
386
|
success = True
|
|
239
387
|
break
|
|
388
|
+
if not success:
|
|
389
|
+
if pkg_type == 'cargo':
|
|
390
|
+
success = True
|
|
240
391
|
else:
|
|
241
392
|
success = True
|
|
242
393
|
|
|
@@ -244,7 +395,11 @@ def download_wget(link, target_dir, compressed_only):
|
|
|
244
395
|
raise Exception('Not supported compression type (link:{0})'.format(link))
|
|
245
396
|
|
|
246
397
|
logger.info(f"wget: {link}")
|
|
247
|
-
|
|
398
|
+
if pkg_type == 'cargo':
|
|
399
|
+
outfile = os.path.join(target_dir, f'{oss_name}.tar.gz')
|
|
400
|
+
downloaded_file = wget.download(link, out=outfile)
|
|
401
|
+
else:
|
|
402
|
+
downloaded_file = wget.download(link, target_dir)
|
|
248
403
|
if platform.system() != "Windows":
|
|
249
404
|
signal.alarm(0)
|
|
250
405
|
else:
|
|
@@ -255,9 +410,10 @@ def download_wget(link, target_dir, compressed_only):
|
|
|
255
410
|
logger.debug(f"wget - downloaded: {downloaded_file}")
|
|
256
411
|
except Exception as error:
|
|
257
412
|
success = False
|
|
413
|
+
msg = str(error)
|
|
258
414
|
logger.warning(f"wget - failed: {error}")
|
|
259
415
|
|
|
260
|
-
return success, downloaded_file
|
|
416
|
+
return success, downloaded_file, msg, oss_name, oss_version
|
|
261
417
|
|
|
262
418
|
|
|
263
419
|
def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
|
|
@@ -265,14 +421,14 @@ def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
|
|
|
265
421
|
try:
|
|
266
422
|
files_path = [os.path.join(src_dir, x) for x in os.listdir(src_dir)]
|
|
267
423
|
for fname in files_path:
|
|
268
|
-
extract_compressed_file(fname, target_dir, remove_after_extract)
|
|
424
|
+
extract_compressed_file(fname, target_dir, remove_after_extract, True)
|
|
269
425
|
except Exception as error:
|
|
270
426
|
logger.debug(f"Extract files in dir - failed: {error}")
|
|
271
427
|
return False
|
|
272
428
|
return True
|
|
273
429
|
|
|
274
430
|
|
|
275
|
-
def extract_compressed_file(fname, extract_path, remove_after_extract=True):
|
|
431
|
+
def extract_compressed_file(fname, extract_path, remove_after_extract=True, compressed_only=True):
|
|
276
432
|
success = True
|
|
277
433
|
try:
|
|
278
434
|
is_compressed_file = True
|
|
@@ -292,9 +448,12 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True):
|
|
|
292
448
|
unzip(fname, extract_path)
|
|
293
449
|
elif fname.endswith(".bz2"):
|
|
294
450
|
decompress_bz2(fname, extract_path)
|
|
451
|
+
elif fname.endswith(".whl"):
|
|
452
|
+
unzip(fname, extract_path)
|
|
295
453
|
else:
|
|
296
454
|
is_compressed_file = False
|
|
297
|
-
|
|
455
|
+
if compressed_only:
|
|
456
|
+
success = False
|
|
298
457
|
logger.warning(f"Unsupported file extension: {fname}")
|
|
299
458
|
|
|
300
459
|
if remove_after_extract and is_compressed_file:
|
|
@@ -364,16 +523,64 @@ def gem_download(link, target_dir, checkout_to):
|
|
|
364
523
|
fetch_result = subprocess.check_output(fetch_cmd, universal_newlines=True)
|
|
365
524
|
fetch_result = fetch_result.replace('\n', '').split(' ')[-1]
|
|
366
525
|
downloaded_gem = f"{fetch_result}.gem"
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
526
|
+
if not os.path.isfile(downloaded_gem):
|
|
527
|
+
success = False
|
|
528
|
+
else:
|
|
529
|
+
# gem unpack
|
|
530
|
+
subprocess.check_output(['gem', 'unpack', downloaded_gem], universal_newlines=True)
|
|
531
|
+
# move unpacked file to target directory
|
|
532
|
+
shutil.move(fetch_result, target_dir)
|
|
372
533
|
except Exception as error:
|
|
373
534
|
success = False
|
|
374
535
|
logger.warning(f"gem download - failed: {error}")
|
|
375
536
|
return success
|
|
376
537
|
|
|
377
538
|
|
|
539
|
+
def main():
|
|
540
|
+
parser = argparse.ArgumentParser(description='FOSSLight Downloader', prog='fosslight_download', add_help=False)
|
|
541
|
+
parser.add_argument('-h', '--help', help='Print help message', action='store_true', dest='help')
|
|
542
|
+
parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
|
|
543
|
+
parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
|
|
544
|
+
parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
|
|
545
|
+
parser.add_argument('-c', '--checkout_to', help='Checkout to branch or tag', type=str, dest='checkout_to', default="")
|
|
546
|
+
parser.add_argument('-z', '--compressed_only', help='Unzip only compressed file',
|
|
547
|
+
action='store_true', dest='compressed_only', default=False)
|
|
548
|
+
parser.add_argument('-o', '--output', help='Generate output file', action='store_true', dest='output', default=False)
|
|
549
|
+
|
|
550
|
+
src_link = ""
|
|
551
|
+
target_dir = os.getcwd()
|
|
552
|
+
log_dir = os.getcwd()
|
|
553
|
+
checkout_to = ""
|
|
554
|
+
compressed_only = False
|
|
555
|
+
output = False
|
|
556
|
+
|
|
557
|
+
try:
|
|
558
|
+
args = parser.parse_args()
|
|
559
|
+
except SystemExit:
|
|
560
|
+
sys.exit(0)
|
|
561
|
+
|
|
562
|
+
if args.help:
|
|
563
|
+
print_help_msg_download()
|
|
564
|
+
if args.source:
|
|
565
|
+
src_link = args.source
|
|
566
|
+
if args.target_dir:
|
|
567
|
+
target_dir = args.target_dir
|
|
568
|
+
if args.log_dir:
|
|
569
|
+
log_dir = args.log_dir
|
|
570
|
+
if args.checkout_to:
|
|
571
|
+
checkout_to = args.checkout_to
|
|
572
|
+
if args.compressed_only:
|
|
573
|
+
compressed_only = args.compressed_only
|
|
574
|
+
if args.output:
|
|
575
|
+
output = args.output
|
|
576
|
+
|
|
577
|
+
if not src_link:
|
|
578
|
+
print_help_msg_download()
|
|
579
|
+
else:
|
|
580
|
+
cli_download_and_extract(src_link, target_dir, log_dir, checkout_to,
|
|
581
|
+
compressed_only, "", "", "", False,
|
|
582
|
+
output)
|
|
583
|
+
|
|
584
|
+
|
|
378
585
|
if __name__ == '__main__':
|
|
379
586
|
main()
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2025 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import fnmatch
|
|
8
|
+
from typing import List
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def excluding_files(patterns: List[str], path_to_scan: str) -> List[str]:
|
|
12
|
+
excluded_paths = set()
|
|
13
|
+
|
|
14
|
+
# Normalize patterns: e.g., 'sample/', 'sample/*' -> 'sample'
|
|
15
|
+
# Replace backslash with slash
|
|
16
|
+
normalized_patterns = []
|
|
17
|
+
for pattern in patterns:
|
|
18
|
+
pattern = pattern.replace('\\', '/')
|
|
19
|
+
if pattern.endswith('/') or pattern.endswith('/*'):
|
|
20
|
+
pattern = pattern.rstrip('/*')
|
|
21
|
+
normalized_patterns.append(pattern)
|
|
22
|
+
|
|
23
|
+
# Traverse directories
|
|
24
|
+
for root, dirs, files in os.walk(path_to_scan):
|
|
25
|
+
remove_dir_list = []
|
|
26
|
+
|
|
27
|
+
# (1) Directory matching
|
|
28
|
+
for d in dirs:
|
|
29
|
+
dir_name = d
|
|
30
|
+
dir_path = os.path.relpath(os.path.join(root, d), path_to_scan).replace('\\', '/')
|
|
31
|
+
matched = False
|
|
32
|
+
|
|
33
|
+
for pat in normalized_patterns:
|
|
34
|
+
# Match directory name
|
|
35
|
+
if fnmatch.fnmatch(dir_name, pat):
|
|
36
|
+
matched = True
|
|
37
|
+
|
|
38
|
+
# Match the full relative path
|
|
39
|
+
if not matched:
|
|
40
|
+
if fnmatch.fnmatch(dir_path, pat) or fnmatch.fnmatch(dir_path, pat + "/*"):
|
|
41
|
+
matched = True
|
|
42
|
+
|
|
43
|
+
# If matched, exclude all files under this directory and stop checking patterns
|
|
44
|
+
if matched:
|
|
45
|
+
sub_root_path = os.path.join(root, d)
|
|
46
|
+
for sr, _, sf in os.walk(sub_root_path):
|
|
47
|
+
for sub_file in sf:
|
|
48
|
+
sub_file_path = os.path.relpath(os.path.join(sr, sub_file), path_to_scan)
|
|
49
|
+
excluded_paths.add(sub_file_path.replace('\\', '/'))
|
|
50
|
+
remove_dir_list.append(d)
|
|
51
|
+
break
|
|
52
|
+
|
|
53
|
+
# (1-2) Prune matched directories from further traversal
|
|
54
|
+
for rd in remove_dir_list:
|
|
55
|
+
dirs.remove(rd)
|
|
56
|
+
|
|
57
|
+
# (2) File matching
|
|
58
|
+
for f in files:
|
|
59
|
+
file_path = os.path.relpath(os.path.join(root, f), path_to_scan).replace('\\', '/')
|
|
60
|
+
for pat in normalized_patterns:
|
|
61
|
+
if fnmatch.fnmatch(file_path, pat) or fnmatch.fnmatch(file_path, pat + "/*"):
|
|
62
|
+
excluded_paths.add(file_path)
|
|
63
|
+
break
|
|
64
|
+
|
|
65
|
+
return sorted(excluded_paths)
|
fosslight_util/help.py
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
# Copyright (c) 2021 LG Electronics Inc.
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
import sys
|
|
6
|
-
|
|
6
|
+
try:
|
|
7
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
8
|
+
except ImportError:
|
|
9
|
+
from importlib_metadata import version, PackageNotFoundError # Python <3.8
|
|
7
10
|
|
|
8
11
|
_HELP_MESSAGE_COMMON = """
|
|
9
12
|
_______ _______ _______ _______ ___ ___ __
|
|
@@ -31,16 +34,20 @@ _HELP_MESSAGE_DOWNLOAD = """
|
|
|
31
34
|
Optional:
|
|
32
35
|
-h\t\t Print help message
|
|
33
36
|
-t\t\t Output path name
|
|
34
|
-
-d\t\t Directory name to save the log file
|
|
37
|
+
-d\t\t Directory name to save the log file
|
|
38
|
+
-s\t\t Source link to download
|
|
39
|
+
-t\t\t Directory to download source code
|
|
40
|
+
-c\t\t Checkout to branch or tag/ or version
|
|
41
|
+
-z\t\t Unzip only compressed file
|
|
42
|
+
-o\t\t Generate summary output file with this option"""
|
|
35
43
|
|
|
36
44
|
|
|
37
45
|
class PrintHelpMsg():
|
|
38
|
-
message_suffix = ""
|
|
39
46
|
|
|
40
|
-
def __init__(self, value):
|
|
47
|
+
def __init__(self, value: str = ""):
|
|
41
48
|
self.message_suffix = value
|
|
42
49
|
|
|
43
|
-
def print_help_msg(self, exitopt):
|
|
50
|
+
def print_help_msg(self, exitopt: bool) -> None:
|
|
44
51
|
print(_HELP_MESSAGE_COMMON)
|
|
45
52
|
print(self.message_suffix)
|
|
46
53
|
|
|
@@ -48,14 +55,19 @@ class PrintHelpMsg():
|
|
|
48
55
|
sys.exit()
|
|
49
56
|
|
|
50
57
|
|
|
51
|
-
def print_package_version(pkg_name, msg="", exitopt=True):
|
|
58
|
+
def print_package_version(pkg_name: str, msg: str = "", exitopt: bool = True) -> str:
|
|
52
59
|
if msg == "":
|
|
53
60
|
msg = f"{pkg_name} Version:"
|
|
54
|
-
|
|
55
|
-
|
|
61
|
+
try:
|
|
62
|
+
cur_version = version(pkg_name)
|
|
63
|
+
except PackageNotFoundError:
|
|
64
|
+
cur_version = "unknown"
|
|
56
65
|
|
|
57
66
|
if exitopt:
|
|
67
|
+
print(f'{msg} {cur_version}')
|
|
58
68
|
sys.exit(0)
|
|
69
|
+
else:
|
|
70
|
+
return cur_version
|
|
59
71
|
|
|
60
72
|
|
|
61
73
|
def print_help_msg_download(exitOpt=True):
|