bundle-deps 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bundle_deps-0.1.0/PKG-INFO +24 -0
- bundle_deps-0.1.0/README.md +15 -0
- bundle_deps-0.1.0/bundle_deps/__init__.py +1 -0
- bundle_deps-0.1.0/bundle_deps/__main__.py +4 -0
- bundle_deps-0.1.0/bundle_deps/archive.py +36 -0
- bundle_deps-0.1.0/bundle_deps/cli.py +34 -0
- bundle_deps-0.1.0/bundle_deps/download.py +202 -0
- bundle_deps-0.1.0/bundle_deps/export.py +120 -0
- bundle_deps-0.1.0/bundle_deps/git_source.py +51 -0
- bundle_deps-0.1.0/bundle_deps/locks.py +152 -0
- bundle_deps-0.1.0/bundle_deps/utils.py +56 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/PKG-INFO +24 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/SOURCES.txt +17 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/dependency_links.txt +1 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/entry_points.txt +2 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/requires.txt +2 -0
- bundle_deps-0.1.0/bundle_deps.egg-info/top_level.txt +1 -0
- bundle_deps-0.1.0/pyproject.toml +17 -0
- bundle_deps-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bundle-deps
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bundle a project's npm and uv dependencies for offline use
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: requests
|
|
8
|
+
Requires-Dist: tqdm
|
|
9
|
+
|
|
10
|
+
# bundle-deps
|
|
11
|
+
|
|
12
|
+
Bundle a project's npm and uv dependencies (from its lock files) for offline use.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
pip install bundle-deps # from PyPI
|
|
17
|
+
pip install git+<repo-url> # or straight from git
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
bundle-deps <project_path> [--commit-sha SHA | --latest-commit] [--manylinux]
|
|
22
|
+
[--no-pack] [--bundle-name NAME] [--output-dir DIR] [-v]
|
|
23
|
+
|
|
24
|
+
Also runnable as `python -m bundle_deps`.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# bundle-deps
|
|
2
|
+
|
|
3
|
+
Bundle a project's npm and uv dependencies (from its lock files) for offline use.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
pip install bundle-deps # from PyPI
|
|
8
|
+
pip install git+<repo-url> # or straight from git
|
|
9
|
+
|
|
10
|
+
## Usage
|
|
11
|
+
|
|
12
|
+
bundle-deps <project_path> [--commit-sha SHA | --latest-commit] [--manylinux]
|
|
13
|
+
[--no-pack] [--bundle-name NAME] [--output-dir DIR] [-v]
|
|
14
|
+
|
|
15
|
+
Also runnable as `python -m bundle_deps`.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Package bundling tool."""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tarfile
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def create_bundle_archive(source_dir, output_file):
|
|
6
|
+
"""
|
|
7
|
+
The function takes the directory that contains
|
|
8
|
+
the dependencies and zip it into a bundle
|
|
9
|
+
"""
|
|
10
|
+
try:
|
|
11
|
+
# create a gz gile
|
|
12
|
+
with tarfile.open(output_file, "w:gz") as tar:
|
|
13
|
+
# add the dir
|
|
14
|
+
tar.add(source_dir, arcname=os.path.basename(source_dir))
|
|
15
|
+
except (tarfile.TarError, OSError) as error:
|
|
16
|
+
raise RuntimeError(
|
|
17
|
+
f"Failed to create bundle archive: {source_dir}"
|
|
18
|
+
) from error
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def create_dependencies_archive(source_dir, output_file):
|
|
22
|
+
"""
|
|
23
|
+
The function creates an archive of every dependency
|
|
24
|
+
"""
|
|
25
|
+
try:
|
|
26
|
+
# create a new gz fie and add all of the files in the dir
|
|
27
|
+
with tarfile.open(output_file, "w:gz") as tar:
|
|
28
|
+
for file in os.listdir(source_dir):
|
|
29
|
+
file_path = os.path.join(source_dir, file)
|
|
30
|
+
tar.add(file_path, arcname=file)
|
|
31
|
+
except (tarfile.TarError, OSError) as error:
|
|
32
|
+
raise RuntimeError(
|
|
33
|
+
f"Failed to create dependencies archive: {source_dir}"
|
|
34
|
+
) from error
|
|
35
|
+
|
|
36
|
+
print(f"Dependencies archive created: {output_file}")
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
3
|
+
from bundle_deps.export import export_project
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def create_parser():
|
|
7
|
+
"""
|
|
8
|
+
The function creates the parser for the flags of the cli tool
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
parser = argparse.ArgumentParser(description="Package bundling tool")
|
|
12
|
+
parser.add_argument("project_path", help="Path to project directory")
|
|
13
|
+
parser.add_argument("--commit-sha" , help="Commit SHA to compare current dependencies against")
|
|
14
|
+
parser.add_argument("--latest-commit", action="store_true", help="Get the latest Commit SHA to compare current dependencies against")
|
|
15
|
+
parser.add_argument("--verbose","-v" ,action="store_true", help="Print each dependency as it is downloaded instead of showing a progress bar")
|
|
16
|
+
parser.add_argument("--manylinux", action="store_true", help="Download only manylinux wheels (plus pure python wheels) for uv dependencies")
|
|
17
|
+
parser.add_argument("--no-pack", action="store_true", help="Does not pack the dependencies in .tar.gz")
|
|
18
|
+
parser.add_argument("--bundle-name", help="Allows to choose the bundle name")
|
|
19
|
+
parser.add_argument("--output-dir", help="Allows to choose the output dir for the bundle")
|
|
20
|
+
|
|
21
|
+
return parser
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main():
|
|
25
|
+
|
|
26
|
+
# creates the parser and get the args
|
|
27
|
+
parser = create_parser()
|
|
28
|
+
args = parser.parse_args()
|
|
29
|
+
try:
|
|
30
|
+
export_project(args.project_path, args.commit_sha, args.latest_commit, args.manylinux, args.verbose, args.no_pack, args.bundle_name, args.output_dir)
|
|
31
|
+
|
|
32
|
+
except (FileNotFoundError, NotADirectoryError, RuntimeError) as error:
|
|
33
|
+
print(f"error: {error}")
|
|
34
|
+
raise SystemExit(1)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import requests
|
|
3
|
+
import tqdm
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
|
|
6
|
+
DOWNLOAD_WORKERS = 8
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def download_file_with_progress(url, file_path):
|
|
10
|
+
"""
|
|
11
|
+
Downloads a single file from url to file_path (no per-file progress bar).
|
|
12
|
+
"""
|
|
13
|
+
with requests.get(url, stream=True, timeout=30) as response:
|
|
14
|
+
response.raise_for_status()
|
|
15
|
+
block_size = 1024 # 1 KB
|
|
16
|
+
|
|
17
|
+
with open(file_path, "wb") as file:
|
|
18
|
+
for chunk in response.iter_content(chunk_size=block_size):
|
|
19
|
+
if chunk:
|
|
20
|
+
file.write(chunk)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def download_dependencies(dependencies, output_dir, plan_files, desc, exists_set=None, verbose=False):
|
|
24
|
+
"""
|
|
25
|
+
The function runs the shared download pipeline for any ecosystem.
|
|
26
|
+
|
|
27
|
+
plan_files(dependency) decides what a single dependency needs and returns
|
|
28
|
+
(files, skips, failed):
|
|
29
|
+
files - list of (url, filename, label) to download, or None to skip
|
|
30
|
+
the dependency without counting it as a failure
|
|
31
|
+
skips - messages to report as skipped
|
|
32
|
+
failed - True when the dependency is already broken before downloading
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
if exists_set is None:
|
|
36
|
+
exists_set = set()
|
|
37
|
+
|
|
38
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
39
|
+
|
|
40
|
+
failed_dependencies = []
|
|
41
|
+
skipped = []
|
|
42
|
+
errors = []
|
|
43
|
+
|
|
44
|
+
def download_one(dependency):
|
|
45
|
+
"""Returns (package, success) or None if skipped"""
|
|
46
|
+
package = f"{dependency['name']}@{dependency['version']}"
|
|
47
|
+
|
|
48
|
+
if package in exists_set:
|
|
49
|
+
skipped.append(f"{package} - already exists")
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
files, skips, dependency_failed = plan_files(dependency)
|
|
53
|
+
|
|
54
|
+
skipped.extend(skips)
|
|
55
|
+
|
|
56
|
+
# the dependency has nothing to download and is not treated as a failure
|
|
57
|
+
if files is None:
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
for url, filename, label in files:
|
|
61
|
+
|
|
62
|
+
file_path = os.path.join(output_dir, filename)
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
if verbose:
|
|
66
|
+
print(f"Downloading: {label}", flush=True)
|
|
67
|
+
download_file_with_progress(url, file_path)
|
|
68
|
+
except requests.RequestException as error:
|
|
69
|
+
errors.append(f"{package}: {error}")
|
|
70
|
+
dependency_failed = True
|
|
71
|
+
|
|
72
|
+
return package, not dependency_failed
|
|
73
|
+
|
|
74
|
+
with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor:
|
|
75
|
+
results = executor.map(download_one, dependencies)
|
|
76
|
+
if not verbose:
|
|
77
|
+
results = tqdm.tqdm(
|
|
78
|
+
results,
|
|
79
|
+
total=len(dependencies),
|
|
80
|
+
desc=desc,
|
|
81
|
+
unit="pack",
|
|
82
|
+
)
|
|
83
|
+
for result in results:
|
|
84
|
+
if result is None:
|
|
85
|
+
continue
|
|
86
|
+
package, success = result
|
|
87
|
+
if success:
|
|
88
|
+
exists_set.add(package)
|
|
89
|
+
else:
|
|
90
|
+
failed_dependencies.append(package)
|
|
91
|
+
|
|
92
|
+
for item in skipped:
|
|
93
|
+
print(f"Skipped: {item}")
|
|
94
|
+
for item in errors:
|
|
95
|
+
print(f"Failed: {item}")
|
|
96
|
+
|
|
97
|
+
if failed_dependencies:
|
|
98
|
+
raise RuntimeError(
|
|
99
|
+
"Failed to download dependencies: "
|
|
100
|
+
+ ", ".join(failed_dependencies)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def plan_npm_files(dependency):
|
|
105
|
+
"""
|
|
106
|
+
The function decides which file a single npm dependency needs
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
url = dependency.get("resolved")
|
|
110
|
+
|
|
111
|
+
# without a URL there is nothing to download, but it is not a failure
|
|
112
|
+
if not url:
|
|
113
|
+
return None, [f"{dependency['name']} - no URL"], False
|
|
114
|
+
|
|
115
|
+
package = f"{dependency['name']}@{dependency['version']}"
|
|
116
|
+
|
|
117
|
+
filename = (
|
|
118
|
+
dependency["name"].replace("/", "_")
|
|
119
|
+
+ "-"
|
|
120
|
+
+ dependency["version"]
|
|
121
|
+
+ ".tgz"
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
return [(url, filename, package)], [], False
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def plan_uv_files(dependency, output_dir, manylinux=False):
|
|
128
|
+
"""
|
|
129
|
+
The function decides which wheels and sdist a single uv dependency needs
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
package = f"{dependency['name']}@{dependency['version']}"
|
|
133
|
+
|
|
134
|
+
files = []
|
|
135
|
+
skips = []
|
|
136
|
+
dependency_failed = False
|
|
137
|
+
|
|
138
|
+
for wheel in dependency.get("wheels", []):
|
|
139
|
+
|
|
140
|
+
url = wheel.get("url")
|
|
141
|
+
|
|
142
|
+
if not url:
|
|
143
|
+
skips.append(f"{package} - no URL")
|
|
144
|
+
dependency_failed = True
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
filename = os.path.basename(url)
|
|
148
|
+
|
|
149
|
+
# with the manylinux flag, skip platform wheels that aren't manylinux
|
|
150
|
+
# (pure python "any" wheels are still needed)
|
|
151
|
+
if manylinux and "manylinux" not in filename and not filename.endswith("-any.whl"):
|
|
152
|
+
skips.append(f"{filename} - not manylinux")
|
|
153
|
+
continue
|
|
154
|
+
|
|
155
|
+
files.append((url, filename, filename))
|
|
156
|
+
|
|
157
|
+
sdist = dependency.get("sdist")
|
|
158
|
+
|
|
159
|
+
if sdist:
|
|
160
|
+
url = sdist.get("url")
|
|
161
|
+
|
|
162
|
+
if url:
|
|
163
|
+
filename = os.path.basename(url)
|
|
164
|
+
|
|
165
|
+
# the sdist is skipped when a wheel already produced the same file
|
|
166
|
+
if not os.path.exists(os.path.join(output_dir, filename)):
|
|
167
|
+
files.append((url, filename, filename))
|
|
168
|
+
|
|
169
|
+
return files, skips, dependency_failed
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def download_npm_dependencies(dependencies, output_dir, exists_set=None, verbose=False):
|
|
173
|
+
"""
|
|
174
|
+
The function downloads the npm dependencies from the dependencies list
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
return download_dependencies(
|
|
178
|
+
dependencies,
|
|
179
|
+
output_dir,
|
|
180
|
+
plan_npm_files,
|
|
181
|
+
"Downloading npm",
|
|
182
|
+
exists_set,
|
|
183
|
+
verbose,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def download_uv_dependencies(dependencies, output_dir, exists_set=None, manylinux=False, verbose=False):
|
|
188
|
+
"""
|
|
189
|
+
The function downloads the uv dependencies from the dependencies list
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
def plan(dependency):
|
|
193
|
+
return plan_uv_files(dependency, output_dir, manylinux)
|
|
194
|
+
|
|
195
|
+
return download_dependencies(
|
|
196
|
+
dependencies,
|
|
197
|
+
output_dir,
|
|
198
|
+
plan,
|
|
199
|
+
"Downloading uv",
|
|
200
|
+
exists_set,
|
|
201
|
+
verbose,
|
|
202
|
+
)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from bundle_deps.utils import runtime_count, temporary_directory, validate_project_path
|
|
6
|
+
from bundle_deps.locks import (
|
|
7
|
+
find_lock_files,
|
|
8
|
+
gather_npm_dependencies_from_locks,
|
|
9
|
+
gather_uv_dependencies_from_lockfiles,
|
|
10
|
+
)
|
|
11
|
+
from bundle_deps.download import download_npm_dependencies, download_uv_dependencies
|
|
12
|
+
from bundle_deps.archive import create_bundle_archive, create_dependencies_archive
|
|
13
|
+
from bundle_deps.git_source import get_git_url, fetch_git_by_sha
|
|
14
|
+
|
|
15
|
+
COMMIT_SHA = os.getenv("GITHUB_SHA") or "example-sha"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@runtime_count
|
|
19
|
+
def export_project(project_path, commit_sha=None, latest_commit=False, manylinux=False, verbose=False, no_pack=False, bundle_name = None, output_dir=None):
|
|
20
|
+
"""
|
|
21
|
+
The function handles the export flow from getting the dependencies
|
|
22
|
+
from the lockfiles to downloading it and creating a bundle
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
# check if the projects path exists or a dir
|
|
26
|
+
project_path = validate_project_path(project_path)
|
|
27
|
+
|
|
28
|
+
# get all the paths of the lock files
|
|
29
|
+
npm_lock_files, uv_lock_files = find_lock_files(project_path)
|
|
30
|
+
|
|
31
|
+
# if theres no lock files stop
|
|
32
|
+
if not npm_lock_files and not uv_lock_files:
|
|
33
|
+
print("No lock files found")
|
|
34
|
+
return
|
|
35
|
+
|
|
36
|
+
# get the timestamp for the bundle name
|
|
37
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
38
|
+
if bundle_name is None:
|
|
39
|
+
bundle_name = f"{timestamp}-{COMMIT_SHA}"
|
|
40
|
+
|
|
41
|
+
# gather all of the dependencies from lockfiles
|
|
42
|
+
npm_dependencies, _ = gather_npm_dependencies_from_locks(npm_lock_files)
|
|
43
|
+
uv_dependencies, _ = gather_uv_dependencies_from_lockfiles(uv_lock_files)
|
|
44
|
+
|
|
45
|
+
# every dependency is downloaded into a temp dir that is removed at the end
|
|
46
|
+
with temporary_directory() as temp_dir:
|
|
47
|
+
|
|
48
|
+
bundle_dir = os.path.join(temp_dir, bundle_name)
|
|
49
|
+
|
|
50
|
+
# make paths for seperate dirs for uv and npm
|
|
51
|
+
npm_dir = os.path.join(bundle_dir, "npm")
|
|
52
|
+
uv_dir = os.path.join(bundle_dir, "uv")
|
|
53
|
+
|
|
54
|
+
if commit_sha or latest_commit:
|
|
55
|
+
|
|
56
|
+
# clone the git repo from a specific sha into its own temp dir
|
|
57
|
+
with temporary_directory() as git_temp_dir:
|
|
58
|
+
|
|
59
|
+
git_path = os.path.join(git_temp_dir, "remote_git")
|
|
60
|
+
|
|
61
|
+
# clone the git repo of the specific sha
|
|
62
|
+
fetch_git_by_sha(get_git_url(), commit_sha, latest_commit, git_path)
|
|
63
|
+
|
|
64
|
+
# get lock files
|
|
65
|
+
git_npm_lock_files, git_uv_lock_files = find_lock_files(git_path)
|
|
66
|
+
|
|
67
|
+
# gather all of the dependencies in
|
|
68
|
+
_, git_npm_set = gather_npm_dependencies_from_locks(git_npm_lock_files)
|
|
69
|
+
_, git_uv_set = gather_uv_dependencies_from_lockfiles(git_uv_lock_files)
|
|
70
|
+
|
|
71
|
+
# download only the dependencies that the old commit does not have
|
|
72
|
+
download_npm_dependencies(npm_dependencies, npm_dir, git_npm_set, verbose=verbose)
|
|
73
|
+
download_uv_dependencies(uv_dependencies, uv_dir, git_uv_set, manylinux, verbose)
|
|
74
|
+
else:
|
|
75
|
+
# download all of the dependencies
|
|
76
|
+
download_npm_dependencies(npm_dependencies, npm_dir, verbose=verbose)
|
|
77
|
+
download_uv_dependencies(uv_dependencies, uv_dir, manylinux=manylinux, verbose=verbose)
|
|
78
|
+
|
|
79
|
+
# if theres no uv dependencies remove the dir
|
|
80
|
+
if len(os.listdir(uv_dir)) == 0:
|
|
81
|
+
print("No uv dependencies required download")
|
|
82
|
+
os.rmdir(uv_dir)
|
|
83
|
+
|
|
84
|
+
# if theres no npm dependencies remove the dir
|
|
85
|
+
if len(os.listdir(npm_dir)) == 0:
|
|
86
|
+
print("No npm dependencies required download")
|
|
87
|
+
os.rmdir(npm_dir)
|
|
88
|
+
else:
|
|
89
|
+
# create npm-dependencies.tgz
|
|
90
|
+
npm_archive = os.path.join(bundle_dir, "npm-dependencies.tgz")
|
|
91
|
+
|
|
92
|
+
create_dependencies_archive(npm_dir, npm_archive)
|
|
93
|
+
|
|
94
|
+
# remove original npm dir
|
|
95
|
+
shutil.rmtree(npm_dir)
|
|
96
|
+
|
|
97
|
+
if not os.listdir(bundle_dir):
|
|
98
|
+
print("No dependencies required download")
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
if no_pack:
|
|
102
|
+
# copy the dependencies out of the temp dir as a plain folder
|
|
103
|
+
output_path = os.path.join(output_dir or os.getcwd(), bundle_name)
|
|
104
|
+
shutil.copytree(bundle_dir, output_path, dirs_exist_ok=True)
|
|
105
|
+
|
|
106
|
+
print(f"Dependencies copied to: {output_path}")
|
|
107
|
+
else:
|
|
108
|
+
# pack the dependencies and copy the archive out of the temp dir
|
|
109
|
+
archive_path = os.path.join(temp_dir, f"{bundle_name}.tar.gz")
|
|
110
|
+
create_bundle_archive(bundle_dir, archive_path)
|
|
111
|
+
|
|
112
|
+
if output_dir is None:
|
|
113
|
+
output_path = os.path.join(os.getcwd(), f"{bundle_name}.tar.gz")
|
|
114
|
+
else:
|
|
115
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
116
|
+
output_path = os.path.join(output_dir, f"{bundle_name}.tar.gz")
|
|
117
|
+
|
|
118
|
+
shutil.copy2(archive_path, output_path)
|
|
119
|
+
|
|
120
|
+
print(f"Bundle created: {output_path}")
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tarfile
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
from bundle_deps.utils import temporary_directory
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_git_url():
|
|
9
|
+
"""Resolve the repo URL dynamically: CI env vars first, then the origin remote."""
|
|
10
|
+
server = os.getenv("GITHUB_SERVER_URL")
|
|
11
|
+
repo = os.getenv("GITHUB_REPOSITORY")
|
|
12
|
+
if server and repo:
|
|
13
|
+
return f"{server}/{repo}.git"
|
|
14
|
+
return subprocess.run(
|
|
15
|
+
["git", "remote", "get-url", "origin"],
|
|
16
|
+
capture_output=True, text=True, check=True,
|
|
17
|
+
).stdout.strip()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def fetch_git_by_sha(git_url, commit_sha, latest_commit=False, git_path="remote_git"):
|
|
21
|
+
"""
|
|
22
|
+
The function clones a repo from a git url and specific commit
|
|
23
|
+
into git_path
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
# the clone and the archive live in a temp dir that is removed at the end
|
|
27
|
+
with temporary_directory() as work_dir:
|
|
28
|
+
|
|
29
|
+
temp_git = os.path.join(work_dir, "temp_git")
|
|
30
|
+
repo_archive = os.path.join(work_dir, "repo.tar")
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
# clone the repo and fetch the specific commit as an archive
|
|
34
|
+
subprocess.run(["git", "clone", "--no-checkout", git_url, temp_git],check=True)
|
|
35
|
+
|
|
36
|
+
commit_sha = "HEAD^" if latest_commit else commit_sha
|
|
37
|
+
|
|
38
|
+
print(f"Processing the commit: {commit_sha}")
|
|
39
|
+
|
|
40
|
+
subprocess.run(["git", "-C", temp_git, "archive", "--format=tar", commit_sha, "-o", repo_archive],check=True)
|
|
41
|
+
|
|
42
|
+
# make dir for the unzipped archive
|
|
43
|
+
os.makedirs(git_path, exist_ok=True)
|
|
44
|
+
|
|
45
|
+
# open the repo archive
|
|
46
|
+
with tarfile.open(repo_archive, "r") as tar:
|
|
47
|
+
tar.extractall(git_path)
|
|
48
|
+
except subprocess.CalledProcessError as error:
|
|
49
|
+
raise RuntimeError(f"Failed to fetch git commit {commit_sha}") from error
|
|
50
|
+
|
|
51
|
+
return git_path
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import json
|
|
3
|
+
import tomllib
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def find_lock_files(project_path):
|
|
7
|
+
"""
|
|
8
|
+
The function finds all of the lock files in the project
|
|
9
|
+
and returns their paths
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
# init lists for uv and npm locks paths
|
|
13
|
+
uv_lock_files = []
|
|
14
|
+
npm_lock_files = []
|
|
15
|
+
|
|
16
|
+
# recursive walk in the project dir in search of lock files
|
|
17
|
+
for root, dirs, files in os.walk(project_path):
|
|
18
|
+
|
|
19
|
+
# check each file
|
|
20
|
+
for file in files:
|
|
21
|
+
|
|
22
|
+
# if file is package-lock or uv.lock append to list
|
|
23
|
+
if file == "package-lock.json":
|
|
24
|
+
npm_lock_files.append(os.path.join(root, file))
|
|
25
|
+
|
|
26
|
+
elif file == "uv.lock":
|
|
27
|
+
uv_lock_files.append(os.path.join(root, file))
|
|
28
|
+
|
|
29
|
+
return npm_lock_files, uv_lock_files
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def extract_npm_dependencies(lock_path):
|
|
33
|
+
"""
|
|
34
|
+
The function takes the npm dependencies , their versions ,
|
|
35
|
+
integrity and url
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
# init list for the dependencies
|
|
39
|
+
dependencies = []
|
|
40
|
+
dependencies_set = set()
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
# open the json file and load it into a dict
|
|
44
|
+
with open(lock_path, "r", encoding="utf-8") as file:
|
|
45
|
+
lock_data = json.load(file)
|
|
46
|
+
except (OSError, json.JSONDecodeError) as error:
|
|
47
|
+
raise RuntimeError(
|
|
48
|
+
f"Failed to read npm lock files: {lock_path}"
|
|
49
|
+
) from error
|
|
50
|
+
|
|
51
|
+
# check each item in the dict
|
|
52
|
+
for path, info in lock_data["packages"].items():
|
|
53
|
+
|
|
54
|
+
# if the item doesnt start with node_modules skip
|
|
55
|
+
if "node_modules/" not in path:
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
# split the name from the path
|
|
59
|
+
package_name = path.split("node_modules/")[-1]
|
|
60
|
+
|
|
61
|
+
# append the dependencies to the list
|
|
62
|
+
dependencies.append({
|
|
63
|
+
"name": package_name,
|
|
64
|
+
"version": info.get("version"),
|
|
65
|
+
"resolved": info.get("resolved"),
|
|
66
|
+
"integrity": info.get("integrity")
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
dependencies_set.add(f"{package_name}@{info.get("version")}")
|
|
70
|
+
|
|
71
|
+
return dependencies , dependencies_set
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def extract_uv_dependencies(lock_path):
|
|
75
|
+
"""
|
|
76
|
+
The function extracts uv dependencies from the lock file
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
# init list for dependencies
|
|
80
|
+
dependencies = []
|
|
81
|
+
dependencies_set = set()
|
|
82
|
+
try:
|
|
83
|
+
|
|
84
|
+
# open the uv.lock file and load it into a dict
|
|
85
|
+
with open(lock_path, "rb") as file:
|
|
86
|
+
lock_data = tomllib.load(file)
|
|
87
|
+
except (OSError, tomllib.TOMLDecodeError) as error:
|
|
88
|
+
raise RuntimeError(
|
|
89
|
+
f"Failed to read uv lock files {lock_path}"
|
|
90
|
+
) from error
|
|
91
|
+
|
|
92
|
+
# check each item in the dict
|
|
93
|
+
for package in lock_data.get("package", []):
|
|
94
|
+
|
|
95
|
+
# take the current package
|
|
96
|
+
package_data = {
|
|
97
|
+
"name": package.get("name"),
|
|
98
|
+
"version": package.get("version"),
|
|
99
|
+
"wheels": package.get("wheels", []),
|
|
100
|
+
"sdist": package.get("sdist")
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
# append to the dependencies list
|
|
104
|
+
dependencies.append(package_data)
|
|
105
|
+
dependencies_set.add(f"{package.get("name")}@{package.get("version")}")
|
|
106
|
+
|
|
107
|
+
return dependencies , dependencies_set
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def gather_npm_dependencies_from_locks(lock_files):
|
|
111
|
+
"""
|
|
112
|
+
The function takes a list of npm lock files and gather all of
|
|
113
|
+
the dependencies into a list and set
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
npm_dependencies = []
|
|
117
|
+
npm_set = set()
|
|
118
|
+
|
|
119
|
+
# gather the dependencies for each npm lockfile
|
|
120
|
+
for lock_file in lock_files:
|
|
121
|
+
|
|
122
|
+
# print message and extract dependencies
|
|
123
|
+
print(f"Processing npm: {lock_file}")
|
|
124
|
+
dependencies, dependencies_set = extract_npm_dependencies(lock_file)
|
|
125
|
+
|
|
126
|
+
# add the dependencies to the set and list
|
|
127
|
+
npm_dependencies.extend(dependencies)
|
|
128
|
+
npm_set.update(dependencies_set)
|
|
129
|
+
|
|
130
|
+
return npm_dependencies, npm_set
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def gather_uv_dependencies_from_lockfiles(lock_files):
|
|
134
|
+
"""
|
|
135
|
+
The function takes a list of uv lock files and gather all of
|
|
136
|
+
the dependencies into a list and set
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
uv_dependencies = []
|
|
140
|
+
uv_set = set()
|
|
141
|
+
|
|
142
|
+
for lock_file in lock_files:
|
|
143
|
+
|
|
144
|
+
# print message and extract dependencies
|
|
145
|
+
print(f"Processing uv: {lock_file}")
|
|
146
|
+
dependencies, dependencies_set = extract_uv_dependencies(lock_file)
|
|
147
|
+
|
|
148
|
+
# add the dependencies to the set and list
|
|
149
|
+
uv_dependencies.extend(dependencies)
|
|
150
|
+
uv_set.update(dependencies_set)
|
|
151
|
+
|
|
152
|
+
return uv_dependencies, uv_set
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import stat
|
|
4
|
+
import time
|
|
5
|
+
import tempfile
|
|
6
|
+
from functools import wraps
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def runtime_count(func):
|
|
11
|
+
@wraps(func)
|
|
12
|
+
def count(*args, **kwargs):
|
|
13
|
+
start_time = time.time()
|
|
14
|
+
result = func(*args, **kwargs)
|
|
15
|
+
end_time = time.time()
|
|
16
|
+
print(f"Total time: {end_time - start_time:.2f} seconds")
|
|
17
|
+
return result
|
|
18
|
+
return count
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def remove_readonly(func, path, exc):
|
|
22
|
+
"""
|
|
23
|
+
The function helps removing a full dir
|
|
24
|
+
"""
|
|
25
|
+
os.chmod(path, stat.S_IWRITE)
|
|
26
|
+
func(path)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@contextmanager
|
|
30
|
+
def temporary_directory():
|
|
31
|
+
"""
|
|
32
|
+
The function yields a temporary directory that is always removed on exit,
|
|
33
|
+
even when it holds read only files
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
path = tempfile.mkdtemp()
|
|
37
|
+
try:
|
|
38
|
+
yield path
|
|
39
|
+
finally:
|
|
40
|
+
shutil.rmtree(path, onexc=remove_readonly)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def validate_project_path(project_path):
|
|
44
|
+
"""
|
|
45
|
+
The function validates the path of the project for export
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# checks if the path exists
|
|
49
|
+
if not os.path.exists(project_path):
|
|
50
|
+
raise FileNotFoundError(f"Path does not exist: {project_path}")
|
|
51
|
+
|
|
52
|
+
# checks if the path is not a dir
|
|
53
|
+
if not os.path.isdir(project_path):
|
|
54
|
+
raise NotADirectoryError(f"Path is not a directory: {project_path}")
|
|
55
|
+
|
|
56
|
+
return project_path
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bundle-deps
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bundle a project's npm and uv dependencies for offline use
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: requests
|
|
8
|
+
Requires-Dist: tqdm
|
|
9
|
+
|
|
10
|
+
# bundle-deps
|
|
11
|
+
|
|
12
|
+
Bundle a project's npm and uv dependencies (from its lock files) for offline use.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
pip install bundle-deps # from PyPI
|
|
17
|
+
pip install git+<repo-url> # or straight from git
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
bundle-deps <project_path> [--commit-sha SHA | --latest-commit] [--manylinux]
|
|
22
|
+
[--no-pack] [--bundle-name NAME] [--output-dir DIR] [-v]
|
|
23
|
+
|
|
24
|
+
Also runnable as `python -m bundle_deps`.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
bundle_deps/__init__.py
|
|
4
|
+
bundle_deps/__main__.py
|
|
5
|
+
bundle_deps/archive.py
|
|
6
|
+
bundle_deps/cli.py
|
|
7
|
+
bundle_deps/download.py
|
|
8
|
+
bundle_deps/export.py
|
|
9
|
+
bundle_deps/git_source.py
|
|
10
|
+
bundle_deps/locks.py
|
|
11
|
+
bundle_deps/utils.py
|
|
12
|
+
bundle_deps.egg-info/PKG-INFO
|
|
13
|
+
bundle_deps.egg-info/SOURCES.txt
|
|
14
|
+
bundle_deps.egg-info/dependency_links.txt
|
|
15
|
+
bundle_deps.egg-info/entry_points.txt
|
|
16
|
+
bundle_deps.egg-info/requires.txt
|
|
17
|
+
bundle_deps.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
bundle_deps
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "bundle-deps"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Bundle a project's npm and uv dependencies for offline use"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = ["requests", "tqdm"]
|
|
12
|
+
|
|
13
|
+
[project.scripts]
|
|
14
|
+
bundle-deps = "bundle_deps.cli:main"
|
|
15
|
+
|
|
16
|
+
[tool.setuptools]
|
|
17
|
+
packages = ["bundle_deps"]
|