sideloader-ng 3.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sideloader_ng-3.1.3/PKG-INFO +321 -0
- sideloader_ng-3.1.3/README.md +303 -0
- sideloader_ng-3.1.3/pyproject.toml +29 -0
- sideloader_ng-3.1.3/src/sideloader/__init__.py +0 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/__init__.py +1 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/hookdeck/__init__.py +5 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/hookdeck/client.py +65 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/jsonbin/__init__.py +6 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/jsonbin/connector.py +193 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/jsonbin/manager.py +157 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/npm/__init__.py +24 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/npm/cleanup.py +120 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/npm/token_generator.py +1271 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/npm/uploader.py +125 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/pypi/__init__.py +18 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/pypi/cleanup.py +218 -0
- sideloader_ng-3.1.3/src/sideloader/adapters/pypi/uploader.py +34 -0
- sideloader_ng-3.1.3/src/sideloader/cli.py +1288 -0
- sideloader_ng-3.1.3/src/sideloader/scripts/cleanup_npm.py +36 -0
- sideloader_ng-3.1.3/src/sideloader/scripts/cleanup_pypi.py +119 -0
- sideloader_ng-3.1.3/src/sideloader/server.py +665 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: sideloader-ng
|
|
3
|
+
Version: 3.1.3
|
|
4
|
+
Summary: Download large files via PyPI packages
|
|
5
|
+
Author: NuVo
|
|
6
|
+
Author-email: NuVo <nullvoid@nullvoid.com>
|
|
7
|
+
Requires-Dist: build>=1.3.0
|
|
8
|
+
Requires-Dist: twine>=6.2.0
|
|
9
|
+
Requires-Dist: wheel>=0.45.1
|
|
10
|
+
Requires-Dist: rich>=13.0.0
|
|
11
|
+
Requires-Dist: httpx>=0.28.1
|
|
12
|
+
Requires-Dist: pip>=25.2
|
|
13
|
+
Requires-Dist: playwright>=1.55.0
|
|
14
|
+
Requires-Dist: pyotp>=2.9.0
|
|
15
|
+
Requires-Dist: requests>=2.32.0
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# ๐ Sideload
|
|
20
|
+
|
|
21
|
+
Download large files via PyPI or npm packages! Sideload automatically splits large files into package-compliant chunks and allows you to download them through a beautiful CLI interface.
|
|
22
|
+
|
|
23
|
+
## Features
|
|
24
|
+
|
|
25
|
+
- โจ **Beautiful CLI** powered by Rich with progress bars and status updates
|
|
26
|
+
- ๐ฆ **Automatic file splitting** into 95MB package-compliant chunks
|
|
27
|
+
- ๐ **Automatic reassembly** of downloaded parts
|
|
28
|
+
- ๐ฏ **Dual backend support** - Use PyPI or npm as temporary storage
|
|
29
|
+
- ๐ **JSONBin integration** for request tracking
|
|
30
|
+
- ๐ก๏ธ **Error handling** with detailed progress monitoring
|
|
31
|
+
- ๐งน **Easy cleanup** - npm packages can be unpublished (much easier than PyPI!)
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
git clone <repository-url>
|
|
37
|
+
cd Sideload
|
|
38
|
+
uv install
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
### Environment Variables
|
|
44
|
+
|
|
45
|
+
Set up your credentials:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Required
|
|
49
|
+
export JSONBIN_TOKEN="your_jsonbin_token"
|
|
50
|
+
export SIDELOAD_COLLECTION_ID="your_collection_id"
|
|
51
|
+
|
|
52
|
+
# Choose storage backend (default: pypi)
|
|
53
|
+
export STORAGE_BACKEND="npm" # or "pypi"
|
|
54
|
+
|
|
55
|
+
# For PyPI backend
|
|
56
|
+
export PYPI_TOKEN="your_pypi_token"
|
|
57
|
+
export PYPI_USER="your_pypi_username"
|
|
58
|
+
export PYPI_PASSWORD="your_pypi_password"
|
|
59
|
+
export PYPI_TOTP="your_totp_secret" # Optional, for 2FA
|
|
60
|
+
|
|
61
|
+
# For npm backend
|
|
62
|
+
export NPM_REGISTRY="https://registry.npmjs.org/" # Optional, defaults to npmjs.org
|
|
63
|
+
export NPM_BASE_PACKAGE="sideloader-base-pkg" # Existing npm package used for all sideload payloads
|
|
64
|
+
|
|
65
|
+
# npm token regeneration (used when sideloader-npm-token cache is missing/expired)
|
|
66
|
+
export NPM_USER="your_npm_username"
|
|
67
|
+
export NPM_PASSWORD="your_npm_password"
|
|
68
|
+
export NPM_TOKEN_SECRET_NAME="sideloader-npm-token" # Kubernetes secret cache name
|
|
69
|
+
export NPM_TOKEN_EXPIRATION_DAYS="90"
|
|
70
|
+
export NPM_PLAYWRIGHT_HEADLESS="false" # Optional: run headed browser
|
|
71
|
+
export NPM_PLAYWRIGHT_USE_XVFB="true" # Optional: use virtual framebuffer when headed
|
|
72
|
+
export NPM_XVFB_DISPLAY=":99" # Optional
|
|
73
|
+
export NPM_XVFB_SCREEN="1920x1080x24" # Optional
|
|
74
|
+
export NPM_PLAYWRIGHT_DEBUG="true" # Optional: verbose Playwright logs
|
|
75
|
+
export NPM_PLAYWRIGHT_DEBUG_DIR="/tmp" # Optional: screenshots on failures/challenges
|
|
76
|
+
|
|
77
|
+
# Mail OTP extraction (IMAP inbox checked by sideloader when npm asks OTP)
|
|
78
|
+
export NPM_OTP_EMAIL_IMAP_HOST="imap.your-provider.com"
|
|
79
|
+
export NPM_OTP_EMAIL_IMAP_PORT="993"
|
|
80
|
+
export NPM_OTP_EMAIL_USERNAME="bot-mailbox@example.com"
|
|
81
|
+
export NPM_OTP_EMAIL_PASSWORD="your_mailbox_password"
|
|
82
|
+
export NPM_OTP_EMAIL_FOLDER="INBOX" # Optional
|
|
83
|
+
export NPM_OTP_EMAIL_FROM="support@npmjs.com" # Optional
|
|
84
|
+
export NPM_OTP_EMAIL_SUBJECT="Your npm one-time password" # Optional
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
When running the server Docker image, headed Playwright with `NPM_PLAYWRIGHT_USE_XVFB=true`
|
|
88
|
+
is automatically wrapped in `xvfb-run` by the container entrypoint.
|
|
89
|
+
|
|
90
|
+
> **npm token caching behavior**:
|
|
91
|
+
> - sideloader reads token cache from Kubernetes secret `sideloader-npm-token`
|
|
92
|
+
> - if the cached token is expired/near-expiry, sideloader regenerates a token via Playwright
|
|
93
|
+
> - OTP is read from email inbox (IMAP) and extracted automatically
|
|
94
|
+
> - regenerated token is stored in the secret with `created_at` and `expires_at` (+90 days by default)
|
|
95
|
+
|
|
96
|
+
### Storage Backend Comparison
|
|
97
|
+
|
|
98
|
+
| Feature | PyPI | npm |
|
|
99
|
+
|---------|------|-----|
|
|
100
|
+
| Package limit | 100 MB | Unlimited |
|
|
101
|
+
| Cleanup | Browser automation required | Simple CLI command |
|
|
102
|
+
| Unpublish | Complex, requires 2FA | Easy with `npm unpublish` |
|
|
103
|
+
| Token Management | Manual (varies) | Auto-regenerated + cached in Kubernetes secret |
|
|
104
|
+
| Speed | Fast | Fast |
|
|
105
|
+
| Recommended for | Single-use uploads | Frequent testing |
|
|
106
|
+
|
|
107
|
+
**npm is recommended** for development and testing due to its much simpler cleanup process!
|
|
108
|
+
|
|
109
|
+
### Download a File
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
# Basic usage
|
|
113
|
+
uv run sideload download https://example.com/largefile.zip
|
|
114
|
+
|
|
115
|
+
# Specify output directory
|
|
116
|
+
uv run sideload download https://example.com/largefile.zip --output ./downloads/
|
|
117
|
+
|
|
118
|
+
# Override credentials
|
|
119
|
+
uv run sideload download https://example.com/largefile.zip --token YOUR_TOKEN --collection YOUR_COLLECTION
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### How it Works
|
|
123
|
+
|
|
124
|
+
1. **Submit Request**: The CLI creates a new request in your JSONBin collection
|
|
125
|
+
2. **Monitor Progress**: Real-time progress monitoring with beautiful progress bars
|
|
126
|
+
3. **Server Processing**: Server downloads the file and splits it into chunks
|
|
127
|
+
4. **Upload to Storage**: Chunks are uploaded to PyPI or npm (depending on `STORAGE_BACKEND`)
|
|
128
|
+
5. **Download Packages**: CLI automatically downloads all packages containing file parts
|
|
129
|
+
6. **Reassemble**: Extracts and concatenates parts to rebuild the original file
|
|
130
|
+
7. **Cleanup**: Use cleanup scripts to remove packages after download
|
|
131
|
+
|
|
132
|
+
### Cleanup
|
|
133
|
+
|
|
134
|
+
#### npm (Recommended - Much Easier!)
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
# Clean up specific packages
|
|
138
|
+
python src/sideloader/scripts/cleanup_npm.py sideloader-base-pkg@1.1700000000.123 sideloader-base-pkg@1.1700000000.124
|
|
139
|
+
|
|
140
|
+
# Or use the server's built-in cleanup
|
|
141
|
+
uv run sideloader-server --clean-request-id YOUR_BIN_ID
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
npm packages can be easily unpublished using the npm CLI, making cleanup much simpler than PyPI!
|
|
145
|
+
|
|
146
|
+
#### PyPI (Requires Browser Automation)
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
# Clean up specific packages
|
|
150
|
+
python src/sideloader/scripts/cleanup_pypi.py package_name1 package_name2
|
|
151
|
+
|
|
152
|
+
# Or use the server's built-in cleanup
|
|
153
|
+
uv run sideloader-server --clean-request-id YOUR_BIN_ID
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
PyPI cleanup uses Playwright browser automation since PyPI doesn't provide a simple unpublish API.
|
|
157
|
+
|
|
158
|
+
### CLI Interface
|
|
159
|
+
|
|
160
|
+
The CLI provides:
|
|
161
|
+
|
|
162
|
+
- ๐ **Colorful output** with status indicators
|
|
163
|
+
- ๐ **Progress bars** for downloads and processing
|
|
164
|
+
- ๐ **Real-time monitoring** of server-side processing
|
|
165
|
+
- โ
**Success/error reporting** with detailed information
|
|
166
|
+
- ๐ **Summary tables** showing download statistics
|
|
167
|
+
|
|
168
|
+
### Example Output
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
๐ SIDELOAD
|
|
172
|
+
Download large files via PyPI/npm packages
|
|
173
|
+
|
|
174
|
+
๐ Requesting download for: https://example.com/largefile.zip
|
|
175
|
+
โ
Created sideload request: abc123def456
|
|
176
|
+
|
|
177
|
+
๐ก Monitoring Progress
|
|
178
|
+
๐ฅ Downloading... (45%) โโโโโโโโโโโโโโโโโ
|
|
179
|
+
๐จ Building packages...
|
|
180
|
+
๐ค Uploading part 1/3...
|
|
181
|
+
|
|
182
|
+
๐ Download Summary
|
|
183
|
+
โโโโโโโโโโโโโโโโโโโโโณโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
184
|
+
โ Property โ Value โ
|
|
185
|
+
โกโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฉ
|
|
186
|
+
โ Original Filename โ largefile.zip โ
|
|
187
|
+
โ File Size โ 250,123,456 bytes โ
|
|
188
|
+
โ Total Packages โ 3 โ
|
|
189
|
+
โ Status โ โ
UPLOADED โ
|
|
190
|
+
โโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
191
|
+
|
|
192
|
+
๐ฆ Downloading Packages
|
|
193
|
+
๐ฆ Downloading package 1/3... โโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
194
|
+
|
|
195
|
+
๐ง Reassembling File
|
|
196
|
+
๐ Assembling part 1/3... โโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
197
|
+
|
|
198
|
+
โจ Complete
|
|
199
|
+
๐ File successfully downloaded to: largefile.zip
|
|
200
|
+
๐ File size: 250,123,456 bytes
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Development
|
|
204
|
+
|
|
205
|
+
### Server Setup
|
|
206
|
+
|
|
207
|
+
The server component handles file processing and PyPI uploads:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
# Set environment variables
|
|
211
|
+
export JSONBIN_TOKEN="your_token"
|
|
212
|
+
export PYPI_TOKEN="your_pypi_token"
|
|
213
|
+
|
|
214
|
+
# Run the server
|
|
215
|
+
uv run python src/sideloader/main.py
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### Kubernetes Deployment
|
|
219
|
+
|
|
220
|
+
Before installing the Helm chart, create the required secret in your namespace:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
kubectl create secret generic sideload-secrets \
|
|
224
|
+
--namespace <namespace> \
|
|
225
|
+
--from-literal=jsonbin-token="YOUR_JSONBIN_TOKEN" \
|
|
226
|
+
--from-literal=pypi-token="YOUR_PYPI_TOKEN" \
|
|
227
|
+
--from-literal=pypi-user="YOUR_PYPI_USERNAME" \
|
|
228
|
+
--from-literal=pypi-password="YOUR_PYPI_PASSWORD" \
|
|
229
|
+
--from-literal=pypi-totp="YOUR_PYPI_TOTP_SECRET" \
|
|
230
|
+
--from-literal=npm-token="OPTIONAL_EXISTING_NPM_TOKEN" \
|
|
231
|
+
--from-literal=npm-user="YOUR_NPM_USERNAME" \
|
|
232
|
+
--from-literal=npm-password="YOUR_NPM_PASSWORD" \
|
|
233
|
+
--from-literal=npm-otp-email-imap-host="imap.your-provider.com" \
|
|
234
|
+
--from-literal=npm-otp-email-imap-port="993" \
|
|
235
|
+
--from-literal=npm-otp-email-username="bot-mailbox@example.com" \
|
|
236
|
+
--from-literal=npm-otp-email-password="your_mailbox_password" \
|
|
237
|
+
--from-literal=npm-otp-email-folder="INBOX" \
|
|
238
|
+
--from-literal=npm-otp-email-from="support@npmjs.com" \
|
|
239
|
+
--from-literal=npm-otp-email-subject="Your npm one-time password"
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
`sideloader-npm-token` is managed automatically by sideloader and does not need manual creation.
|
|
243
|
+
If you want to create it yourself, `--dry-run-npm-token` now prints a full Secret YAML manifest.
|
|
244
|
+
|
|
245
|
+
### npm Token Dry-Run
|
|
246
|
+
|
|
247
|
+
Use this to validate cache read/regenerate/write behavior without processing a sideload request:
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
# Standard flow: use cached token if valid, otherwise regenerate/write cache
|
|
251
|
+
uv run sideloader-server --dry-run-npm-token
|
|
252
|
+
|
|
253
|
+
# Force regeneration flow (Playwright + mail OTP + cache write)
|
|
254
|
+
uv run sideloader-server --dry-run-npm-token --force-regenerate-npm-token
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Docker-based dry-run test (with virtual framebuffer):
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
# Build test image
|
|
261
|
+
docker build -f Dockerfile.npm-dryrun -t sideloader-npm-dryrun .
|
|
262
|
+
|
|
263
|
+
# Run dry-run + force regeneration
|
|
264
|
+
docker run --rm -it \
|
|
265
|
+
-e PYTHONUNBUFFERED=true \
|
|
266
|
+
-e NPM_USER="your_npm_username" \
|
|
267
|
+
-e NPM_PASSWORD="your_npm_password" \
|
|
268
|
+
-e NPM_OTP_EMAIL_IMAP_HOST="imap.gmail.com" \
|
|
269
|
+
-e NPM_OTP_EMAIL_IMAP_PORT="993" \
|
|
270
|
+
-e NPM_OTP_EMAIL_USERNAME="your.address@gmail.com" \
|
|
271
|
+
-e NPM_OTP_EMAIL_PASSWORD="your_16_char_google_app_password" \
|
|
272
|
+
-e NPM_OTP_EMAIL_FROM="support@npmjs.com" \
|
|
273
|
+
-e NPM_OTP_EMAIL_SUBJECT="Your npm one-time password" \
|
|
274
|
+
sideloader-npm-dryrun \
|
|
275
|
+
--dry-run-npm-token --force-regenerate-npm-token
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Note: this local Docker run is not in-cluster Kubernetes, so it will print Secret YAML for manual apply instead of writing to Kubernetes automatically.
|
|
279
|
+
|
|
280
|
+
### npm Publish/Unpublish Test Commands
|
|
281
|
+
|
|
282
|
+
Use these to test npm publishing with an existing base package:
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
# Upload a local file as one npm artifact
|
|
286
|
+
uv run sideloader-server --npm-test-upload-file /absolute/path/to/file.zip
|
|
287
|
+
|
|
288
|
+
# Optional custom dist-tag
|
|
289
|
+
uv run sideloader-server --npm-test-upload-file /absolute/path/to/file.zip --npm-test-tag sideload-docker-compose-f249932
|
|
290
|
+
|
|
291
|
+
# Unpublish a specific uploaded version
|
|
292
|
+
uv run sideloader-server --npm-test-unpublish sideloader-base-pkg@1.1700000000.123456
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Then install the chart:
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
helm install sideload ./helm --namespace <namespace>
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
For npm token regeneration with headed Playwright + Xvfb in Kubernetes, ensure Helm keeps:
|
|
302
|
+
- `serverRunAsRoot: true`
|
|
303
|
+
- `npm.playwrightHeadless: false`
|
|
304
|
+
- `npm.playwrightUseXvfb: true`
|
|
305
|
+
|
|
306
|
+
### Project Structure
|
|
307
|
+
|
|
308
|
+
```
|
|
309
|
+
src/sideloader/
|
|
310
|
+
โโโ __init__.py # Package initialization
|
|
311
|
+
โโโ cli.py # CLI client
|
|
312
|
+
โโโ server.py # Server component
|
|
313
|
+
โโโ jsonbin_connector.py # JSONBin API wrapper
|
|
314
|
+
โโโ pypi_cleanup.py # PyPI package deletion via Playwright
|
|
315
|
+
โโโ scripts/
|
|
316
|
+
โโโ cleanup_pypi.py # Admin script to delete all sideload packages
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
## License
|
|
320
|
+
|
|
321
|
+
[Your License Here]
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# ๐ Sideload
|
|
2
|
+
|
|
3
|
+
Download large files via PyPI or npm packages! Sideload automatically splits large files into package-compliant chunks and allows you to download them through a beautiful CLI interface.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- โจ **Beautiful CLI** powered by Rich with progress bars and status updates
|
|
8
|
+
- ๐ฆ **Automatic file splitting** into 95MB package-compliant chunks
|
|
9
|
+
- ๐ **Automatic reassembly** of downloaded parts
|
|
10
|
+
- ๐ฏ **Dual backend support** - Use PyPI or npm as temporary storage
|
|
11
|
+
- ๐ **JSONBin integration** for request tracking
|
|
12
|
+
- ๐ก๏ธ **Error handling** with detailed progress monitoring
|
|
13
|
+
- ๐งน **Easy cleanup** - npm packages can be unpublished (much easier than PyPI!)
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
git clone <repository-url>
|
|
19
|
+
cd Sideload
|
|
20
|
+
uv install
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
### Environment Variables
|
|
26
|
+
|
|
27
|
+
Set up your credentials:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
# Required
|
|
31
|
+
export JSONBIN_TOKEN="your_jsonbin_token"
|
|
32
|
+
export SIDELOAD_COLLECTION_ID="your_collection_id"
|
|
33
|
+
|
|
34
|
+
# Choose storage backend (default: pypi)
|
|
35
|
+
export STORAGE_BACKEND="npm" # or "pypi"
|
|
36
|
+
|
|
37
|
+
# For PyPI backend
|
|
38
|
+
export PYPI_TOKEN="your_pypi_token"
|
|
39
|
+
export PYPI_USER="your_pypi_username"
|
|
40
|
+
export PYPI_PASSWORD="your_pypi_password"
|
|
41
|
+
export PYPI_TOTP="your_totp_secret" # Optional, for 2FA
|
|
42
|
+
|
|
43
|
+
# For npm backend
|
|
44
|
+
export NPM_REGISTRY="https://registry.npmjs.org/" # Optional, defaults to npmjs.org
|
|
45
|
+
export NPM_BASE_PACKAGE="sideloader-base-pkg" # Existing npm package used for all sideload payloads
|
|
46
|
+
|
|
47
|
+
# npm token regeneration (used when sideloader-npm-token cache is missing/expired)
|
|
48
|
+
export NPM_USER="your_npm_username"
|
|
49
|
+
export NPM_PASSWORD="your_npm_password"
|
|
50
|
+
export NPM_TOKEN_SECRET_NAME="sideloader-npm-token" # Kubernetes secret cache name
|
|
51
|
+
export NPM_TOKEN_EXPIRATION_DAYS="90"
|
|
52
|
+
export NPM_PLAYWRIGHT_HEADLESS="false" # Optional: run headed browser
|
|
53
|
+
export NPM_PLAYWRIGHT_USE_XVFB="true" # Optional: use virtual framebuffer when headed
|
|
54
|
+
export NPM_XVFB_DISPLAY=":99" # Optional
|
|
55
|
+
export NPM_XVFB_SCREEN="1920x1080x24" # Optional
|
|
56
|
+
export NPM_PLAYWRIGHT_DEBUG="true" # Optional: verbose Playwright logs
|
|
57
|
+
export NPM_PLAYWRIGHT_DEBUG_DIR="/tmp" # Optional: screenshots on failures/challenges
|
|
58
|
+
|
|
59
|
+
# Mail OTP extraction (IMAP inbox checked by sideloader when npm asks OTP)
|
|
60
|
+
export NPM_OTP_EMAIL_IMAP_HOST="imap.your-provider.com"
|
|
61
|
+
export NPM_OTP_EMAIL_IMAP_PORT="993"
|
|
62
|
+
export NPM_OTP_EMAIL_USERNAME="bot-mailbox@example.com"
|
|
63
|
+
export NPM_OTP_EMAIL_PASSWORD="your_mailbox_password"
|
|
64
|
+
export NPM_OTP_EMAIL_FOLDER="INBOX" # Optional
|
|
65
|
+
export NPM_OTP_EMAIL_FROM="support@npmjs.com" # Optional
|
|
66
|
+
export NPM_OTP_EMAIL_SUBJECT="Your npm one-time password" # Optional
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
When running the server Docker image, headed Playwright with `NPM_PLAYWRIGHT_USE_XVFB=true`
|
|
70
|
+
is automatically wrapped in `xvfb-run` by the container entrypoint.
|
|
71
|
+
|
|
72
|
+
> **npm token caching behavior**:
|
|
73
|
+
> - sideloader reads token cache from Kubernetes secret `sideloader-npm-token`
|
|
74
|
+
> - if the cached token is expired/near-expiry, sideloader regenerates a token via Playwright
|
|
75
|
+
> - OTP is read from email inbox (IMAP) and extracted automatically
|
|
76
|
+
> - regenerated token is stored in the secret with `created_at` and `expires_at` (+90 days by default)
|
|
77
|
+
|
|
78
|
+
### Storage Backend Comparison
|
|
79
|
+
|
|
80
|
+
| Feature | PyPI | npm |
|
|
81
|
+
|---------|------|-----|
|
|
82
|
+
| Package limit | 100 MB | Unlimited |
|
|
83
|
+
| Cleanup | Browser automation required | Simple CLI command |
|
|
84
|
+
| Unpublish | Complex, requires 2FA | Easy with `npm unpublish` |
|
|
85
|
+
| Token Management | Manual (varies) | Auto-regenerated + cached in Kubernetes secret |
|
|
86
|
+
| Speed | Fast | Fast |
|
|
87
|
+
| Recommended for | Single-use uploads | Frequent testing |
|
|
88
|
+
|
|
89
|
+
**npm is recommended** for development and testing due to its much simpler cleanup process!
|
|
90
|
+
|
|
91
|
+
### Download a File
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
# Basic usage
|
|
95
|
+
uv run sideload download https://example.com/largefile.zip
|
|
96
|
+
|
|
97
|
+
# Specify output directory
|
|
98
|
+
uv run sideload download https://example.com/largefile.zip --output ./downloads/
|
|
99
|
+
|
|
100
|
+
# Override credentials
|
|
101
|
+
uv run sideload download https://example.com/largefile.zip --token YOUR_TOKEN --collection YOUR_COLLECTION
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### How it Works
|
|
105
|
+
|
|
106
|
+
1. **Submit Request**: The CLI creates a new request in your JSONBin collection
|
|
107
|
+
2. **Monitor Progress**: Real-time progress monitoring with beautiful progress bars
|
|
108
|
+
3. **Server Processing**: Server downloads the file and splits it into chunks
|
|
109
|
+
4. **Upload to Storage**: Chunks are uploaded to PyPI or npm (depending on `STORAGE_BACKEND`)
|
|
110
|
+
5. **Download Packages**: CLI automatically downloads all packages containing file parts
|
|
111
|
+
6. **Reassemble**: Extracts and concatenates parts to rebuild the original file
|
|
112
|
+
7. **Cleanup**: Use cleanup scripts to remove packages after download
|
|
113
|
+
|
|
114
|
+
### Cleanup
|
|
115
|
+
|
|
116
|
+
#### npm (Recommended - Much Easier!)
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
# Clean up specific packages
|
|
120
|
+
python src/sideloader/scripts/cleanup_npm.py sideloader-base-pkg@1.1700000000.123 sideloader-base-pkg@1.1700000000.124
|
|
121
|
+
|
|
122
|
+
# Or use the server's built-in cleanup
|
|
123
|
+
uv run sideloader-server --clean-request-id YOUR_BIN_ID
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
npm packages can be easily unpublished using the npm CLI, making cleanup much simpler than PyPI!
|
|
127
|
+
|
|
128
|
+
#### PyPI (Requires Browser Automation)
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# Clean up specific packages
|
|
132
|
+
python src/sideloader/scripts/cleanup_pypi.py package_name1 package_name2
|
|
133
|
+
|
|
134
|
+
# Or use the server's built-in cleanup
|
|
135
|
+
uv run sideloader-server --clean-request-id YOUR_BIN_ID
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
PyPI cleanup uses Playwright browser automation since PyPI doesn't provide a simple unpublish API.
|
|
139
|
+
|
|
140
|
+
### CLI Interface
|
|
141
|
+
|
|
142
|
+
The CLI provides:
|
|
143
|
+
|
|
144
|
+
- ๐ **Colorful output** with status indicators
|
|
145
|
+
- ๐ **Progress bars** for downloads and processing
|
|
146
|
+
- ๐ **Real-time monitoring** of server-side processing
|
|
147
|
+
- โ
**Success/error reporting** with detailed information
|
|
148
|
+
- ๐ **Summary tables** showing download statistics
|
|
149
|
+
|
|
150
|
+
### Example Output
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
๐ SIDELOAD
|
|
154
|
+
Download large files via PyPI/npm packages
|
|
155
|
+
|
|
156
|
+
๐ Requesting download for: https://example.com/largefile.zip
|
|
157
|
+
โ
Created sideload request: abc123def456
|
|
158
|
+
|
|
159
|
+
๐ก Monitoring Progress
|
|
160
|
+
๐ฅ Downloading... (45%) โโโโโโโโโโโโโโโโโ
|
|
161
|
+
๐จ Building packages...
|
|
162
|
+
๐ค Uploading part 1/3...
|
|
163
|
+
|
|
164
|
+
๐ Download Summary
|
|
165
|
+
โโโโโโโโโโโโโโโโโโโโโณโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
166
|
+
โ Property โ Value โ
|
|
167
|
+
โกโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโฉ
|
|
168
|
+
โ Original Filename โ largefile.zip โ
|
|
169
|
+
โ File Size โ 250,123,456 bytes โ
|
|
170
|
+
โ Total Packages โ 3 โ
|
|
171
|
+
โ Status โ โ
UPLOADED โ
|
|
172
|
+
โโโโโโโโโโโโโโโโโโโโโดโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
173
|
+
|
|
174
|
+
๐ฆ Downloading Packages
|
|
175
|
+
๐ฆ Downloading package 1/3... โโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
176
|
+
|
|
177
|
+
๐ง Reassembling File
|
|
178
|
+
๐ Assembling part 1/3... โโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
179
|
+
|
|
180
|
+
โจ Complete
|
|
181
|
+
๐ File successfully downloaded to: largefile.zip
|
|
182
|
+
๐ File size: 250,123,456 bytes
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Development
|
|
186
|
+
|
|
187
|
+
### Server Setup
|
|
188
|
+
|
|
189
|
+
The server component handles file processing and PyPI uploads:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
# Set environment variables
|
|
193
|
+
export JSONBIN_TOKEN="your_token"
|
|
194
|
+
export PYPI_TOKEN="your_pypi_token"
|
|
195
|
+
|
|
196
|
+
# Run the server
|
|
197
|
+
uv run python src/sideloader/main.py
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### Kubernetes Deployment
|
|
201
|
+
|
|
202
|
+
Before installing the Helm chart, create the required secret in your namespace:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
kubectl create secret generic sideload-secrets \
|
|
206
|
+
--namespace <namespace> \
|
|
207
|
+
--from-literal=jsonbin-token="YOUR_JSONBIN_TOKEN" \
|
|
208
|
+
--from-literal=pypi-token="YOUR_PYPI_TOKEN" \
|
|
209
|
+
--from-literal=pypi-user="YOUR_PYPI_USERNAME" \
|
|
210
|
+
--from-literal=pypi-password="YOUR_PYPI_PASSWORD" \
|
|
211
|
+
--from-literal=pypi-totp="YOUR_PYPI_TOTP_SECRET" \
|
|
212
|
+
--from-literal=npm-token="OPTIONAL_EXISTING_NPM_TOKEN" \
|
|
213
|
+
--from-literal=npm-user="YOUR_NPM_USERNAME" \
|
|
214
|
+
--from-literal=npm-password="YOUR_NPM_PASSWORD" \
|
|
215
|
+
--from-literal=npm-otp-email-imap-host="imap.your-provider.com" \
|
|
216
|
+
--from-literal=npm-otp-email-imap-port="993" \
|
|
217
|
+
--from-literal=npm-otp-email-username="bot-mailbox@example.com" \
|
|
218
|
+
--from-literal=npm-otp-email-password="your_mailbox_password" \
|
|
219
|
+
--from-literal=npm-otp-email-folder="INBOX" \
|
|
220
|
+
--from-literal=npm-otp-email-from="support@npmjs.com" \
|
|
221
|
+
--from-literal=npm-otp-email-subject="Your npm one-time password"
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
`sideloader-npm-token` is managed automatically by sideloader and does not need manual creation.
|
|
225
|
+
If you want to create it yourself, `--dry-run-npm-token` now prints a full Secret YAML manifest.
|
|
226
|
+
|
|
227
|
+
### npm Token Dry-Run
|
|
228
|
+
|
|
229
|
+
Use this to validate cache read/regenerate/write behavior without processing a sideload request:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
# Standard flow: use cached token if valid, otherwise regenerate/write cache
|
|
233
|
+
uv run sideloader-server --dry-run-npm-token
|
|
234
|
+
|
|
235
|
+
# Force regeneration flow (Playwright + mail OTP + cache write)
|
|
236
|
+
uv run sideloader-server --dry-run-npm-token --force-regenerate-npm-token
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
Docker-based dry-run test (with virtual framebuffer):
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
# Build test image
|
|
243
|
+
docker build -f Dockerfile.npm-dryrun -t sideloader-npm-dryrun .
|
|
244
|
+
|
|
245
|
+
# Run dry-run + force regeneration
|
|
246
|
+
docker run --rm -it \
|
|
247
|
+
-e PYTHONUNBUFFERED=true \
|
|
248
|
+
-e NPM_USER="your_npm_username" \
|
|
249
|
+
-e NPM_PASSWORD="your_npm_password" \
|
|
250
|
+
-e NPM_OTP_EMAIL_IMAP_HOST="imap.gmail.com" \
|
|
251
|
+
-e NPM_OTP_EMAIL_IMAP_PORT="993" \
|
|
252
|
+
-e NPM_OTP_EMAIL_USERNAME="your.address@gmail.com" \
|
|
253
|
+
-e NPM_OTP_EMAIL_PASSWORD="your_16_char_google_app_password" \
|
|
254
|
+
-e NPM_OTP_EMAIL_FROM="support@npmjs.com" \
|
|
255
|
+
-e NPM_OTP_EMAIL_SUBJECT="Your npm one-time password" \
|
|
256
|
+
sideloader-npm-dryrun \
|
|
257
|
+
--dry-run-npm-token --force-regenerate-npm-token
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
Note: this local Docker run is not in-cluster Kubernetes, so it will print Secret YAML for manual apply instead of writing to Kubernetes automatically.
|
|
261
|
+
|
|
262
|
+
### npm Publish/Unpublish Test Commands
|
|
263
|
+
|
|
264
|
+
Use these to test npm publishing with an existing base package:
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
# Upload a local file as one npm artifact
|
|
268
|
+
uv run sideloader-server --npm-test-upload-file /absolute/path/to/file.zip
|
|
269
|
+
|
|
270
|
+
# Optional custom dist-tag
|
|
271
|
+
uv run sideloader-server --npm-test-upload-file /absolute/path/to/file.zip --npm-test-tag sideload-docker-compose-f249932
|
|
272
|
+
|
|
273
|
+
# Unpublish a specific uploaded version
|
|
274
|
+
uv run sideloader-server --npm-test-unpublish sideloader-base-pkg@1.1700000000.123456
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
Then install the chart:
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
helm install sideload ./helm --namespace <namespace>
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
For npm token regeneration with headed Playwright + Xvfb in Kubernetes, ensure Helm keeps:
|
|
284
|
+
- `serverRunAsRoot: true`
|
|
285
|
+
- `npm.playwrightHeadless: false`
|
|
286
|
+
- `npm.playwrightUseXvfb: true`
|
|
287
|
+
|
|
288
|
+
### Project Structure
|
|
289
|
+
|
|
290
|
+
```
|
|
291
|
+
src/sideloader/
|
|
292
|
+
โโโ __init__.py # Package initialization
|
|
293
|
+
โโโ cli.py # CLI client
|
|
294
|
+
โโโ server.py # Server component
|
|
295
|
+
โโโ jsonbin_connector.py # JSONBin API wrapper
|
|
296
|
+
โโโ pypi_cleanup.py # PyPI package deletion via Playwright
|
|
297
|
+
โโโ scripts/
|
|
298
|
+
โโโ cleanup_pypi.py # Admin script to delete all sideload packages
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
## License
|
|
302
|
+
|
|
303
|
+
[Your License Here]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sideloader-ng"
|
|
3
|
+
version = "3.1.3"
|
|
4
|
+
description = "Download large files via PyPI packages"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [{ name = "NuVo", email = "nullvoid@nullvoid.com" }]
|
|
7
|
+
requires-python = ">=3.12"
|
|
8
|
+
dependencies = [
|
|
9
|
+
"build>=1.3.0",
|
|
10
|
+
"twine>=6.2.0",
|
|
11
|
+
"wheel>=0.45.1",
|
|
12
|
+
"rich>=13.0.0",
|
|
13
|
+
"httpx>=0.28.1",
|
|
14
|
+
"pip>=25.2",
|
|
15
|
+
"playwright>=1.55.0",
|
|
16
|
+
"pyotp>=2.9.0",
|
|
17
|
+
"requests>=2.32.0",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.scripts]
|
|
21
|
+
sideloader = "sideloader.cli:cli_main"
|
|
22
|
+
sideloader-server = "sideloader.server:server_main"
|
|
23
|
+
|
|
24
|
+
[tool.uv.build-backend]
|
|
25
|
+
module-name = "sideloader"
|
|
26
|
+
|
|
27
|
+
[build-system]
|
|
28
|
+
requires = ["uv_build>=0.8.15,<0.9.0"]
|
|
29
|
+
build-backend = "uv_build"
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Adapters module - External service integrations for Sideload."""
|