copyparty 1.12.2__py3-none-any.whl → 1.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- copyparty/__main__.py +4 -2
- copyparty/__version__.py +3 -3
- copyparty/cfg.py +1 -0
- copyparty/httpcli.py +174 -3
- copyparty/httpconn.py +1 -0
- copyparty/httpsrv.py +2 -0
- copyparty/up2k.py +115 -52
- copyparty/util.py +37 -3
- copyparty/web/a/u2c.py +8 -4
- copyparty/web/browser.css.gz +0 -0
- copyparty/web/browser.js.gz +0 -0
- copyparty/web/up2k.js.gz +0 -0
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/METADATA +17 -4
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/RECORD +18 -18
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/LICENSE +0 -0
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/WHEEL +0 -0
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/entry_points.txt +0 -0
- {copyparty-1.12.2.dist-info → copyparty-1.13.0.dist-info}/top_level.txt +0 -0
copyparty/__main__.py
CHANGED
@@ -850,7 +850,7 @@ def add_qr(ap, tty):
|
|
850
850
|
|
851
851
|
def add_fs(ap):
|
852
852
|
ap2 = ap.add_argument_group("filesystem options")
|
853
|
-
rm_re_def = "
|
853
|
+
rm_re_def = "15/0.1" if ANYWIN else "0/0"
|
854
854
|
ap2.add_argument("--rm-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be deleted because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=rm_retry)")
|
855
855
|
ap2.add_argument("--mv-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be renamed because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=mv_retry)")
|
856
856
|
ap2.add_argument("--iobuf", metavar="BYTES", type=int, default=256*1024, help="file I/O buffer-size; if your volumes are on a network drive, try increasing to \033[32m524288\033[0m or even \033[32m4194304\033[0m (and let me know if that improves your performance)")
|
@@ -1085,6 +1085,8 @@ def add_optouts(ap):
|
|
1085
1085
|
ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
|
1086
1086
|
ap2.add_argument("--no-tarcmp", action="store_true", help="disable download as compressed tar (?tar=gz, ?tar=bz2, ?tar=xz, ?tar=gz:9, ...)")
|
1087
1087
|
ap2.add_argument("--no-lifetime", action="store_true", help="do not allow clients (or server config) to schedule an upload to be deleted after a given time")
|
1088
|
+
ap2.add_argument("--no-pipe", action="store_true", help="disable race-the-beam (lockstep download of files which are currently being uploaded) (volflag=nopipe)")
|
1089
|
+
ap2.add_argument("--no-db-ip", action="store_true", help="do not write uploader IPs into the database")
|
1088
1090
|
|
1089
1091
|
|
1090
1092
|
def add_safety(ap):
|
@@ -1210,7 +1212,7 @@ def add_db_general(ap, hcores):
|
|
1210
1212
|
ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching absolute-filesystem-paths during e2ds folder scans (volflag=nohash)")
|
1211
1213
|
ap2.add_argument("--no-idx", metavar="PTN", type=u, default=noidx, help="regex: disable indexing of matching absolute-filesystem-paths during e2ds folder scans (volflag=noidx)")
|
1212
1214
|
ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
|
1213
|
-
ap2.add_argument("--re-dhash", action="store_true", help="
|
1215
|
+
ap2.add_argument("--re-dhash", action="store_true", help="force a cache rebuild on startup; enable this once if it gets out of sync (should never be necessary)")
|
1214
1216
|
ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice -- only useful for offloading uploads to a cloud service or something (volflag=noforget)")
|
1215
1217
|
ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see \033[33m--help-dbd\033[0m (volflag=dbd)")
|
1216
1218
|
ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
|
copyparty/__version__.py
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
|
3
|
-
VERSION = (1,
|
4
|
-
CODENAME = "
|
5
|
-
BUILD_DT = (2024, 4,
|
3
|
+
VERSION = (1, 13, 0)
|
4
|
+
CODENAME = "race the beam"
|
5
|
+
BUILD_DT = (2024, 4, 20)
|
6
6
|
|
7
7
|
S_VERSION = ".".join(map(str, VERSION))
|
8
8
|
S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)
|
copyparty/cfg.py
CHANGED
copyparty/httpcli.py
CHANGED
@@ -36,6 +36,7 @@ from .bos import bos
|
|
36
36
|
from .star import StreamTar
|
37
37
|
from .sutil import StreamArc, gfilter
|
38
38
|
from .szip import StreamZip
|
39
|
+
from .up2k import up2k_chunksize
|
39
40
|
from .util import unquote # type: ignore
|
40
41
|
from .util import (
|
41
42
|
APPLESAN_RE,
|
@@ -123,6 +124,7 @@ class HttpCli(object):
|
|
123
124
|
self.ico = conn.ico # mypy404
|
124
125
|
self.thumbcli = conn.thumbcli # mypy404
|
125
126
|
self.u2fh = conn.u2fh # mypy404
|
127
|
+
self.pipes = conn.pipes # mypy404
|
126
128
|
self.log_func = conn.log_func # mypy404
|
127
129
|
self.log_src = conn.log_src # mypy404
|
128
130
|
self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey
|
@@ -2925,17 +2927,42 @@ class HttpCli(object):
|
|
2925
2927
|
|
2926
2928
|
return txt
|
2927
2929
|
|
2928
|
-
def tx_file(self, req_path ) :
|
2930
|
+
def tx_file(self, req_path , ptop = None) :
|
2929
2931
|
status = 200
|
2930
2932
|
logmsg = "{:4} {} ".format("", self.req)
|
2931
2933
|
logtail = ""
|
2932
2934
|
|
2935
|
+
if ptop is not None:
|
2936
|
+
try:
|
2937
|
+
dp, fn = os.path.split(req_path)
|
2938
|
+
tnam = fn + ".PARTIAL"
|
2939
|
+
if self.args.dotpart:
|
2940
|
+
tnam = "." + tnam
|
2941
|
+
ap_data = os.path.join(dp, tnam)
|
2942
|
+
st_data = bos.stat(ap_data)
|
2943
|
+
if not st_data.st_size:
|
2944
|
+
raise Exception("partial is empty")
|
2945
|
+
x = self.conn.hsrv.broker.ask("up2k.find_job_by_ap", ptop, req_path)
|
2946
|
+
job = json.loads(x.get())
|
2947
|
+
if not job:
|
2948
|
+
raise Exception("not found in registry")
|
2949
|
+
self.pipes.set(req_path, job)
|
2950
|
+
except Exception as ex:
|
2951
|
+
self.log("will not pipe [%s]; %s" % (ap_data, ex), 6)
|
2952
|
+
ptop = None
|
2953
|
+
|
2933
2954
|
#
|
2934
2955
|
# if request is for foo.js, check if we have foo.js.gz
|
2935
2956
|
|
2936
2957
|
file_ts = 0.0
|
2937
2958
|
editions = {}
|
2938
2959
|
for ext in ("", ".gz"):
|
2960
|
+
if ptop is not None:
|
2961
|
+
sz = job["size"]
|
2962
|
+
file_ts = job["lmod"]
|
2963
|
+
editions["plain"] = (ap_data, sz)
|
2964
|
+
break
|
2965
|
+
|
2939
2966
|
try:
|
2940
2967
|
fs_path = req_path + ext
|
2941
2968
|
st = bos.stat(fs_path)
|
@@ -3092,6 +3119,11 @@ class HttpCli(object):
|
|
3092
3119
|
self.send_headers(length=upper - lower, status=status, mime=mime)
|
3093
3120
|
return True
|
3094
3121
|
|
3122
|
+
if ptop is not None:
|
3123
|
+
return self.tx_pipe(
|
3124
|
+
ptop, req_path, ap_data, job, lower, upper, status, mime, logmsg
|
3125
|
+
)
|
3126
|
+
|
3095
3127
|
ret = True
|
3096
3128
|
with open_func(*open_args) as f:
|
3097
3129
|
self.send_headers(length=upper - lower, status=status, mime=mime)
|
@@ -3111,6 +3143,143 @@ class HttpCli(object):
|
|
3111
3143
|
|
3112
3144
|
return ret
|
3113
3145
|
|
3146
|
+
def tx_pipe(
|
3147
|
+
self,
|
3148
|
+
ptop ,
|
3149
|
+
req_path ,
|
3150
|
+
ap_data ,
|
3151
|
+
job ,
|
3152
|
+
lower ,
|
3153
|
+
upper ,
|
3154
|
+
status ,
|
3155
|
+
mime ,
|
3156
|
+
logmsg ,
|
3157
|
+
) :
|
3158
|
+
M = 1048576
|
3159
|
+
self.send_headers(length=upper - lower, status=status, mime=mime)
|
3160
|
+
wr_slp = self.args.s_wr_slp
|
3161
|
+
wr_sz = self.args.s_wr_sz
|
3162
|
+
file_size = job["size"]
|
3163
|
+
chunk_size = up2k_chunksize(file_size)
|
3164
|
+
num_need = -1
|
3165
|
+
data_end = 0
|
3166
|
+
remains = upper - lower
|
3167
|
+
broken = False
|
3168
|
+
spins = 0
|
3169
|
+
tier = 0
|
3170
|
+
tiers = ["uncapped", "reduced speed", "one byte per sec"]
|
3171
|
+
|
3172
|
+
while lower < upper and not broken:
|
3173
|
+
with self.u2mutex:
|
3174
|
+
job = self.pipes.get(req_path)
|
3175
|
+
if not job:
|
3176
|
+
x = self.conn.hsrv.broker.ask("up2k.find_job_by_ap", ptop, req_path)
|
3177
|
+
job = json.loads(x.get())
|
3178
|
+
if job:
|
3179
|
+
self.pipes.set(req_path, job)
|
3180
|
+
|
3181
|
+
if not job:
|
3182
|
+
t = "pipe: OK, upload has finished; yeeting remainder"
|
3183
|
+
self.log(t, 2)
|
3184
|
+
data_end = file_size
|
3185
|
+
break
|
3186
|
+
|
3187
|
+
if num_need != len(job["need"]):
|
3188
|
+
num_need = len(job["need"])
|
3189
|
+
data_end = 0
|
3190
|
+
for cid in job["hash"]:
|
3191
|
+
if cid in job["need"]:
|
3192
|
+
break
|
3193
|
+
data_end += chunk_size
|
3194
|
+
t = "pipe: can stream %.2f MiB; requested range is %.2f to %.2f"
|
3195
|
+
self.log(t % (data_end / M, lower / M, upper / M), 6)
|
3196
|
+
with self.u2mutex:
|
3197
|
+
if data_end > self.u2fh.aps.get(ap_data, data_end):
|
3198
|
+
try:
|
3199
|
+
fhs = self.u2fh.cache[ap_data].all_fhs
|
3200
|
+
for fh in fhs:
|
3201
|
+
fh.flush()
|
3202
|
+
self.u2fh.aps[ap_data] = data_end
|
3203
|
+
self.log("pipe: flushed %d up2k-FDs" % (len(fhs),))
|
3204
|
+
except Exception as ex:
|
3205
|
+
self.log("pipe: u2fh flush failed: %r" % (ex,))
|
3206
|
+
|
3207
|
+
if lower >= data_end:
|
3208
|
+
if data_end:
|
3209
|
+
t = "pipe: uploader is too slow; aborting download at %.2f MiB"
|
3210
|
+
self.log(t % (data_end / M))
|
3211
|
+
raise Pebkac(416, "uploader is too slow")
|
3212
|
+
|
3213
|
+
raise Pebkac(416, "no data available yet; please retry in a bit")
|
3214
|
+
|
3215
|
+
slack = data_end - lower
|
3216
|
+
if slack >= 8 * M:
|
3217
|
+
ntier = 0
|
3218
|
+
winsz = M
|
3219
|
+
bufsz = wr_sz
|
3220
|
+
slp = wr_slp
|
3221
|
+
else:
|
3222
|
+
winsz = max(40, int(M * (slack / (12 * M))))
|
3223
|
+
base_rate = M if not wr_slp else wr_sz / wr_slp
|
3224
|
+
if winsz > base_rate:
|
3225
|
+
ntier = 0
|
3226
|
+
bufsz = wr_sz
|
3227
|
+
slp = wr_slp
|
3228
|
+
elif winsz > 300:
|
3229
|
+
ntier = 1
|
3230
|
+
bufsz = winsz // 5
|
3231
|
+
slp = 0.2
|
3232
|
+
else:
|
3233
|
+
ntier = 2
|
3234
|
+
bufsz = winsz = slp = 1
|
3235
|
+
|
3236
|
+
if tier != ntier:
|
3237
|
+
tier = ntier
|
3238
|
+
self.log("moved to tier %d (%s)" % (tier, tiers[tier]))
|
3239
|
+
|
3240
|
+
try:
|
3241
|
+
with open(ap_data, "rb", self.args.iobuf) as f:
|
3242
|
+
f.seek(lower)
|
3243
|
+
page = f.read(min(winsz, data_end - lower, upper - lower))
|
3244
|
+
if not page:
|
3245
|
+
raise Exception("got 0 bytes (EOF?)")
|
3246
|
+
except Exception as ex:
|
3247
|
+
self.log("pipe: read failed at %.2f MiB: %s" % (lower / M, ex), 3)
|
3248
|
+
with self.u2mutex:
|
3249
|
+
self.pipes.c.pop(req_path, None)
|
3250
|
+
spins += 1
|
3251
|
+
if spins > 3:
|
3252
|
+
raise Pebkac(500, "file became unreadable")
|
3253
|
+
time.sleep(2)
|
3254
|
+
continue
|
3255
|
+
|
3256
|
+
spins = 0
|
3257
|
+
pofs = 0
|
3258
|
+
while pofs < len(page):
|
3259
|
+
if slp:
|
3260
|
+
time.sleep(slp)
|
3261
|
+
|
3262
|
+
try:
|
3263
|
+
buf = page[pofs : pofs + bufsz]
|
3264
|
+
self.s.sendall(buf)
|
3265
|
+
zi = len(buf)
|
3266
|
+
remains -= zi
|
3267
|
+
lower += zi
|
3268
|
+
pofs += zi
|
3269
|
+
except:
|
3270
|
+
broken = True
|
3271
|
+
break
|
3272
|
+
|
3273
|
+
if lower < upper and not broken:
|
3274
|
+
with open(req_path, "rb") as f:
|
3275
|
+
remains = sendfile_py(self.log, lower, upper, f, self.s, wr_sz, wr_slp)
|
3276
|
+
|
3277
|
+
spd = self._spd((upper - lower) - remains)
|
3278
|
+
if self.do_log:
|
3279
|
+
self.log("{}, {}".format(logmsg, spd))
|
3280
|
+
|
3281
|
+
return not broken
|
3282
|
+
|
3114
3283
|
def tx_zip(
|
3115
3284
|
self,
|
3116
3285
|
fmt ,
|
@@ -3748,7 +3917,7 @@ class HttpCli(object):
|
|
3748
3917
|
if not allvols:
|
3749
3918
|
ret = [{"kinshi": 1}]
|
3750
3919
|
|
3751
|
-
jtxt = '{"u":%s,"c":%s}' % (uret, json.dumps(ret,
|
3920
|
+
jtxt = '{"u":%s,"c":%s}' % (uret, json.dumps(ret, separators=(",\n", ": ")))
|
3752
3921
|
zi = len(uret.split('\n"pd":')) - 1
|
3753
3922
|
self.log("%s #%d+%d %.2fsec" % (lm, zi, len(ret), time.time() - t0))
|
3754
3923
|
self.reply(jtxt.encode("utf-8", "replace"), mime="application/json")
|
@@ -4027,7 +4196,9 @@ class HttpCli(object):
|
|
4027
4196
|
):
|
4028
4197
|
return self.tx_md(vn, abspath)
|
4029
4198
|
|
4030
|
-
return self.tx_file(
|
4199
|
+
return self.tx_file(
|
4200
|
+
abspath, None if st.st_size or "nopipe" in vn.flags else vn.realpath
|
4201
|
+
)
|
4031
4202
|
|
4032
4203
|
elif is_dir and not self.can_read:
|
4033
4204
|
if self._use_dirkey(abspath):
|
copyparty/httpconn.py
CHANGED
copyparty/httpsrv.py
CHANGED
@@ -61,6 +61,7 @@ from .u2idx import U2idx
|
|
61
61
|
from .util import (
|
62
62
|
E_SCK,
|
63
63
|
FHC,
|
64
|
+
CachedDict,
|
64
65
|
Daemon,
|
65
66
|
Garda,
|
66
67
|
Magician,
|
@@ -126,6 +127,7 @@ class HttpSrv(object):
|
|
126
127
|
self.t_periodic = None
|
127
128
|
|
128
129
|
self.u2fh = FHC()
|
130
|
+
self.pipes = CachedDict(0.2)
|
129
131
|
self.metrics = Metrics(self)
|
130
132
|
self.nreq = 0
|
131
133
|
self.nsus = 0
|
copyparty/up2k.py
CHANGED
@@ -136,6 +136,7 @@ class Up2k(object):
|
|
136
136
|
self.need_rescan = set()
|
137
137
|
self.db_act = 0.0
|
138
138
|
|
139
|
+
self.reg_mutex = threading.Lock()
|
139
140
|
self.registry = {}
|
140
141
|
self.flags = {}
|
141
142
|
self.droppable = {}
|
@@ -143,7 +144,7 @@ class Up2k(object):
|
|
143
144
|
self.volsize = {}
|
144
145
|
self.volstate = {}
|
145
146
|
self.vol_act = {}
|
146
|
-
self.busy_aps
|
147
|
+
self.busy_aps = {}
|
147
148
|
self.dupesched = {}
|
148
149
|
self.snap_prev = {}
|
149
150
|
|
@@ -200,11 +201,15 @@ class Up2k(object):
|
|
200
201
|
Daemon(self.deferred_init, "up2k-deferred-init")
|
201
202
|
|
202
203
|
def reload(self, rescan_all_vols ) :
|
203
|
-
"""mutex me"""
|
204
|
+
"""mutex(main) me"""
|
204
205
|
self.log("reload #{} scheduled".format(self.gid + 1))
|
205
206
|
all_vols = self.asrv.vfs.all_vols
|
206
207
|
|
207
|
-
|
208
|
+
with self.reg_mutex:
|
209
|
+
scan_vols = [
|
210
|
+
k for k, v in all_vols.items() if v.realpath not in self.registry
|
211
|
+
]
|
212
|
+
|
208
213
|
if rescan_all_vols:
|
209
214
|
scan_vols = list(all_vols.keys())
|
210
215
|
|
@@ -217,7 +222,7 @@ class Up2k(object):
|
|
217
222
|
if self.stop:
|
218
223
|
# up-mt consistency not guaranteed if init is interrupted;
|
219
224
|
# drop caches for a full scan on next boot
|
220
|
-
with self.mutex:
|
225
|
+
with self.mutex, self.reg_mutex:
|
221
226
|
self._drop_caches()
|
222
227
|
|
223
228
|
if self.pp:
|
@@ -283,10 +288,27 @@ class Up2k(object):
|
|
283
288
|
min(1000 * 24 * 60 * 60 - 1, time.time() - self.db_act)
|
284
289
|
),
|
285
290
|
}
|
286
|
-
return json.dumps(ret,
|
291
|
+
return json.dumps(ret, separators=(",\n", ": "))
|
292
|
+
|
293
|
+
def find_job_by_ap(self, ptop , ap ) :
|
294
|
+
try:
|
295
|
+
if ANYWIN:
|
296
|
+
ap = ap.replace("\\", "/")
|
297
|
+
|
298
|
+
vp = ap[len(ptop) :].strip("/")
|
299
|
+
dn, fn = vsplit(vp)
|
300
|
+
with self.reg_mutex:
|
301
|
+
tab2 = self.registry[ptop]
|
302
|
+
for job in tab2.values():
|
303
|
+
if job["prel"] == dn and job["name"] == fn:
|
304
|
+
return json.dumps(job, separators=(",\n", ": "))
|
305
|
+
except:
|
306
|
+
pass
|
307
|
+
|
308
|
+
return "{}"
|
287
309
|
|
288
310
|
def get_unfinished_by_user(self, uname, ip) :
|
289
|
-
if PY2 or not self.
|
311
|
+
if PY2 or not self.reg_mutex.acquire(timeout=2):
|
290
312
|
return '[{"timeout":1}]'
|
291
313
|
|
292
314
|
ret = []
|
@@ -315,17 +337,25 @@ class Up2k(object):
|
|
315
337
|
)
|
316
338
|
ret.append(zt5)
|
317
339
|
finally:
|
318
|
-
self.
|
340
|
+
self.reg_mutex.release()
|
341
|
+
|
342
|
+
if ANYWIN:
|
343
|
+
ret = [(x[0], x[1].replace("\\", "/"), x[2], x[3], x[4]) for x in ret]
|
319
344
|
|
320
345
|
ret.sort(reverse=True)
|
321
346
|
ret2 = [
|
322
|
-
{
|
347
|
+
{
|
348
|
+
"at": at,
|
349
|
+
"vp": "/" + quotep(vp),
|
350
|
+
"pd": 100 - ((nn * 100) // (nh or 1)),
|
351
|
+
"sz": sz,
|
352
|
+
}
|
323
353
|
for (at, vp, sz, nn, nh) in ret
|
324
354
|
]
|
325
|
-
return json.dumps(ret2,
|
355
|
+
return json.dumps(ret2, separators=(",\n", ": "))
|
326
356
|
|
327
357
|
def get_unfinished(self) :
|
328
|
-
if PY2 or not self.
|
358
|
+
if PY2 or not self.reg_mutex.acquire(timeout=0.5):
|
329
359
|
return ""
|
330
360
|
|
331
361
|
ret = {}
|
@@ -347,17 +377,17 @@ class Up2k(object):
|
|
347
377
|
|
348
378
|
ret[ptop] = (nbytes, nfiles)
|
349
379
|
finally:
|
350
|
-
self.
|
380
|
+
self.reg_mutex.release()
|
351
381
|
|
352
|
-
return json.dumps(ret,
|
382
|
+
return json.dumps(ret, separators=(",\n", ": "))
|
353
383
|
|
354
384
|
def get_volsize(self, ptop ) :
|
355
|
-
with self.
|
385
|
+
with self.reg_mutex:
|
356
386
|
return self._get_volsize(ptop)
|
357
387
|
|
358
388
|
def get_volsizes(self, ptops ) :
|
359
389
|
ret = []
|
360
|
-
with self.
|
390
|
+
with self.reg_mutex:
|
361
391
|
for ptop in ptops:
|
362
392
|
ret.append(self._get_volsize(ptop))
|
363
393
|
|
@@ -385,7 +415,7 @@ class Up2k(object):
|
|
385
415
|
def _rescan(
|
386
416
|
self, all_vols , scan_vols , wait , fscan
|
387
417
|
) :
|
388
|
-
"""mutex me"""
|
418
|
+
"""mutex(main) me"""
|
389
419
|
if not wait and self.pp:
|
390
420
|
return "cannot initiate; scan is already in progress"
|
391
421
|
|
@@ -667,7 +697,7 @@ class Up2k(object):
|
|
667
697
|
self.log(msg, c=3)
|
668
698
|
|
669
699
|
live_vols = []
|
670
|
-
with self.mutex:
|
700
|
+
with self.mutex, self.reg_mutex:
|
671
701
|
# only need to protect register_vpath but all in one go feels right
|
672
702
|
for vol in vols:
|
673
703
|
try:
|
@@ -709,7 +739,7 @@ class Up2k(object):
|
|
709
739
|
|
710
740
|
if self.args.re_dhash or [zv for zv in vols if "e2tsr" in zv.flags]:
|
711
741
|
self.args.re_dhash = False
|
712
|
-
with self.mutex:
|
742
|
+
with self.mutex, self.reg_mutex:
|
713
743
|
self._drop_caches()
|
714
744
|
|
715
745
|
for vol in vols:
|
@@ -786,7 +816,9 @@ class Up2k(object):
|
|
786
816
|
self.volstate[vol.vpath] = "online (mtp soon)"
|
787
817
|
|
788
818
|
for vol in need_vac:
|
789
|
-
|
819
|
+
with self.mutex, self.reg_mutex:
|
820
|
+
reg = self.register_vpath(vol.realpath, vol.flags)
|
821
|
+
|
790
822
|
assert reg
|
791
823
|
cur, _ = reg
|
792
824
|
with self.mutex:
|
@@ -800,7 +832,9 @@ class Up2k(object):
|
|
800
832
|
if vol.flags["dbd"] == "acid":
|
801
833
|
continue
|
802
834
|
|
803
|
-
|
835
|
+
with self.mutex, self.reg_mutex:
|
836
|
+
reg = self.register_vpath(vol.realpath, vol.flags)
|
837
|
+
|
804
838
|
try:
|
805
839
|
assert reg
|
806
840
|
cur, db_path = reg
|
@@ -847,6 +881,7 @@ class Up2k(object):
|
|
847
881
|
def register_vpath(
|
848
882
|
self, ptop , flags
|
849
883
|
) :
|
884
|
+
"""mutex(main,reg) me"""
|
850
885
|
histpath = self.asrv.vfs.histtab.get(ptop)
|
851
886
|
if not histpath:
|
852
887
|
self.log("no histpath for [{}]".format(ptop))
|
@@ -1030,7 +1065,9 @@ class Up2k(object):
|
|
1030
1065
|
dev = cst.st_dev if vol.flags.get("xdev") else 0
|
1031
1066
|
|
1032
1067
|
with self.mutex:
|
1033
|
-
|
1068
|
+
with self.reg_mutex:
|
1069
|
+
reg = self.register_vpath(top, vol.flags)
|
1070
|
+
|
1034
1071
|
assert reg and self.pp
|
1035
1072
|
cur, db_path = reg
|
1036
1073
|
|
@@ -1627,7 +1664,7 @@ class Up2k(object):
|
|
1627
1664
|
|
1628
1665
|
def _build_tags_index(self, vol ) :
|
1629
1666
|
ptop = vol.realpath
|
1630
|
-
with self.mutex:
|
1667
|
+
with self.mutex, self.reg_mutex:
|
1631
1668
|
reg = self.register_vpath(ptop, vol.flags)
|
1632
1669
|
|
1633
1670
|
assert reg and self.pp
|
@@ -1648,6 +1685,7 @@ class Up2k(object):
|
|
1648
1685
|
return ret
|
1649
1686
|
|
1650
1687
|
def _drop_caches(self) :
|
1688
|
+
"""mutex(main,reg) me"""
|
1651
1689
|
self.log("dropping caches for a full filesystem scan")
|
1652
1690
|
for vol in self.asrv.vfs.all_vols.values():
|
1653
1691
|
reg = self.register_vpath(vol.realpath, vol.flags)
|
@@ -1823,7 +1861,7 @@ class Up2k(object):
|
|
1823
1861
|
params ,
|
1824
1862
|
flt ,
|
1825
1863
|
) :
|
1826
|
-
"""mutex me"""
|
1864
|
+
"""mutex(main) me"""
|
1827
1865
|
n = 0
|
1828
1866
|
c2 = cur.connection.cursor()
|
1829
1867
|
tf = tempfile.SpooledTemporaryFile(1024 * 1024 * 8, "w+b", prefix="cpp-tq-")
|
@@ -2157,7 +2195,7 @@ class Up2k(object):
|
|
2157
2195
|
ip ,
|
2158
2196
|
at ,
|
2159
2197
|
) :
|
2160
|
-
"""will mutex"""
|
2198
|
+
"""will mutex(main)"""
|
2161
2199
|
assert self.mtag
|
2162
2200
|
|
2163
2201
|
try:
|
@@ -2189,7 +2227,7 @@ class Up2k(object):
|
|
2189
2227
|
abspath ,
|
2190
2228
|
tags ,
|
2191
2229
|
) :
|
2192
|
-
"""mutex me"""
|
2230
|
+
"""mutex(main) me"""
|
2193
2231
|
assert self.mtag
|
2194
2232
|
|
2195
2233
|
if not bos.path.isfile(abspath):
|
@@ -2474,28 +2512,36 @@ class Up2k(object):
|
|
2474
2512
|
|
2475
2513
|
cur.connection.commit()
|
2476
2514
|
|
2477
|
-
def
|
2478
|
-
|
2479
|
-
|
2480
|
-
|
2481
|
-
|
2482
|
-
def handle_json(self, cj , busy_aps ) :
|
2515
|
+
def handle_json(
|
2516
|
+
self, cj , busy_aps
|
2517
|
+
) :
|
2518
|
+
# busy_aps is u2fh (always undefined if -j0) so this is safe
|
2483
2519
|
self.busy_aps = busy_aps
|
2520
|
+
got_lock = False
|
2484
2521
|
try:
|
2485
2522
|
# bit expensive; 3.9=10x 3.11=2x
|
2486
2523
|
if self.mutex.acquire(timeout=10):
|
2487
|
-
|
2488
|
-
self.
|
2524
|
+
got_lock = True
|
2525
|
+
with self.reg_mutex:
|
2526
|
+
return self._handle_json(cj)
|
2489
2527
|
else:
|
2490
2528
|
t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
|
2491
2529
|
raise Pebkac(503, t.format(self.blocked or "[unknown]"))
|
2492
2530
|
except TypeError:
|
2493
2531
|
if not PY2:
|
2494
2532
|
raise
|
2495
|
-
with self.mutex:
|
2496
|
-
self.
|
2533
|
+
with self.mutex, self.reg_mutex:
|
2534
|
+
return self._handle_json(cj)
|
2535
|
+
finally:
|
2536
|
+
if got_lock:
|
2537
|
+
self.mutex.release()
|
2497
2538
|
|
2539
|
+
def _handle_json(self, cj ) :
|
2498
2540
|
ptop = cj["ptop"]
|
2541
|
+
if not self.register_vpath(ptop, cj["vcfg"]):
|
2542
|
+
if ptop not in self.registry:
|
2543
|
+
raise Pebkac(410, "location unavailable")
|
2544
|
+
|
2499
2545
|
cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
|
2500
2546
|
cj["poke"] = now = self.db_act = self.vol_act[ptop] = time.time()
|
2501
2547
|
wark = self._get_wark(cj)
|
@@ -2510,7 +2556,7 @@ class Up2k(object):
|
|
2510
2556
|
# refuse out-of-order / multithreaded uploading if sprs False
|
2511
2557
|
sprs = self.fstab.get(pdir) != "ng"
|
2512
2558
|
|
2513
|
-
|
2559
|
+
if True:
|
2514
2560
|
jcur = self.cur.get(ptop)
|
2515
2561
|
reg = self.registry[ptop]
|
2516
2562
|
vfs = self.asrv.vfs.all_vols[cj["vtop"]]
|
@@ -2948,7 +2994,7 @@ class Up2k(object):
|
|
2948
2994
|
def handle_chunk(
|
2949
2995
|
self, ptop , wark , chash
|
2950
2996
|
) :
|
2951
|
-
with self.mutex:
|
2997
|
+
with self.mutex, self.reg_mutex:
|
2952
2998
|
self.db_act = self.vol_act[ptop] = time.time()
|
2953
2999
|
job = self.registry[ptop].get(wark)
|
2954
3000
|
if not job:
|
@@ -2991,7 +3037,7 @@ class Up2k(object):
|
|
2991
3037
|
return chunksize, ofs, path, job["lmod"], job["sprs"]
|
2992
3038
|
|
2993
3039
|
def release_chunk(self, ptop , wark , chash ) :
|
2994
|
-
with self.
|
3040
|
+
with self.reg_mutex:
|
2995
3041
|
job = self.registry[ptop].get(wark)
|
2996
3042
|
if job:
|
2997
3043
|
job["busy"].pop(chash, None)
|
@@ -2999,7 +3045,7 @@ class Up2k(object):
|
|
2999
3045
|
return True
|
3000
3046
|
|
3001
3047
|
def confirm_chunk(self, ptop , wark , chash ) :
|
3002
|
-
with self.mutex:
|
3048
|
+
with self.mutex, self.reg_mutex:
|
3003
3049
|
self.db_act = self.vol_act[ptop] = time.time()
|
3004
3050
|
try:
|
3005
3051
|
job = self.registry[ptop][wark]
|
@@ -3022,16 +3068,16 @@ class Up2k(object):
|
|
3022
3068
|
|
3023
3069
|
if self.args.nw:
|
3024
3070
|
self.regdrop(ptop, wark)
|
3025
|
-
return ret, dst
|
3026
3071
|
|
3027
3072
|
return ret, dst
|
3028
3073
|
|
3029
3074
|
def finish_upload(self, ptop , wark , busy_aps ) :
|
3030
3075
|
self.busy_aps = busy_aps
|
3031
|
-
with self.mutex:
|
3076
|
+
with self.mutex, self.reg_mutex:
|
3032
3077
|
self._finish_upload(ptop, wark)
|
3033
3078
|
|
3034
3079
|
def _finish_upload(self, ptop , wark ) :
|
3080
|
+
"""mutex(main,reg) me"""
|
3035
3081
|
try:
|
3036
3082
|
job = self.registry[ptop][wark]
|
3037
3083
|
pdir = djoin(job["ptop"], job["prel"])
|
@@ -3104,6 +3150,7 @@ class Up2k(object):
|
|
3104
3150
|
cur.connection.commit()
|
3105
3151
|
|
3106
3152
|
def regdrop(self, ptop , wark ) :
|
3153
|
+
"""mutex(main,reg) me"""
|
3107
3154
|
olds = self.droppable[ptop]
|
3108
3155
|
if wark:
|
3109
3156
|
olds.append(wark)
|
@@ -3198,16 +3245,23 @@ class Up2k(object):
|
|
3198
3245
|
at ,
|
3199
3246
|
skip_xau = False,
|
3200
3247
|
) :
|
3248
|
+
"""mutex(main) me"""
|
3201
3249
|
self.db_rm(db, rd, fn, sz)
|
3202
3250
|
|
3251
|
+
if not ip:
|
3252
|
+
db_ip = ""
|
3253
|
+
else:
|
3254
|
+
# plugins may expect this to look like an actual IP
|
3255
|
+
db_ip = "1.1.1.1" if self.args.no_db_ip else ip
|
3256
|
+
|
3203
3257
|
sql = "insert into up values (?,?,?,?,?,?,?)"
|
3204
|
-
v = (wark, int(ts), sz, rd, fn,
|
3258
|
+
v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
|
3205
3259
|
try:
|
3206
3260
|
db.execute(sql, v)
|
3207
3261
|
except:
|
3208
3262
|
assert self.mem_cur
|
3209
3263
|
rd, fn = s3enc(self.mem_cur, rd, fn)
|
3210
|
-
v = (wark, int(ts), sz, rd, fn,
|
3264
|
+
v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
|
3211
3265
|
db.execute(sql, v)
|
3212
3266
|
|
3213
3267
|
self.volsize[db] += sz
|
@@ -3311,7 +3365,7 @@ class Up2k(object):
|
|
3311
3365
|
vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
|
3312
3366
|
vn, rem = vn.get_dbv(rem)
|
3313
3367
|
ptop = vn.realpath
|
3314
|
-
with self.mutex:
|
3368
|
+
with self.mutex, self.reg_mutex:
|
3315
3369
|
abrt_cfg = self.flags.get(ptop, {}).get("u2abort", 1)
|
3316
3370
|
addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
|
3317
3371
|
user = (uname or "\n") if abrt_cfg in (1, 3) else ""
|
@@ -3319,7 +3373,10 @@ class Up2k(object):
|
|
3319
3373
|
for wark, job in reg.items():
|
3320
3374
|
if (user and user != job["user"]) or (addr and addr != job["addr"]):
|
3321
3375
|
continue
|
3322
|
-
|
3376
|
+
jrem = djoin(job["prel"], job["name"])
|
3377
|
+
if ANYWIN:
|
3378
|
+
jrem = jrem.replace("\\", "/")
|
3379
|
+
if jrem == rem:
|
3323
3380
|
if job["ptop"] != ptop:
|
3324
3381
|
t = "job.ptop [%s] != vol.ptop [%s] ??"
|
3325
3382
|
raise Exception(t % (job["ptop"] != ptop))
|
@@ -3415,7 +3472,7 @@ class Up2k(object):
|
|
3415
3472
|
continue
|
3416
3473
|
|
3417
3474
|
n_files += 1
|
3418
|
-
with self.mutex:
|
3475
|
+
with self.mutex, self.reg_mutex:
|
3419
3476
|
cur = None
|
3420
3477
|
try:
|
3421
3478
|
ptop = dbv.realpath
|
@@ -3533,6 +3590,7 @@ class Up2k(object):
|
|
3533
3590
|
def _mv_file(
|
3534
3591
|
self, uname , svp , dvp , curs
|
3535
3592
|
) :
|
3593
|
+
"""mutex(main) me; will mutex(reg)"""
|
3536
3594
|
svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
|
3537
3595
|
svn, srem = svn.get_dbv(srem)
|
3538
3596
|
|
@@ -3613,7 +3671,9 @@ class Up2k(object):
|
|
3613
3671
|
if c2 and c2 != c1:
|
3614
3672
|
self._copy_tags(c1, c2, w)
|
3615
3673
|
|
3616
|
-
|
3674
|
+
with self.reg_mutex:
|
3675
|
+
has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
|
3676
|
+
|
3617
3677
|
if not is_xvol:
|
3618
3678
|
has_dupes = self._relink(w, svn.realpath, srem, dabs)
|
3619
3679
|
|
@@ -3743,7 +3803,10 @@ class Up2k(object):
|
|
3743
3803
|
drop_tags ,
|
3744
3804
|
sz ,
|
3745
3805
|
) :
|
3746
|
-
"""
|
3806
|
+
"""
|
3807
|
+
mutex(main,reg) me
|
3808
|
+
forgets file in db, fixes symlinks, does not delete
|
3809
|
+
"""
|
3747
3810
|
srd, sfn = vsplit(vrem)
|
3748
3811
|
has_dupes = False
|
3749
3812
|
self.log("forgetting {}".format(vrem))
|
@@ -4068,7 +4131,7 @@ class Up2k(object):
|
|
4068
4131
|
self.do_snapshot()
|
4069
4132
|
|
4070
4133
|
def do_snapshot(self) :
|
4071
|
-
with self.mutex:
|
4134
|
+
with self.mutex, self.reg_mutex:
|
4072
4135
|
for k, reg in self.registry.items():
|
4073
4136
|
self._snap_reg(k, reg)
|
4074
4137
|
|
@@ -4136,7 +4199,7 @@ class Up2k(object):
|
|
4136
4199
|
|
4137
4200
|
path2 = "{}.{}".format(path, os.getpid())
|
4138
4201
|
body = {"droppable": self.droppable[ptop], "registry": reg}
|
4139
|
-
j = json.dumps(body,
|
4202
|
+
j = json.dumps(body, sort_keys=True, separators=(",\n", ": ")).encode("utf-8")
|
4140
4203
|
with gzip.GzipFile(path2, "wb") as f:
|
4141
4204
|
f.write(j)
|
4142
4205
|
|
@@ -4209,7 +4272,7 @@ class Up2k(object):
|
|
4209
4272
|
raise Exception("invalid hash task")
|
4210
4273
|
|
4211
4274
|
try:
|
4212
|
-
if not self._hash_t(task):
|
4275
|
+
if not self._hash_t(task) and self.stop:
|
4213
4276
|
return
|
4214
4277
|
except Exception as ex:
|
4215
4278
|
self.log("failed to hash %s: %s" % (task, ex), 1)
|
@@ -4219,7 +4282,7 @@ class Up2k(object):
|
|
4219
4282
|
) :
|
4220
4283
|
ptop, vtop, flags, rd, fn, ip, at, usr, skip_xau = task
|
4221
4284
|
# self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
|
4222
|
-
with self.mutex:
|
4285
|
+
with self.mutex, self.reg_mutex:
|
4223
4286
|
if not self.register_vpath(ptop, flags):
|
4224
4287
|
return True
|
4225
4288
|
|
@@ -4237,7 +4300,7 @@ class Up2k(object):
|
|
4237
4300
|
|
4238
4301
|
wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
|
4239
4302
|
|
4240
|
-
with self.mutex:
|
4303
|
+
with self.mutex, self.reg_mutex:
|
4241
4304
|
self.idx_wark(
|
4242
4305
|
self.flags[ptop],
|
4243
4306
|
rd,
|
copyparty/util.py
CHANGED
@@ -738,15 +738,46 @@ class CachedSet(object):
|
|
738
738
|
self.oldest = now
|
739
739
|
|
740
740
|
|
741
|
+
class CachedDict(object):
|
742
|
+
def __init__(self, maxage ) :
|
743
|
+
self.c = {}
|
744
|
+
self.maxage = maxage
|
745
|
+
self.oldest = 0.0
|
746
|
+
|
747
|
+
def set(self, k , v ) :
|
748
|
+
now = time.time()
|
749
|
+
self.c[k] = (now, v)
|
750
|
+
if now - self.oldest < self.maxage:
|
751
|
+
return
|
752
|
+
|
753
|
+
c = self.c = {k: v for k, v in self.c.items() if now - v[0] < self.maxage}
|
754
|
+
try:
|
755
|
+
self.oldest = min([x[0] for x in c.values()])
|
756
|
+
except:
|
757
|
+
self.oldest = now
|
758
|
+
|
759
|
+
def get(self, k ) :
|
760
|
+
try:
|
761
|
+
ts, ret = self.c[k]
|
762
|
+
now = time.time()
|
763
|
+
if now - ts > self.maxage:
|
764
|
+
del self.c[k]
|
765
|
+
return None
|
766
|
+
return ret
|
767
|
+
except:
|
768
|
+
return None
|
769
|
+
|
770
|
+
|
741
771
|
class FHC(object):
|
742
772
|
class CE(object):
|
743
773
|
def __init__(self, fh ) :
|
744
774
|
self.ts = 0
|
745
775
|
self.fhs = [fh]
|
776
|
+
self.all_fhs = set([fh])
|
746
777
|
|
747
778
|
def __init__(self) :
|
748
779
|
self.cache = {}
|
749
|
-
self.aps
|
780
|
+
self.aps = {}
|
750
781
|
|
751
782
|
def close(self, path ) :
|
752
783
|
try:
|
@@ -758,7 +789,7 @@ class FHC(object):
|
|
758
789
|
fh.close()
|
759
790
|
|
760
791
|
del self.cache[path]
|
761
|
-
self.aps
|
792
|
+
del self.aps[path]
|
762
793
|
|
763
794
|
def clean(self) :
|
764
795
|
if not self.cache:
|
@@ -779,9 +810,12 @@ class FHC(object):
|
|
779
810
|
return self.cache[path].fhs.pop()
|
780
811
|
|
781
812
|
def put(self, path , fh ) :
|
782
|
-
self.aps
|
813
|
+
if path not in self.aps:
|
814
|
+
self.aps[path] = 0
|
815
|
+
|
783
816
|
try:
|
784
817
|
ce = self.cache[path]
|
818
|
+
ce.all_fhs.add(fh)
|
785
819
|
ce.fhs.append(fh)
|
786
820
|
except:
|
787
821
|
ce = self.CE(fh)
|
copyparty/web/a/u2c.py
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
2
|
from __future__ import print_function, unicode_literals
|
3
3
|
|
4
|
-
S_VERSION = "1.
|
5
|
-
S_BUILD_DT = "2024-
|
4
|
+
S_VERSION = "1.16"
|
5
|
+
S_BUILD_DT = "2024-04-20"
|
6
6
|
|
7
7
|
"""
|
8
8
|
u2c.py: upload to copyparty
|
@@ -563,7 +563,7 @@ def handshake(ar, file, search):
|
|
563
563
|
else:
|
564
564
|
if ar.touch:
|
565
565
|
req["umod"] = True
|
566
|
-
if ar.
|
566
|
+
if ar.ow:
|
567
567
|
req["replace"] = True
|
568
568
|
|
569
569
|
headers = {"Content-Type": "text/plain"} # <=1.5.1 compat
|
@@ -1140,6 +1140,7 @@ source file/folder selection uses rsync syntax, meaning that:
|
|
1140
1140
|
ap.add_argument("-x", type=unicode, metavar="REGEX", default="", help="skip file if filesystem-abspath matches REGEX, example: '.*/\\.hist/.*'")
|
1141
1141
|
ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
|
1142
1142
|
ap.add_argument("--touch", action="store_true", help="if last-modified timestamps differ, push local to server (need write+delete perms)")
|
1143
|
+
ap.add_argument("--ow", action="store_true", help="overwrite existing files instead of autorenaming")
|
1143
1144
|
ap.add_argument("--version", action="store_true", help="show version and exit")
|
1144
1145
|
|
1145
1146
|
ap = app.add_argument_group("compatibility")
|
@@ -1148,7 +1149,7 @@ source file/folder selection uses rsync syntax, meaning that:
|
|
1148
1149
|
|
1149
1150
|
ap = app.add_argument_group("folder sync")
|
1150
1151
|
ap.add_argument("--dl", action="store_true", help="delete local files after uploading")
|
1151
|
-
ap.add_argument("--dr", action="store_true", help="delete remote files which don't exist locally")
|
1152
|
+
ap.add_argument("--dr", action="store_true", help="delete remote files which don't exist locally (implies --ow)")
|
1152
1153
|
ap.add_argument("--drd", action="store_true", help="delete remote files during upload instead of afterwards; reduces peak disk space usage, but will reupload instead of detecting renames")
|
1153
1154
|
|
1154
1155
|
ap = app.add_argument_group("performance tweaks")
|
@@ -1178,6 +1179,9 @@ source file/folder selection uses rsync syntax, meaning that:
|
|
1178
1179
|
if ar.drd:
|
1179
1180
|
ar.dr = True
|
1180
1181
|
|
1182
|
+
if ar.dr:
|
1183
|
+
ar.ow = True
|
1184
|
+
|
1181
1185
|
for k in "dl dr drd".split():
|
1182
1186
|
errs = []
|
1183
1187
|
if ar.safe and getattr(ar, k):
|
copyparty/web/browser.css.gz
CHANGED
Binary file
|
copyparty/web/browser.js.gz
CHANGED
Binary file
|
copyparty/web/up2k.js.gz
CHANGED
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: copyparty
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.13.0
|
4
4
|
Summary: Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
|
5
5
|
Author-email: ed <copyparty@ocv.me>
|
6
6
|
License: MIT
|
@@ -64,6 +64,8 @@ turn almost any device into a file server with resumable uploads/downloads using
|
|
64
64
|
|
65
65
|
📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer)
|
66
66
|
|
67
|
+
🎬 **videos:** [upload](https://a.ocv.me/pub/demo/pics-vids/up2k.webm) // [cli-upload](https://a.ocv.me/pub/demo/pics-vids/u2cli.webm) // [race-the-beam](https://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm)
|
68
|
+
|
67
69
|
|
68
70
|
## readme toc
|
69
71
|
|
@@ -71,7 +73,7 @@ turn almost any device into a file server with resumable uploads/downloads using
|
|
71
73
|
* [quickstart](#quickstart) - just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
|
72
74
|
* [at home](#at-home) - make it accessible over the internet
|
73
75
|
* [on servers](#on-servers) - you may also want these, especially on servers
|
74
|
-
* [features](#features)
|
76
|
+
* [features](#features) - also see [comparison to similar software](./docs/versus.md)
|
75
77
|
* [testimonials](#testimonials) - small collection of user feedback
|
76
78
|
* [motivations](#motivations) - project goals / philosophy
|
77
79
|
* [notes](#notes) - general notes
|
@@ -92,6 +94,7 @@ turn almost any device into a file server with resumable uploads/downloads using
|
|
92
94
|
* [file-search](#file-search) - dropping files into the browser also lets you see if they exist on the server
|
93
95
|
* [unpost](#unpost) - undo/delete accidental uploads
|
94
96
|
* [self-destruct](#self-destruct) - uploads can be given a lifetime
|
97
|
+
* [race the beam](#race-the-beam) - download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm))
|
95
98
|
* [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
|
96
99
|
* [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
|
97
100
|
* [media player](#media-player) - plays almost every audio format there is
|
@@ -181,7 +184,7 @@ enable thumbnails (images/audio/video), media indexing, and audio transcoding by
|
|
181
184
|
|
182
185
|
* **Alpine:** `apk add py3-pillow ffmpeg`
|
183
186
|
* **Debian:** `apt install --no-install-recommends python3-pil ffmpeg`
|
184
|
-
* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg`
|
187
|
+
* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg --allowerasing`
|
185
188
|
* **FreeBSD:** `pkg install py39-sqlite3 py39-pillow ffmpeg`
|
186
189
|
* **MacOS:** `port install py-Pillow ffmpeg`
|
187
190
|
* **MacOS** (alternative): `brew install pillow ffmpeg`
|
@@ -236,6 +239,8 @@ firewall-cmd --reload
|
|
236
239
|
|
237
240
|
## features
|
238
241
|
|
242
|
+
also see [comparison to similar software](./docs/versus.md)
|
243
|
+
|
239
244
|
* backend stuff
|
240
245
|
* ☑ IPv6
|
241
246
|
* ☑ [multiprocessing](#performance) (actual multithreading)
|
@@ -258,6 +263,7 @@ firewall-cmd --reload
|
|
258
263
|
* ☑ write-only folders
|
259
264
|
* ☑ [unpost](#unpost): undo/delete accidental uploads
|
260
265
|
* ☑ [self-destruct](#self-destruct) (specified server-side or client-side)
|
266
|
+
* ☑ [race the beam](#race-the-beam) (almost like peer-to-peer)
|
261
267
|
* ☑ symlink/discard duplicates (content-matching)
|
262
268
|
* download
|
263
269
|
* ☑ single files in browser
|
@@ -683,7 +689,7 @@ up2k has several advantages:
|
|
683
689
|
> it is perfectly safe to restart / upgrade copyparty while someone is uploading to it!
|
684
690
|
> all known up2k clients will resume just fine 💪
|
685
691
|
|
686
|
-
see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
|
692
|
+
see [up2k](./docs/devnotes.md#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
|
687
693
|
|
688
694
|

|
689
695
|
|
@@ -749,6 +755,13 @@ clients can specify a shorter expiration time using the [up2k ui](#uploading) --
|
|
749
755
|
specifying a custom expiration time client-side will affect the timespan in which unposts are permitted, so keep an eye on the estimates in the up2k ui
|
750
756
|
|
751
757
|
|
758
|
+
### race the beam
|
759
|
+
|
760
|
+
download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm)) -- it's almost like peer-to-peer
|
761
|
+
|
762
|
+
requires the file to be uploaded using up2k (which is the default drag-and-drop uploader), alternatively the command-line program
|
763
|
+
|
764
|
+
|
752
765
|
## file manager
|
753
766
|
|
754
767
|
cut/paste, rename, and delete files/folders (if you have permission)
|
@@ -1,19 +1,19 @@
|
|
1
1
|
copyparty/__init__.py,sha256=fUINM1abqDGzCCH_JcXdOnLdKOV-SrTI2Xo2QgQW2P4,1703
|
2
|
-
copyparty/__main__.py,sha256=
|
3
|
-
copyparty/__version__.py,sha256=
|
2
|
+
copyparty/__main__.py,sha256=u0RNQ2iZwlKCqngQofFjDsYyKKD50s5q_oht8RONSuU,95386
|
3
|
+
copyparty/__version__.py,sha256=Zw2SLEDiKYiPpg31tkxLz17OfRS4iKV7v7yLmGIf3fE,255
|
4
4
|
copyparty/authsrv.py,sha256=-MZQnSsrEGD8cTtR90qT5-xoc9xD8PP8cjvBL75HMKo,84501
|
5
5
|
copyparty/broker_mp.py,sha256=4mEZC5tiHUazJMgYuwInNo2dxS7jrbzrGb1qs2UBt9k,3948
|
6
6
|
copyparty/broker_mpw.py,sha256=4ZI7bJYOwUibeAJVv9_FPGNmHrr9eOtkj_Kz0JEppTU,3197
|
7
7
|
copyparty/broker_thr.py,sha256=eKr--HJGig5zqvNGwH9UoBG9Nvi9mT2axrRmJwknd0s,1759
|
8
8
|
copyparty/broker_util.py,sha256=CnX_LAhQQqouONcDLtVkVlcBX3Z6pWuKDQDmmbHGEg4,1489
|
9
9
|
copyparty/cert.py,sha256=nCeDdzcCpvjPPUcxT4Oh7wvL_8zvddu4oXtbA-zOb8g,7607
|
10
|
-
copyparty/cfg.py,sha256=
|
10
|
+
copyparty/cfg.py,sha256=LawUJv8faoWrFWPudtWpcRlakrW6dp8uUKMYR86Nza8,9305
|
11
11
|
copyparty/dxml.py,sha256=lZpg-kn-kQsXRtNY1n6fRaS-b7uXzMCyv8ovKnhZcZc,1548
|
12
12
|
copyparty/fsutil.py,sha256=c4fTvmclKbVABNsjU4rGddsjCgRwi9YExAyo-06ATc8,3932
|
13
13
|
copyparty/ftpd.py,sha256=OIExjfqOEw-Y_ygez6cIZUQec4SFOmoxEH_WOVvw-aE,15961
|
14
|
-
copyparty/httpcli.py,sha256=
|
15
|
-
copyparty/httpconn.py,sha256=
|
16
|
-
copyparty/httpsrv.py,sha256=
|
14
|
+
copyparty/httpcli.py,sha256=aIMmB_kMvI7PbDvHw5WCmMVMdHPuLJ2rer_NPlgMuvE,155100
|
15
|
+
copyparty/httpconn.py,sha256=6MOQgBtOGrlVRr6ZiHBKYzkzcls-YWwaWEtqE6DweM0,6873
|
16
|
+
copyparty/httpsrv.py,sha256=Xf6wI5V25gzAoyEpiKH8VjEFwUqTzW5z8pcRfo2J40c,16421
|
17
17
|
copyparty/ico.py,sha256=AYHdK6NlYBfBgafVYXia3jHQ9XHZdUL1D8WftLMAzIU,3545
|
18
18
|
copyparty/mdns.py,sha256=CcraggbDxTT1ntYzD8Ebgqmw5Q4HkyZcfh5ymtCV_ak,17469
|
19
19
|
copyparty/metrics.py,sha256=OqXFkAuoVhayGAGd_Sv-OQ9SVmdXYV8M7CxitkzE3lo,8854
|
@@ -31,8 +31,8 @@ copyparty/tftpd.py,sha256=7EHAZ9LnjAXupwRNIENJ2eA8Q0lFynnwwbziV3fyzns,13157
|
|
31
31
|
copyparty/th_cli.py,sha256=eSW7sBiaZAsh_XffXFzb035CTSbS3J3Q0G-BMzQGuSY,4385
|
32
32
|
copyparty/th_srv.py,sha256=C2ZBE6ddINCuYDympRQQmhj0ULdlD6HOM6qNK-UB4so,27191
|
33
33
|
copyparty/u2idx.py,sha256=JBEqKX1ZM8GIvQrDYb5VQ_5QiFNFsjWF6H9drHlPVEY,12709
|
34
|
-
copyparty/up2k.py,sha256=
|
35
|
-
copyparty/util.py,sha256=
|
34
|
+
copyparty/up2k.py,sha256=co26DRvhYgwHpwbvuXUDQlCWSMjDZiIE29KVXHMwiKs,142833
|
35
|
+
copyparty/util.py,sha256=f9e6vxtnsHxBkctkcJX_iaAg_2AWk4K4p67MMzox144,82942
|
36
36
|
copyparty/bos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
37
|
copyparty/bos/bos.py,sha256=Wb7eWsXJgR5AFlBR9ZOyKrLTwy-Kct9RrGiOu4Jo37Y,1622
|
38
38
|
copyparty/bos/path.py,sha256=yEjCq2ki9CvxA5sCT8pS0keEXwugs0ZeUyUhdBziOCI,777
|
@@ -55,9 +55,9 @@ copyparty/stolen/ifaddr/_posix.py,sha256=-67NdfGrCktfQPakT2fLbjl2U00QMvyBGkSvrUu
|
|
55
55
|
copyparty/stolen/ifaddr/_shared.py,sha256=cJACl8cOxQ-HSYphZTzKMAjAx_TAFyJwUPjfD102Xqw,6111
|
56
56
|
copyparty/stolen/ifaddr/_win32.py,sha256=EE-QyoBgeB7lYQ6z62VjXNaRozaYfCkaJBHGNA8QtZM,4026
|
57
57
|
copyparty/web/baguettebox.js.gz,sha256=Qcx5ZJWWCU4S1J0ULVXuVKWnm_SuCiEknMlt_uwIkJ8,7830
|
58
|
-
copyparty/web/browser.css.gz,sha256=
|
58
|
+
copyparty/web/browser.css.gz,sha256=VruUcE9yZm8bpJrPml1lcnJwznaR43Db864qW8Rv4d4,11470
|
59
59
|
copyparty/web/browser.html,sha256=uAejLJd11rV_tQx3h2nHnJ1XY6zn1JV-meIAv74Lc8o,4873
|
60
|
-
copyparty/web/browser.js.gz,sha256=
|
60
|
+
copyparty/web/browser.js.gz,sha256=_ePuKLuMIrJbKNcwqXcipJHiVzK5ZiaG5JtEExTy440,67893
|
61
61
|
copyparty/web/browser2.html,sha256=ciQlgr9GWuIapdsRBFNRvRFvN5T_5n920LqDMbsj5-g,1605
|
62
62
|
copyparty/web/cf.html,sha256=lJThtNFNAQT1ClCHHlivAkDGE0LutedwopXD62Z8Nys,589
|
63
63
|
copyparty/web/dbg-audio.js.gz,sha256=Ma-KZtK8LnmiwNvNKFKXMPYl_Nn_3U7GsJ6-DRWC2HE,688
|
@@ -77,12 +77,12 @@ copyparty/web/splash.js.gz,sha256=2R8UYlAN8WpIABg8clgWckWqgD8nKtz3eGZFu2y1g88,14
|
|
77
77
|
copyparty/web/svcs.html,sha256=s7vUSrCrELC3iTemksodRBhQpssO7s4xW1vA-CX6vU8,11702
|
78
78
|
copyparty/web/svcs.js.gz,sha256=k81ZvZ3I-f4fMHKrNGGOgOlvXnCBz0mVjD-8mieoWCA,520
|
79
79
|
copyparty/web/ui.css.gz,sha256=skuzZHqTU0ag5hButpQmKI9wM7ro-UJ2PnpTodTWYF4,2616
|
80
|
-
copyparty/web/up2k.js.gz,sha256=
|
80
|
+
copyparty/web/up2k.js.gz,sha256=3IKVXjZq7byJWFKyHVylIIbWozsJ6IL7CrOUCibE8BY,22114
|
81
81
|
copyparty/web/util.js.gz,sha256=3Ys57MotfGguhuCdDhj3afyX8wlz3ZgG5ZrnY4_Zqcw,14377
|
82
82
|
copyparty/web/w.hash.js.gz,sha256=__hBMd5oZWfTrb8ZCJNT21isoSqyrxKE6qdaKGQVAhc,1060
|
83
83
|
copyparty/web/a/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
84
84
|
copyparty/web/a/partyfuse.py,sha256=MuRkaSuYsdfWfBFMOkbPwDXqSvNTw3sd7QhhlKCDZ8I,32311
|
85
|
-
copyparty/web/a/u2c.py,sha256=
|
85
|
+
copyparty/web/a/u2c.py,sha256=qyK4G1mkICAjmo99YV8ubi2Zk6GG8S8yldW6D18Pnos,38626
|
86
86
|
copyparty/web/a/webdav-cfg.bat,sha256=Y4NoGZlksAIg4cBMb7KdJrpKC6Nx97onaTl6yMjaimk,1449
|
87
87
|
copyparty/web/dd/2.png,sha256=gJ14XFPzaw95L6z92fSq9eMPikSQyu-03P1lgiGe0_I,258
|
88
88
|
copyparty/web/dd/3.png,sha256=4lho8Koz5tV7jJ4ODo6GMTScZfkqsT05yp48EDFIlyg,252
|
@@ -102,9 +102,9 @@ copyparty/web/deps/prismd.css.gz,sha256=ObUlksQVr-OuYlTz-I4B23TeBg2QDVVGRnWBz8cV
|
|
102
102
|
copyparty/web/deps/scp.woff2,sha256=w99BDU5i8MukkMEL-iW0YO9H4vFFZSPWxbkH70ytaAg,8612
|
103
103
|
copyparty/web/deps/sha512.ac.js.gz,sha256=lFZaCLumgWxrvEuDr4bqdKHsqjX82AbVAb7_F45Yk88,7033
|
104
104
|
copyparty/web/deps/sha512.hw.js.gz,sha256=vqoXeracj-99Z5MfY3jK2N4WiSzYQdfjy0RnUlQDhSU,8110
|
105
|
-
copyparty-1.
|
106
|
-
copyparty-1.
|
107
|
-
copyparty-1.
|
108
|
-
copyparty-1.
|
109
|
-
copyparty-1.
|
110
|
-
copyparty-1.
|
105
|
+
copyparty-1.13.0.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
|
106
|
+
copyparty-1.13.0.dist-info/METADATA,sha256=sa7IcvkdDV532CI8LFuQJg8dzUbplA45cB8Qmn3Ppmw,119305
|
107
|
+
copyparty-1.13.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
108
|
+
copyparty-1.13.0.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
|
109
|
+
copyparty-1.13.0.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
|
110
|
+
copyparty-1.13.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|