copyparty 1.15.7__py3-none-any.whl → 1.15.8__py3-none-any.whl

copyparty/__main__.py

@@ -1009,7 +1009,7 @@ def add_upload(ap):
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
     ap2.add_argument("--u2j", metavar="JOBS", type=int, default=2, help="web-client: number of file chunks to upload in parallel; 1 or 2 is good for low-latency (same-country) connections, 4-8 for android clients, 16 for cross-atlantic (max=64)")
-    ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for this size. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
+    ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for \033[33mdefault\033[0m, and never exceed \033[33mmax\033[0m. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
     ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 15, 7)
+VERSION = (1, 15, 8)
 CODENAME = "fill the drives"
-BUILD_DT = (2024, 10, 14)
+BUILD_DT = (2024, 10, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -59,6 +59,7 @@ if PY2:
 LEELOO_DALLAS = "leeloo_dallas"
 
 SEE_LOG = "see log for details"
+SEESLOG = " (see serverlog for details)"
 SSEELOG = " ({})".format(SEE_LOG)
 BAD_CFG = "invalid config; {}".format(SEE_LOG)
 SBADCFG = " ({})".format(BAD_CFG)

copyparty/httpcli.py

@@ -1874,7 +1874,7 @@ class HttpCli(object):
         f, fn = ren_open(fn, *open_a, **params)
         try:
             path = os.path.join(fdir, fn)
-            post_sz, sha_hex, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
+            post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
         finally:
             f.close()
 
@@ -2337,7 +2337,7 @@ class HttpCli(object):
         broker = self.conn.hsrv.broker
         x = broker.ask("up2k.handle_chunks", ptop, wark, chashes)
         response = x.get()
-        chashes, chunksize, cstarts, path, lastmod, sprs = response
+        chashes, chunksize, cstarts, path, lastmod, fsize, sprs = response
         maxsize = chunksize * len(chashes)
         cstart0 = cstarts[0]
         locked = chashes  # remaining chunks to be received in this request
@@ -2345,6 +2345,50 @@ class HttpCli(object):
         num_left = -1  # num chunks left according to most recent up2k release
         treport = time.time()  # ratelimit up2k reporting to reduce overhead
 
+        if "x-up2k-subc" in self.headers:
+            sc_ofs = int(self.headers["x-up2k-subc"])
+            chash = chashes[0]
+
+            u2sc = self.conn.hsrv.u2sc
+            try:
+                sc_pofs, hasher = u2sc[chash]
+                if not sc_ofs:
+                    t = "client restarted the chunk; forgetting subchunk offset %d"
+                    self.log(t % (sc_pofs,))
+                    raise Exception()
+            except:
+                sc_pofs = 0
+                hasher = hashlib.sha512()
+
+            et = "subchunk protocol error; resetting chunk "
+            if sc_pofs != sc_ofs:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: the expected resume-point was %d, not %d"
+                raise Pebkac(400, t % (et, chash, sc_pofs, sc_ofs))
+            if len(cstarts) > 1:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: only a single subchunk can be uploaded in one request; you are sending %d chunks"
+                raise Pebkac(400, t % (et, chash, len(cstarts)))
+            csize = min(chunksize, fsize - cstart0[0])
+            cstart0[0] += sc_ofs  # also sets cstarts[0][0]
+            sc_next_ofs = sc_ofs + postsize
+            if sc_next_ofs > csize:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: subchunk offset (%d) plus postsize (%d) exceeds chunksize (%d)"
+                raise Pebkac(400, t % (et, chash, sc_ofs, postsize, csize))
+            else:
+                final_subchunk = sc_next_ofs == csize
+                t = "subchunk %s %d:%d/%d %s"
+                zs = "END" if final_subchunk else ""
+                self.log(t % (chash[:15], sc_ofs, sc_next_ofs, csize, zs), 6)
+                if final_subchunk:
+                    u2sc.pop(chash, None)
+                else:
+                    u2sc[chash] = (sc_next_ofs, hasher)
+        else:
+            hasher = None
+            final_subchunk = True
+
         try:
             if self.args.nw:
                 path = os.devnull
@@ -2375,9 +2419,11 @@ class HttpCli(object):
                     reader = read_socket(
                         self.sr, self.args.s_rd_sz, min(remains, chunksize)
                     )
-                    post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
+                    post_sz, _, sha_b64 = hashcopy(
+                        reader, f, hasher, 0, self.args.s_wr_slp
+                    )
 
-                    if sha_b64 != chash:
+                    if sha_b64 != chash and final_subchunk:
                         try:
                             self.bakflip(
                                 f, path, cstart[0], post_sz, chash, sha_b64, vfs.flags
@@ -2409,7 +2455,8 @@ class HttpCli(object):
 
                     # be quick to keep the tcp winsize scale;
                     # if we can't confirm rn then that's fine
-                    written.append(chash)
+                    if final_subchunk:
+                        written.append(chash)
                     now = time.time()
                     if now - treport < 1:
                         continue
@@ -2797,7 +2844,7 @@ class HttpCli(object):
                         tabspath = os.path.join(fdir, tnam)
                         self.log("writing to {}".format(tabspath))
                         sz, sha_hex, sha_b64 = hashcopy(
-                            p_data, f, self.args.s_wr_slp, max_sz
+                            p_data, f, None, max_sz, self.args.s_wr_slp
                         )
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
@@ -3127,7 +3174,7 @@ class HttpCli(object):
             wunlink(self.log, fp, vfs.flags)
 
         with open(fsenc(fp), "wb", self.args.iobuf) as f:
-            sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
+            sz, sha512, _ = hashcopy(p_data, f, None, 0, self.args.s_wr_slp)
 
         if lim:
             lim.nup(self.ip)

copyparty/httpsrv.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import hashlib
 import math
 import os
 import re
@@ -141,6 +142,7 @@ class HttpSrv(object):
         self.t_periodic  = None
 
         self.u2fh = FHC()
+        self.u2sc    = {}
         self.pipes = CachedDict(0.2)
         self.metrics = Metrics(self)
         self.nreq = 0

copyparty/up2k.py

@@ -20,7 +20,7 @@ from copy import deepcopy
 from queue import Queue
 
 from .__init__ import ANYWIN, PY2, TYPE_CHECKING, WINDOWS, E
-from .authsrv import LEELOO_DALLAS, SSEELOG, VFS, AuthSrv
+from .authsrv import LEELOO_DALLAS, SEESLOG, VFS, AuthSrv
 from .bos import bos
 from .cfg import vf_bmap, vf_cmap, vf_vmap
 from .fsutil import Fstab
@@ -2876,9 +2876,6 @@ class Up2k(object):
                         "user": cj["user"],
                         "addr": ip,
                         "at": at,
-                        "hash": [],
-                        "need": [],
-                        "busy": {},
                     }
                     for k in ["life"]:
                         if k in cj:
@@ -2912,17 +2909,20 @@ class Up2k(object):
                     hashes2, st = self._hashlist_from_file(orig_ap)
                     wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
                     if dwark != wark2:
-                        t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s"
-                        self.log(t % (wark2, dwark, orig_ap))
+                        t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s\n%s"
+                        self.log(t % (wark2, dwark, orig_ap, rj))
                         lost.append(dupe[3:])
                         continue
                     data_ok = True
                     job = rj
                     break
 
-            if job and wark in reg:
-                # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
-                del reg[wark]
+            if job:
+                if wark in reg:
+                    del reg[wark]
+                job["hash"] = job["need"] = []
+                job["done"] = True
+                job["busy"] = {}
 
             if lost:
                 c2 = None
@@ -2950,7 +2950,7 @@ class Up2k(object):
                     path = djoin(rj["ptop"], rj["prel"], fn)
                     try:
                         st = bos.stat(path)
-                        if st.st_size > 0 or not rj["need"]:
+                        if st.st_size > 0 or "done" in rj:
                             # upload completed or both present
                             break
                     except:
@@ -2964,13 +2964,13 @@ class Up2k(object):
                 inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
                 orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
 
-                if self.args.nw or n4g or not st:
+                if self.args.nw or n4g or not st or "done" not in rj:
                     pass
 
                 elif st.st_size != rj["size"]:
-                    t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}"
+                    t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}\n{}"
                     t = t.format(
-                        wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path
+                        wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path, rj
                     )
                     self.log(t)
                     del reg[wark]
@@ -2980,8 +2980,8 @@ class Up2k(object):
                     hashes2, _ = self._hashlist_from_file(orig_ap)
                     wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
                     if wark != wark2:
-                        t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s"
-                        self.log(t % (wark2, wark, orig_ap))
+                        t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s\n%s"
+                        self.log(t % (wark2, wark, orig_ap, rj))
                         del reg[wark]
 
             if job or wark in reg:
@@ -2996,7 +2996,7 @@ class Up2k(object):
                     dst = djoin(cj["ptop"], cj["prel"], cj["name"])
                     vsrc = djoin(job["vtop"], job["prel"], job["name"])
                     vsrc = vsrc.replace("\\", "/")  # just for prints anyways
-                    if job["need"]:
+                    if "done" not in job:
                         self.log("unfinished:\n  {0}\n  {1}".format(src, dst))
                         err = "partial upload exists at a different location; please resume uploading here instead:\n"
                         err += "/" + quotep(vsrc) + " "
@@ -3357,14 +3357,14 @@ class Up2k(object):
 
     def handle_chunks(
         self, ptop , wark , chashes 
-    )       :
+    )        :
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
             if not job:
                 known = " ".join([x for x in self.registry[ptop].keys()])
                 self.log("unknown wark [{}], known: {}".format(wark, known))
-                raise Pebkac(400, "unknown wark" + SSEELOG)
+                raise Pebkac(400, "unknown wark" + SEESLOG)
 
             if "t0c" not in job:
                 job["t0c"] = time.time()
@@ -3380,7 +3380,7 @@ class Up2k(object):
                 try:
                     nchunk = uniq.index(chashes[0])
                 except:
-                    raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
+                    raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0],))
                 expanded = [chashes[0]]
                 for prefix in chashes[1:]:
                     nchunk += 1
@@ -3414,7 +3414,7 @@ class Up2k(object):
             for chash in chashes:
                 nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
                 if not nchunk:
-                    raise Pebkac(400, "unknown chunk %s" % (chash))
+                    raise Pebkac(400, "unknown chunk %s" % (chash,))
 
                 ofs = [chunksize * x for x in nchunk]
                 coffsets.append(ofs)
@@ -3439,7 +3439,7 @@ class Up2k(object):
 
         job["poke"] = time.time()
 
-        return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
+        return chashes, chunksize, coffsets, path, job["lmod"], job["size"], job["sprs"]
 
     def fast_confirm_chunks(
         self, ptop , wark , chashes 
@@ -3508,11 +3508,13 @@ class Up2k(object):
             src = djoin(pdir, job["tnam"])
             dst = djoin(pdir, job["name"])
         except Exception as ex:
-            raise Pebkac(500, "finish_upload, wark, " + repr(ex))
+            self.log(min_ex(), 1)
+            raise Pebkac(500, "finish_upload, wark, %r%s" % (ex, SEESLOG))
 
         if job["need"]:
-            t = "finish_upload {} with remaining chunks {}"
-            raise Pebkac(500, t.format(wark, job["need"]))
+            self.log(min_ex(), 1)
+            t = "finish_upload %s with remaining chunks %s%s"
+            raise Pebkac(500, t % (wark, job["need"], SEESLOG))
 
         upt = job.get("at") or time.time()
         vflags = self.flags[ptop]
@@ -4033,7 +4035,9 @@ class Up2k(object):
                         self.db_act = self.vol_act[dbv.realpath] = time.time()
                         svpf = "/".join(x for x in [dbv.vpath, vrem, fn[0]] if x)
                         if not svpf.startswith(svp + "/"):  # assert
-                            raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
+                            self.log(min_ex(), 1)
+                            t = "mv: bug at %s, top %s%s"
+                            raise Pebkac(500, t % (svpf, svp, SEESLOG))
 
                         dvpf = dvp + svpf[len(svp) :]
                         self._mv_file(uname, ip, svpf, dvpf, curs)
@@ -4048,7 +4052,9 @@ class Up2k(object):
         for zsl in (rm_ok, rm_ng):
             for ap in reversed(zsl):
                 if not ap.startswith(sabs):
-                    raise Pebkac(500, "mv_d: bug at {}, top {}".format(ap, sabs))
+                    self.log(min_ex(), 1)
+                    t = "mv_d: bug at %s, top %s%s"
+                    raise Pebkac(500, t % (ap, sabs, SEESLOG))
 
                 rem = ap[len(sabs) :].replace(os.sep, "/").lstrip("/")
                 vp = vjoin(dvp, rem)

copyparty/util.py

@@ -2637,10 +2637,12 @@ def yieldfile(fn , bufsz )    :
 def hashcopy(
     fin   ,
     fout  ,
-    slp  = 0,
-    max_sz  = 0,
+    hashobj ,
+    max_sz ,
+    slp ,
 )    :
-    hashobj = hashlib.sha512()
+    if not hashobj:
+        hashobj = hashlib.sha512()
     tlen = 0
     for buf in fin:
         tlen += len(buf)

copyparty/web/a/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "2.2"
-S_BUILD_DT = "2024-10-13"
+S_VERSION = "2.4"
+S_BUILD_DT = "2024-10-16"
 
 """
 u2c.py: upload to copyparty
@@ -62,6 +62,9 @@ else:
 
     unicode = str
 
+
+WTF8 = "replace" if PY2 else "surrogateescape"
+
 VT100 = platform.system() != "Windows"
 
 
@@ -228,7 +231,7 @@ class File(object):
         self.lmod = lmod  # type: float
 
         self.abs = os.path.join(top, rel)  # type: bytes
-        self.name = self.rel.split(b"/")[-1].decode("utf-8", "replace")  # type: str
+        self.name = self.rel.split(b"/")[-1].decode("utf-8", WTF8)  # type: str
 
         # set by get_hashlist
         self.cids = []  # type: list[tuple[str, int, int]]  # [ hash, ofs, sz ]
@@ -267,10 +270,41 @@ class FileSlice(object):
                 raise Exception(9)
             tlen += clen
 
-        self.len = tlen
+        self.len = self.tlen = tlen
         self.cdr = self.car + self.len
         self.ofs = 0  # type: int
-        self.f = open(file.abs, "rb", 512 * 1024)
+
+        self.f = None
+        self.seek = self._seek0
+        self.read = self._read0
+
+    def subchunk(self, maxsz, nth):
+        if self.tlen <= maxsz:
+            return -1
+
+        if not nth:
+            self.car0 = self.car
+            self.cdr0 = self.cdr
+
+        self.car = self.car0 + maxsz * nth
+        if self.car >= self.cdr0:
+            return -2
+
+        self.cdr = self.car + min(self.cdr0 - self.car, maxsz)
+        self.len = self.cdr - self.car
+        self.seek(0)
+        return nth
+
+    def unsub(self):
+        self.car = self.car0
+        self.cdr = self.cdr0
+        self.len = self.tlen
+
+    def _open(self):
+        self.seek = self._seek
+        self.read = self._read
+
+        self.f = open(self.file.abs, "rb", 512 * 1024)
         self.f.seek(self.car)
 
         # https://stackoverflow.com/questions/4359495/what-is-exactly-a-file-like-object-in-python
@@ -282,10 +316,14 @@ class FileSlice(object):
         except:
             pass  # py27 probably
 
+    def close(self, *a, **ka):
+        return  # until _open
+
     def tell(self):
         return self.ofs
 
-    def seek(self, ofs, wh=0):
+    def _seek(self, ofs, wh=0):
+
         if wh == 1:
             ofs = self.ofs + ofs
         elif wh == 2:
@@ -299,12 +337,21 @@ class FileSlice(object):
         self.ofs = ofs
         self.f.seek(self.car + ofs)
 
-    def read(self, sz):
+    def _read(self, sz):
+
         sz = min(sz, self.len - self.ofs)
         ret = self.f.read(sz)
         self.ofs += len(ret)
         return ret
 
+    def _seek0(self, ofs, wh=0):
+        self._open()
+        return self.seek(ofs, wh)
+
+    def _read0(self, sz):
+        self._open()
+        return self.read(sz)
+
 
 class MTHash(object):
     def __init__(self, cores):
@@ -557,13 +604,17 @@ def walkdir(err, top, excl, seen):
     for ap, inf in sorted(statdir(err, top)):
         if excl.match(ap):
             continue
-        yield ap, inf
         if stat.S_ISDIR(inf.st_mode):
+            yield ap, inf
             try:
                 for x in walkdir(err, ap, excl, seen):
                     yield x
             except Exception as ex:
                 err.append((ap, str(ex)))
+        elif stat.S_ISREG(inf.st_mode):
+            yield ap, inf
+        else:
+            err.append((ap, "irregular filetype 0%o" % (inf.st_mode,)))
 
 
 def walkdirs(err, tops, excl):
@@ -609,11 +660,12 @@ def walkdirs(err, tops, excl):
 
 # mostly from copyparty/util.py
 def quotep(btxt):
+    # type: (bytes) -> bytes
     quot1 = quote(btxt, safe=b"/")
     if not PY2:
         quot1 = quot1.encode("ascii")
 
-    return quot1.replace(b" ", b"+")  # type: ignore
+    return quot1.replace(b" ", b"%20")  # type: ignore
 
 
 # from copyparty/util.py
@@ -641,7 +693,7 @@ def up2k_chunksize(filesize):
     while True:
         for mul in [1, 2]:
             nchunks = math.ceil(filesize * 1.0 / chunksize)
-            if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks < 4096):
+            if nchunks <= 256 or (chunksize >= 32 * 1024 * 1024 and nchunks <= 4096):
                 return chunksize
 
             chunksize += stepsize
@@ -720,7 +772,7 @@ def handshake(ar, file, search):
         url = file.url
     else:
         if b"/" in file.rel:
-            url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8", "replace")
+            url = quotep(file.rel.rsplit(b"/", 1)[0]).decode("utf-8")
         else:
             url = ""
         url = ar.vtop + url
@@ -766,15 +818,15 @@ def handshake(ar, file, search):
     if search:
         return r["hits"], False
 
-    file.url = r["purl"]
+    file.url = quotep(r["purl"].encode("utf-8", WTF8)).decode("utf-8")
     file.name = r["name"]
     file.wark = r["wark"]
 
     return r["hash"], r["sprs"]
 
 
-def upload(fsl, stats):
-    # type: (FileSlice, str) -> None
+def upload(fsl, stats, maxsz):
+    # type: (FileSlice, str, int) -> None
     """upload a range of file data, defined by one or more `cid` (chunk-hash)"""
 
     ctxt = fsl.cids[0]
@@ -792,21 +844,33 @@ def upload(fsl, stats):
     if stats:
         headers["X-Up2k-Stat"] = stats
 
+    nsub = 0
     try:
-        sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
-
-        if sc == 400:
-            if (
-                "already being written" in txt
-                or "already got that" in txt
-                or "only sibling chunks" in txt
-            ):
-                fsl.file.nojoin = 1
-
-        if sc >= 400:
-            raise Exception("http %s: %s" % (sc, txt))
+        while nsub != -1:
+            nsub = fsl.subchunk(maxsz, nsub)
+            if nsub == -2:
+                return
+            if nsub >= 0:
+                headers["X-Up2k-Subc"] = str(maxsz * nsub)
+                headers.pop(CLEN, None)
+                nsub += 1
+
+            sc, txt = web.req("POST", fsl.file.url, headers, fsl, MO)
+
+            if sc == 400:
+                if (
+                    "already being written" in txt
+                    or "already got that" in txt
+                    or "only sibling chunks" in txt
+                ):
+                    fsl.file.nojoin = 1
+
+            if sc >= 400:
+                raise Exception("http %s: %s" % (sc, txt))
     finally:
         fsl.f.close()
+        if nsub != -1:
+            fsl.unsub()
 
 
 class Ctl(object):
@@ -938,7 +1002,7 @@ class Ctl(object):
                     print("  %d up %s" % (ncs - nc, cid))
                     stats = "%d/0/0/%d" % (nf, self.nfiles - nf)
                     fslice = FileSlice(file, [cid])
-                    upload(fslice, stats)
+                    upload(fslice, stats, self.ar.szm)
 
             print("  ok!")
             if file.recheck:
@@ -1057,7 +1121,7 @@ class Ctl(object):
                         print("      ls ~{0}".format(srd))
                         zt = (
                             self.ar.vtop,
-                            quotep(rd.replace(b"\\", b"/")).decode("utf-8", "replace"),
+                            quotep(rd.replace(b"\\", b"/")).decode("utf-8"),
                         )
                         sc, txt = web.req("GET", "%s%s?ls&lt&dots" % zt, {})
                         if sc >= 400:
@@ -1066,7 +1130,7 @@ class Ctl(object):
                         j = json.loads(txt)
                         for f in j["dirs"] + j["files"]:
                             rfn = f["href"].split("?")[0].rstrip("/")
-                            ls[unquote(rfn.encode("utf-8", "replace"))] = f
+                            ls[unquote(rfn.encode("utf-8", WTF8))] = f
                     except Exception as ex:
                         print("   mkdir ~{0}  ({1})".format(srd, ex))
 
@@ -1080,7 +1144,7 @@ class Ctl(object):
                             lnodes = [x.split(b"/")[-1] for x in zls]
                         bnames = [x for x in ls if x not in lnodes and x != b".hist"]
                         vpath = self.ar.url.split("://")[-1].split("/", 1)[-1]
-                        names = [x.decode("utf-8", "replace") for x in bnames]
+                        names = [x.decode("utf-8", WTF8) for x in bnames]
                         locs = [vpath + srd + "/" + x for x in names]
                         while locs:
                             req = locs
@@ -1286,7 +1350,7 @@ class Ctl(object):
                 self._check_if_done()
                 continue
 
-            njoin = (self.ar.sz * 1024 * 1024) // chunksz
+            njoin = self.ar.sz // chunksz
             cs = hs[:]
             while cs:
                 fsl = FileSlice(file, cs[:1])
@@ -1338,7 +1402,7 @@ class Ctl(object):
             )
 
             try:
-                upload(fsl, stats)
+                upload(fsl, stats, self.ar.szm)
             except Exception as ex:
                 t = "upload failed, retrying: %s #%s+%d (%s)\n"
                 eprint(t % (file.name, cids[0][:8], len(cids) - 1, ex))
@@ -1427,6 +1491,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
     ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("--sz", type=int, metavar="MiB", default=64, help="try to make each POST this big")
+    ap.add_argument("--szm", type=int, metavar="MiB", default=96, help="max size of each POST (default is cloudflare max)")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
     ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
@@ -1454,6 +1519,9 @@ source file/folder selection uses rsync syntax, meaning that:
     if ar.dr:
         ar.ow = True
 
+    ar.sz *= 1024 * 1024
+    ar.szm *= 1024 * 1024
+
     ar.x = "|".join(ar.x or [])
 
     setattr(ar, "wlist", ar.url == "-")

{copyparty-1.15.7.dist-info → copyparty-1.15.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: copyparty
-Version: 1.15.7
+Version: 1.15.8
 Summary: Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
 Author-email: ed <copyparty@ocv.me>
 License: MIT
@@ -273,7 +273,7 @@ also see [comparison to similar software](./docs/versus.md)
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
-    * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
+    * **no filesize limit!** even on Cloudflare
   * ☑ stash: simple PUT filedropper
   * ☑ filename randomizer
   * ☑ write-only folders
@@ -708,7 +708,7 @@ up2k has several advantages:
   * uploads resume if you reboot your browser or pc, just upload the same files again
   * server detects any corruption; the client reuploads affected chunks
   * the client doesn't upload anything that already exists on the server
-  * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
+  * no filesize limit, even when a proxy limits the request size (for example Cloudflare)
 * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
 * the last-modified timestamp of the file is preserved
 
@@ -744,6 +744,8 @@ note that since up2k has to read each file twice, `[🎈] bup` can *theoreticall
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
+if the server is behind a proxy which imposes a request-size limit, you can configure up2k to sneak below the limit with server-option `--u2sz` (the default is 96 MiB to support Cloudflare)
+
 
 ### file-search
 

{copyparty-1.15.7.dist-info → copyparty-1.15.8.dist-info}/RECORD

@@ -1,7 +1,7 @@
 copyparty/__init__.py,sha256=Chqw7uXX4r_-a2p6-xthrrqVHFI4aZdW45sWU7UvqeE,2597
-copyparty/__main__.py,sha256=jPcqdsSJZP6AMoPygb6t8PwxmLXGK3jUeUvt28JcnyE,110130
-copyparty/__version__.py,sha256=Z9byW4PqObsayHm5sQR8iTi3mFCP5dkS0YnU-6VoRT4,258
-copyparty/authsrv.py,sha256=clsE8whf32_eIRES1r7VvsVkc4OiJCjiBEJM6pYkwwI,98670
+copyparty/__main__.py,sha256=oTzWyuZotTwi-dHpD41mVp_EW2mPSWDMKP8o_nxp9Yk,110180
+copyparty/__version__.py,sha256=V6jLFRwMXDp4buHI89fYcQoINvqpP0XYLPVMn_QHEDg,258
+copyparty/authsrv.py,sha256=-lImrFH6pm3gcI76vZiFfEgrQx3_STYTSS2soYX5y1Y,98711
 copyparty/broker_mp.py,sha256=jsHUM2BSfRVRyZT869iPCqYEHSqedk6VkwvygZwbEZE,4017
 copyparty/broker_mpw.py,sha256=PYFgQfssOCfdI6qayW1ZjO1j1-7oez094muhYMbPOz0,3339
 copyparty/broker_thr.py,sha256=MXrwjusP0z1LPURUhi5jx_TL3jrXhYcDrJPDSKu6EEU,1705
@@ -11,9 +11,9 @@ copyparty/cfg.py,sha256=33nLatBUmzRFKQ4KpoQei3ZY6EqRrlaHpQnvCNFXcHI,10112
 copyparty/dxml.py,sha256=lZpg-kn-kQsXRtNY1n6fRaS-b7uXzMCyv8ovKnhZcZc,1548
 copyparty/fsutil.py,sha256=5CshJWO7CflfaRRNOb3JxghUH7W5rmS_HWNmKfx42MM,4538
 copyparty/ftpd.py,sha256=G_h1urfIikzfCWGXnW9p-rioWdNM_Je6vWYq0-QSbC8,17580
-copyparty/httpcli.py,sha256=PIlYJa-1QTzsASD9jRbRJGsn8vAUNSg15qJEigcs9cs,192083
+copyparty/httpcli.py,sha256=xhM8unCDyo7Gj3hmLdhncXUCgKHgb6a300bolO1WF4U,194152
 copyparty/httpconn.py,sha256=mQSgljh0Q-jyWjF4tQLrHbRKRe9WKl19kGqsGMsJpWo,6880
-copyparty/httpsrv.py,sha256=k0xSadmUNSyUAWlZdOMFdsBP1fQK50Ptv7O6O4jFy2Q,17182
+copyparty/httpsrv.py,sha256=d_UiGnQKniBoEV68lNFgnYm-byda7uj56mFf-YC7piI,17223
 copyparty/ico.py,sha256=eWSxEae4wOCfheHl-m-wchYvFRAR_97kJDb4NGaB-Z8,3561
 copyparty/mdns.py,sha256=vC078llnL1v0pvL3mnwacuStFHPJUQuxo9Opj-IbHL4,18155
 copyparty/metrics.py,sha256=aV09nntEmKMIyde8xoPtj1ehDOQVQOHchRF4uMMNzqM,8855
@@ -31,8 +31,8 @@ copyparty/tftpd.py,sha256=jZbf2JpeJmkuQWJErmAPG-dKhtYNvIUHbkAgodSXw9Y,13582
 copyparty/th_cli.py,sha256=o6FMkerYvAXS455z3DUossVztu_nzFlYSQhs6qN6Jt8,4636
 copyparty/th_srv.py,sha256=hI9wY1E_9N9Cgqvtr8zADeVqqiLGTiTdAnYAA7WFvJw,29346
 copyparty/u2idx.py,sha256=JjgqwgJBNj6sTn4PJfuqM3VEHqlmoyGC5bk4_92K2h0,13414
-copyparty/up2k.py,sha256=PSVSdG00wC76mi0E7HF1WBFco4lujpSCTTtSzUusPVM,165096
-copyparty/util.py,sha256=Vj8F50G0taqNd7c8TQd56IRbRZGFE1r0wnnrymbk7j0,92349
+copyparty/up2k.py,sha256=Zn33f8EkFpAnK_RTDyVNA97CKlS9MS3k5mnnvYh1w1k,165351
+copyparty/util.py,sha256=7_-17F94TsLw644xLT8FfO_fH0DwViD54-grej3f8sY,92379
 copyparty/bos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 copyparty/bos/bos.py,sha256=Wb7eWsXJgR5AFlBR9ZOyKrLTwy-Kct9RrGiOu4Jo37Y,1622
 copyparty/bos/path.py,sha256=yEjCq2ki9CvxA5sCT8pS0keEXwugs0ZeUyUhdBziOCI,777
@@ -54,10 +54,10 @@ copyparty/stolen/ifaddr/__init__.py,sha256=vpREjAyPubr5s1NJi91icXV3q1o4DrKAvHABw
 copyparty/stolen/ifaddr/_posix.py,sha256=-67NdfGrCktfQPakT2fLbjl2U00QMvyBGkSvrUuTOrU,2626
 copyparty/stolen/ifaddr/_shared.py,sha256=uNC4SdEIgdSLKvuUzsf1aM-H1Xrc_9mpLoOT43YukGs,6206
 copyparty/stolen/ifaddr/_win32.py,sha256=EE-QyoBgeB7lYQ6z62VjXNaRozaYfCkaJBHGNA8QtZM,4026
-copyparty/web/baguettebox.js.gz,sha256=4dS8-r4si84ca71l98672ahnRI86Aq95MU-bc5knykk,7962
+copyparty/web/baguettebox.js.gz,sha256=YIaxFDsubJfGIdzzxA-cL6GwJVmpWZyaPhW9hHcOIIw,7964
 copyparty/web/browser.css.gz,sha256=4bAS9Xkl2fflhaxRSRSVoYQcpXsg1mCWxsYjId7phbU,11610
 copyparty/web/browser.html,sha256=ISpfvWEawufJCYZIqvuXiyUgiXgjmOTtScz4zrEaypI,4870
-copyparty/web/browser.js.gz,sha256=0WhsCM1Kr0PFryxz7hF31apLgvsJac4jWy8_ePfp_40,84989
+copyparty/web/browser.js.gz,sha256=7rubbEoqFlNn7FPF0mz3L2LQeWuPzw95MYpIaZmcczE,84985
 copyparty/web/browser2.html,sha256=NRUZ08GH-e2YcGXcoz0UjYg6JIVF42u4IMX4HHwWTmg,1587
 copyparty/web/cf.html,sha256=lJThtNFNAQT1ClCHHlivAkDGE0LutedwopXD62Z8Nys,589
 copyparty/web/dbg-audio.js.gz,sha256=Ma-KZtK8LnmiwNvNKFKXMPYl_Nn_3U7GsJ6-DRWC2HE,688
@@ -79,13 +79,13 @@ copyparty/web/splash.html,sha256=pUbsso_W3Q7bso8fy8qxh-fHDrrLm39mBBTIlTeH63w,523
 copyparty/web/splash.js.gz,sha256=Xoccku-2vE3tABo-88q3Cl4koHs_AE76T8QvMy4u6T8,2540
 copyparty/web/svcs.html,sha256=P5YZimYLeQMT0uz6u3clQSNZRc5Zs0Ok-ffcbcGSYuc,11762
 copyparty/web/svcs.js.gz,sha256=k81ZvZ3I-f4fMHKrNGGOgOlvXnCBz0mVjD-8mieoWCA,520
-copyparty/web/ui.css.gz,sha256=dTAXEPgWB83CkYXlP4hWq18BrLFzRlYwxoCoM5rp-Yw,2786
-copyparty/web/up2k.js.gz,sha256=t6mUaIYN8lrEJoWO_-2MZ9nIuEplimQgCqdWSsHnuGA,22811
-copyparty/web/util.js.gz,sha256=uh_NAVPiMcOCTy9oxUiDKmKh1GokTKBPe3FYNMMUYlM,14781
-copyparty/web/w.hash.js.gz,sha256=7wP9EZQNXQxwZnCCFUVsi_-6TM9PLZJeZ9krutXRRj8,1060
+copyparty/web/ui.css.gz,sha256=wloSacrHgP722hy4XiOvVY2GI9-V4zvfvzu84LLWS_o,2779
+copyparty/web/up2k.js.gz,sha256=iMaZ2joij8ndkA2iFCTri6MnYRxXzQbRu9uwAnxZBj8,23096
+copyparty/web/util.js.gz,sha256=NvjPYhIa0-C_NhUyW-Ra-XinUCRjj8G3pYq1zJHYWEk,14805
+copyparty/web/w.hash.js.gz,sha256=5weFAxkcfHhu90tUKnVnTu04U_PQjhzUVRfvir199I0,1093
 copyparty/web/a/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 copyparty/web/a/partyfuse.py,sha256=fa9bBYNJHvtWpNVjQyyFzx6JOK7MJL1u0zj80PBYQKs,27960
-copyparty/web/a/u2c.py,sha256=TkLmQL_3lwus0HI-m4cuP8E9qanfB8U9xGvW7zEJ8ho,47576
+copyparty/web/a/u2c.py,sha256=QA0t_k422uLRh8Kt2eQIpcGYaaUZA1CGz_S-z23Yk-4,49343
 copyparty/web/a/webdav-cfg.bat,sha256=Y4NoGZlksAIg4cBMb7KdJrpKC6Nx97onaTl6yMjaimk,1449
 copyparty/web/dd/2.png,sha256=gJ14XFPzaw95L6z92fSq9eMPikSQyu-03P1lgiGe0_I,258
 copyparty/web/dd/3.png,sha256=4lho8Koz5tV7jJ4ODo6GMTScZfkqsT05yp48EDFIlyg,252
@@ -106,9 +106,9 @@ copyparty/web/deps/prismd.css.gz,sha256=ObUlksQVr-OuYlTz-I4B23TeBg2QDVVGRnWBz8cV
 copyparty/web/deps/scp.woff2,sha256=w99BDU5i8MukkMEL-iW0YO9H4vFFZSPWxbkH70ytaAg,8612
 copyparty/web/deps/sha512.ac.js.gz,sha256=lFZaCLumgWxrvEuDr4bqdKHsqjX82AbVAb7_F45Yk88,7033
 copyparty/web/deps/sha512.hw.js.gz,sha256=vqoXeracj-99Z5MfY3jK2N4WiSzYQdfjy0RnUlQDhSU,8110
-copyparty-1.15.7.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
-copyparty-1.15.7.dist-info/METADATA,sha256=DXlK1tdi4bTueoZXTbr33g_wqCpjko4sTPWpTeXXfGs,138774
-copyparty-1.15.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-copyparty-1.15.7.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
-copyparty-1.15.7.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
-copyparty-1.15.7.dist-info/RECORD,,
+copyparty-1.15.8.dist-info/LICENSE,sha256=gOr4h33pCsBEg9uIy9AYmb7qlocL4V9t2uPJS5wllr0,1072
+copyparty-1.15.8.dist-info/METADATA,sha256=R5tW0crtzFZ4vKzGny4gQ6-IOERgBgj7LZtP14Huung,138915
+copyparty-1.15.8.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+copyparty-1.15.8.dist-info/entry_points.txt,sha256=4zw6a3rqASywQomiYLObjjlxybaI65LYYOTJwgKz7b0,128
+copyparty-1.15.8.dist-info/top_level.txt,sha256=LnYUPsDyk-8kFgM6YJLG4h820DQekn81cObKSu9g-sI,10
+copyparty-1.15.8.dist-info/RECORD,,

{copyparty-1.15.7.dist-info → copyparty-1.15.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any