copyparty 1.15.5__py3-none-any.whl → 1.15.7__py3-none-any.whl

copyparty/up2k.py

@@ -353,17 +353,18 @@ class Up2k(object):
             return '[{"timeout":1}]'
 
         ret      = []
+        userset = set([(uname or "\n"), "*"])
         try:
             for ptop, tab2 in self.registry.items():
                 cfg = self.flags.get(ptop, {}).get("u2abort", 1)
                 if not cfg:
                     continue
                 addr = (ip or "\n") if cfg in (1, 2) else ""
-                user = (uname or "\n") if cfg in (1, 3) else ""
+                user = userset if cfg in (1, 3) else None
                 for job in tab2.values():
                     if (
                         "done" in job
-                        or (user and user != job["user"])
+                        or (user and job["user"] not in user)
                         or (addr and addr != job["addr"])
                     ):
                         continue
@@ -1008,6 +1009,7 @@ class Up2k(object):
                 vpath = k
 
         _, flags = self._expr_idx_filter(flags)
+        n4g = bool(flags.get("noforget"))
 
         ft = "\033[0;32m{}{:.0}"
         ff = "\033[0;35m{}{:.0}"
@@ -1061,20 +1063,39 @@ class Up2k(object):
             except:
                 pass
 
+            if reg2 and "dwrk" not in reg2[next(iter(reg2))]:
+                for job in reg2.values():
+                    job["dwrk"] = job["wark"]
+
+            rm = []
             for k, job in reg2.items():
-                fp = djoin(job["ptop"], job["prel"], job["name"])
+                job["ptop"] = ptop
+                if "done" in job:
+                    job["need"] = job["hash"] = emptylist
+                else:
+                    if "need" not in job:
+                        job["need"] = []
+                    if "hash" not in job:
+                        job["hash"] = []
+
+                fp = djoin(ptop, job["prel"], job["name"])
                 if bos.path.exists(fp):
                     reg[k] = job
                     if "done" in job:
-                        job["need"] = job["hash"] = emptylist
                         continue
                     job["poke"] = time.time()
                     job["busy"] = {}
                 else:
                     self.log("ign deleted file in snap: [{}]".format(fp))
+                    if not n4g:
+                        rm.append(k)
+                        continue
+
+            for x in rm:
+                del reg2[x]
 
             if drp is None:
-                drp = [k for k, v in reg.items() if not v.get("need", [])]
+                drp = [k for k, v in reg.items() if not v["need"]]
             else:
                 drp = [x for x in drp if x in reg]
 
@@ -1536,7 +1557,7 @@ class Up2k(object):
                 at = 0
 
             # skip upload hooks by not providing vflags
-            self.db_add(db.c, {}, rd, fn, lmod, sz, "", "", wark, "", "", ip, at)
+            self.db_add(db.c, {}, rd, fn, lmod, sz, "", "", wark, wark, "", "", ip, at)
             db.n += 1
             tfa += 1
             td = time.time() - db.t
@@ -2764,9 +2785,10 @@ class Up2k(object):
 
         cj["name"] = sanitize_fn(cj["name"], "")
         cj["poke"] = now = self.db_act = self.vol_act[ptop] = time.time()
-        wark = self._get_wark(cj)
+        wark = dwark = self._get_wark(cj)
         job = None
         pdir = djoin(ptop, cj["prel"])
+        inc_ap = djoin(pdir, cj["name"])
         try:
             dev = bos.stat(pdir).st_dev
         except:
@@ -2781,6 +2803,7 @@ class Up2k(object):
             reg = self.registry[ptop]
             vfs = self.asrv.vfs.all_vols[cj["vtop"]]
             n4g = bool(vfs.flags.get("noforget"))
+            noclone = bool(vfs.flags.get("noclone"))
             rand = vfs.flags.get("rand") or cj.get("rand")
             lost    = []
 
@@ -2790,6 +2813,12 @@ class Up2k(object):
             vols = [(ptop, jcur)] if jcur else []
             if vfs.flags.get("xlink"):
                 vols += [(k, v) for k, v in self.cur.items() if k != ptop]
+
+            if noclone:
+                wark = up2k_wark_from_metadata(
+                    self.salt, cj["size"], cj["lmod"], cj["prel"], cj["name"]
+                )
+
             if vfs.flags.get("up_ts", "") == "fu" or not cj["lmod"]:
                 # force upload time rather than last-modified
                 cj["lmod"] = int(time.time())
@@ -2802,10 +2831,10 @@ class Up2k(object):
 
                 if self.no_expr_idx:
                     q = r"select * from up where w = ?"
-                    argv = [wark]
+                    argv = [dwark]
                 else:
                     q = r"select * from up where substr(w,1,16)=? and +w=?"
-                    argv = [wark[:16], wark]
+                    argv = [dwark[:16], dwark]
 
                 c2 = cur.execute(q, tuple(argv))
                 for _, dtime, dsize, dp_dir, dp_fn, ip, at in c2:
@@ -2813,6 +2842,9 @@ class Up2k(object):
                         dp_dir, dp_fn = s3dec(dp_dir, dp_fn)
 
                     dp_abs = djoin(ptop, dp_dir, dp_fn)
+                    if noclone and dp_abs != inc_ap:
+                        continue
+
                     try:
                         st = bos.stat(dp_abs)
                         if stat.S_ISLNK(st.st_mode):
@@ -2821,7 +2853,7 @@ class Up2k(object):
                         if st.st_size != dsize:
                             t = "candidate ignored (db/fs desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}"
                             t = t.format(
-                                wark, st.st_size, dsize, st.st_mtime, dtime, dp_abs
+                                dwark, st.st_size, dsize, st.st_mtime, dtime, dp_abs
                             )
                             self.log(t)
                             raise Exception()
@@ -2868,7 +2900,6 @@ class Up2k(object):
                     alts.append((score, -len(alts), j, cur, dp_dir, dp_fn))
 
             job = None
-            inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
             for dupe in sorted(alts, reverse=True):
                 rj = dupe[2]
                 orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
@@ -2878,11 +2909,11 @@ class Up2k(object):
                     break
                 else:
                     self.log("asserting contents of %s" % (orig_ap,))
-                    dhashes, st = self._hashlist_from_file(orig_ap)
-                    dwark = up2k_wark_from_hashlist(self.salt, st.st_size, dhashes)
-                    if wark != dwark:
+                    hashes2, st = self._hashlist_from_file(orig_ap)
+                    wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
+                    if dwark != wark2:
                         t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s"
-                        self.log(t % (dwark, wark, orig_ap))
+                        self.log(t % (wark2, dwark, orig_ap))
                         lost.append(dupe[3:])
                         continue
                     data_ok = True
@@ -2946,11 +2977,11 @@ class Up2k(object):
 
                 elif inc_ap != orig_ap and not data_ok and "done" in reg[wark]:
                     self.log("asserting contents of %s" % (orig_ap,))
-                    dhashes, _ = self._hashlist_from_file(orig_ap)
-                    dwark = up2k_wark_from_hashlist(self.salt, st.st_size, dhashes)
-                    if wark != dwark:
+                    hashes2, _ = self._hashlist_from_file(orig_ap)
+                    wark2 = up2k_wark_from_hashlist(self.salt, st.st_size, hashes2)
+                    if wark != wark2:
                         t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s"
-                        self.log(t % (dwark, wark, orig_ap))
+                        self.log(t % (wark2, wark, orig_ap))
                         del reg[wark]
 
             if job or wark in reg:
@@ -3007,6 +3038,7 @@ class Up2k(object):
 
                         job = deepcopy(job)
                         job["wark"] = wark
+                        job["dwrk"] = dwark
                         job["at"] = cj.get("at") or now
                         zs = "vtop ptop prel name lmod host user addr poke"
                         for k in zs.split():
@@ -3077,7 +3109,7 @@ class Up2k(object):
                                     raise
 
                         if cur and not self.args.nw:
-                            zs = "prel name lmod size ptop vtop wark host user addr at"
+                            zs = "prel name lmod size ptop vtop wark dwrk host user addr at"
                             a = [job[x] for x in zs.split()]
                             self.db_add(cur, vfs.flags, *a)
                             cur.connection.commit()
@@ -3107,6 +3139,7 @@ class Up2k(object):
 
                 job = {
                     "wark": wark,
+                    "dwrk": dwark,
                     "t0": now,
                     "sprs": sprs,
                     "hash": deepcopy(cj["hash"]),
@@ -3149,6 +3182,7 @@ class Up2k(object):
                 "lmod": job["lmod"],
                 "sprs": job.get("sprs", sprs),
                 "hash": job["need"],
+                "dwrk": dwark,
                 "wark": wark,
             }
 
@@ -3175,7 +3209,7 @@ class Up2k(object):
             ):
                 sql = "update up set mt=? where substr(w,1,16)=? and +rd=? and +fn=?"
                 try:
-                    cur.execute(sql, (cj["lmod"], wark[:16], job["prel"], job["name"]))
+                    cur.execute(sql, (cj["lmod"], dwark[:16], job["prel"], job["name"]))
                     cur.connection.commit()
 
                     ap = djoin(job["ptop"], job["prel"], job["name"])
@@ -3416,19 +3450,19 @@ class Up2k(object):
             self.mutex.release()
             return -1, ""
         try:
-            return self._confirm_chunks(ptop, wark, chashes)
+            return self._confirm_chunks(ptop, wark, chashes, chashes)
         finally:
             self.reg_mutex.release()
             self.mutex.release()
 
     def confirm_chunks(
-        self, ptop , wark , chashes 
+        self, ptop , wark , written , locked 
     )   :
         with self.mutex, self.reg_mutex:
-            return self._confirm_chunks(ptop, wark, chashes)
+            return self._confirm_chunks(ptop, wark, written, locked)
 
     def _confirm_chunks(
-        self, ptop , wark , chashes 
+        self, ptop , wark , written , locked 
     )   :
         if True:
             self.db_act = self.vol_act[ptop] = time.time()
@@ -3440,11 +3474,11 @@ class Up2k(object):
             except Exception as ex:
                 return -2, "confirm_chunk, wark(%r)" % (ex,)  # type: ignore
 
-            for chash in chashes:
+            for chash in locked:
                 job["busy"].pop(chash, None)
 
             try:
-                for chash in chashes:
+                for chash in written:
                     job["need"].remove(chash)
             except Exception as ex:
                 return -2, "confirm_chunk, chash(%s) %r" % (chash, ex)  # type: ignore
@@ -3496,7 +3530,7 @@ class Up2k(object):
         except:
             self.log("failed to utime ({}, {})".format(dst, times))
 
-        zs = "prel name lmod size ptop vtop wark host user addr"
+        zs = "prel name lmod size ptop vtop wark dwrk host user addr"
         z2 = [job[x] for x in zs.split()]
         wake_sr = False
         try:
@@ -3569,6 +3603,7 @@ class Up2k(object):
         ptop ,
         vtop ,
         wark ,
+        dwark ,
         host ,
         usr ,
         ip ,
@@ -3591,6 +3626,7 @@ class Up2k(object):
                 ptop,
                 vtop,
                 wark,
+                dwark,
                 host,
                 usr,
                 ip,
@@ -3604,7 +3640,7 @@ class Up2k(object):
             raise
 
         if "e2t" in self.flags[ptop]:
-            self.tagq.put((ptop, wark, rd, fn, sz, ip, at))
+            self.tagq.put((ptop, dwark, rd, fn, sz, ip, at))
             self.n_tagq += 1
 
         return True
@@ -3631,6 +3667,7 @@ class Up2k(object):
         ptop ,
         vtop ,
         wark ,
+        dwark ,
         host ,
         usr ,
         ip ,
@@ -3647,12 +3684,12 @@ class Up2k(object):
             db_ip = "1.1.1.1" if self.args.no_db_ip else ip
 
         sql = "insert into up values (?,?,?,?,?,?,?)"
-        v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
+        v = (dwark, int(ts), sz, rd, fn, db_ip, int(at or 0))
         try:
             db.execute(sql, v)
         except:
             rd, fn = s3enc(self.mem_cur, rd, fn)
-            v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
+            v = (dwark, int(ts), sz, rd, fn, db_ip, int(at or 0))
             db.execute(sql, v)
 
         self.volsize[db] += sz
@@ -3696,10 +3733,10 @@ class Up2k(object):
             for cd in cds:
                 # one for each unique cooldown duration
                 try:
-                    db.execute(q, (cd, wark[:16], rd, fn))
+                    db.execute(q, (cd, dwark[:16], rd, fn))
                 except:
                     rd, fn = s3enc(self.mem_cur, rd, fn)
-                    db.execute(q, (cd, wark[:16], rd, fn))
+                    db.execute(q, (cd, dwark[:16], rd, fn))
 
             if self.xiu_asleep:
                 self.xiu_asleep = False
@@ -3791,10 +3828,12 @@ class Up2k(object):
             with self.mutex, self.reg_mutex:
                 abrt_cfg = self.flags.get(ptop, {}).get("u2abort", 1)
                 addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
-                user = (uname or "\n") if abrt_cfg in (1, 3) else ""
+                user = ((uname or "\n"), "*") if abrt_cfg in (1, 3) else None
                 reg = self.registry.get(ptop, {}) if abrt_cfg else {}
                 for wark, job in reg.items():
-                    if (user and user != job["user"]) or (addr and addr != job["addr"]):
+                    if (addr and addr != job["addr"]) or (
+                        user and job["user"] not in user
+                    ):
                         continue
                     jrem = djoin(job["prel"], job["name"])
                     if ANYWIN:
@@ -4153,6 +4192,7 @@ class Up2k(object):
                     dvn.realpath,
                     dvn.vpath,
                     w,
+                    w,
                     "",
                     "",
                     ip or "",
@@ -4834,6 +4874,7 @@ class Up2k(object):
                 ptop,
                 vtop,
                 wark,
+                wark,
                 "",
                 usr,
                 ip,

copyparty/util.py

@@ -643,11 +643,15 @@ class HLog(logging.Handler):
 
 
 class NetMap(object):
-    def __init__(self, ips , cidrs , keep_lo=False)  :
+    def __init__(
+        self, ips , cidrs , keep_lo=False, strict_cidr=False
+    )  :
         """
         ips: list of plain ipv4/ipv6 IPs, not cidr
         cidrs: list of cidr-notation IPs (ip/prefix)
         """
+        self.mutex = threading.Lock()
+
         if "::" in ips:
             ips = [x for x in ips if x != "::"] + list(
                 [x.split("/")[0] for x in cidrs if ":" in x]
@@ -674,7 +678,7 @@ class NetMap(object):
             bip = socket.inet_pton(fam, ip.split("/")[0])
             self.bip.append(bip)
             self.b2sip[bip] = ip.split("/")[0]
-            self.b2net[bip] = (IPv6Network if v6 else IPv4Network)(ip, False)
+            self.b2net[bip] = (IPv6Network if v6 else IPv4Network)(ip, strict_cidr)
 
         self.bip.sort(reverse=True)
 
@@ -685,8 +689,10 @@ class NetMap(object):
         try:
             return self.cache[ip]
         except:
-            pass
+            with self.mutex:
+                return self._map(ip)
 
+    def _map(self, ip )  :
         v6 = ":" in ip
         ci = IPv6Address(ip) if v6 else IPv4Address(ip)
         bip = next((x for x in self.bip if ci in self.b2net[x]), None)
@@ -1082,7 +1088,7 @@ class Magician(object):
             return ret
 
         mime = magic.from_file(fpath, mime=True)
-        mime = re.split("[; ]", mime, 1)[0]
+        mime = re.split("[; ]", mime, maxsplit=1)[0]
         try:
             return EXTS[mime]
         except:
@@ -2592,6 +2598,31 @@ def build_netmap(csv ):
     return NetMap(ips, cidrs, True)
 
 
+def load_ipu(log , ipus )    :
+    ip_u = {"": "*"}
+    cidr_u = {}
+    for ipu in ipus:
+        try:
+            cidr, uname = ipu.split("=")
+            cip, csz = cidr.split("/")
+        except:
+            t = "\n  invalid value %r for argument --ipu; must be CIDR=UNAME (192.168.0.0/16=amelia)"
+            raise Exception(t % (ipu,))
+        uname2 = cidr_u.get(cidr)
+        if uname2 is not None:
+            t = "\n  invalid value %r for argument --ipu; cidr %s already mapped to %r"
+            raise Exception(t % (ipu, cidr, uname2))
+        cidr_u[cidr] = uname
+        ip_u[cip] = uname
+    try:
+        nm = NetMap(["::"], list(cidr_u.keys()), True, True)
+    except Exception as ex:
+        t = "failed to translate --ipu into netmap, probably due to invalid config: %r"
+        log("root", t % (ex,), 1)
+        raise
+    return ip_u, nm
+
+
 def yieldfile(fn , bufsz )    :
     readsz = min(bufsz, 128 * 1024)
     with open(fsenc(fn), "rb", bufsz) as f:

copyparty/web/a/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "2.1"
-S_BUILD_DT = "2024-09-23"
+S_VERSION = "2.2"
+S_BUILD_DT = "2024-10-13"
 
 """
 u2c.py: upload to copyparty
@@ -728,6 +728,7 @@ def handshake(ar, file, search):
     while True:
         sc = 600
         txt = ""
+        t0 = time.time()
         try:
             zs = json.dumps(req, separators=(",\n", ": "))
             sc, txt = web.req("POST", url, {}, zs.encode("utf-8"), MJ)
@@ -752,7 +753,9 @@ def handshake(ar, file, search):
                 print("\nERROR: login required, or wrong password:\n%s" % (txt,))
                 raise BadAuth()
 
-            eprint("handshake failed, retrying: %s\n  %s\n\n" % (file.name, em))
+            t = "handshake failed, retrying: %s\n  t0=%.3f t1=%.3f td=%.3f\n  %s\n\n"
+            now = time.time()
+            eprint(t % (file.name, t0, now, now - t0, em))
             time.sleep(ar.cd)
 
     try:
@@ -869,8 +872,8 @@ class Ctl(object):
             self.hash_b = 0
             self.up_f = 0
             self.up_c = 0
-            self.up_b = 0
-            self.up_br = 0
+            self.up_b = 0  # num bytes handled
+            self.up_br = 0  # num bytes actually transferred
             self.uploader_busy = 0
             self.serialized = False
 
@@ -1013,11 +1016,14 @@ class Ctl(object):
             t = "%s eta @ %s/s, %s, %d# left\033[K" % (self.eta, spd, sleft, nleft)
             eprint(txt + "\033]0;{0}\033\\\r{0}{1}".format(t, tail))
 
+        if self.ar.wlist:
+            self.at_hash = time.time() - self.t0
+
         if self.hash_b and self.at_hash:
             spd = humansize(self.hash_b / self.at_hash)
             eprint("\nhasher: %.2f sec, %s/s\n" % (self.at_hash, spd))
-        if self.up_b and self.at_up:
-            spd = humansize(self.up_b / self.at_up)
+        if self.up_br and self.at_up:
+            spd = humansize(self.up_br / self.at_up)
             eprint("upload: %.2f sec, %s/s\n" % (self.at_up, spd))
 
         if not self.recheck:
@@ -1136,10 +1142,16 @@ class Ctl(object):
                     self.up_b = self.hash_b
 
             if self.ar.wlist:
+                vp = file.rel.decode("utf-8")
+                if self.ar.chs:
+                    zsl = [
+                        "%s %d %d" % (zsii[0], n, zsii[1])
+                        for n, zsii in enumerate(file.cids)
+                    ]
+                    print("chs: %s\n%s" % (vp, "\n".join(zsl)))
                 zsl = [self.ar.wsalt, str(file.size)] + [x[0] for x in file.kchunks]
                 zb = hashlib.sha512("\n".join(zsl).encode("utf-8")).digest()[:33]
                 wark = ub64enc(zb).decode("utf-8")
-                vp = file.rel.decode("utf-8")
                 if self.ar.jw:
                     print("%s  %s" % (wark, vp))
                 else:
@@ -1177,6 +1189,7 @@ class Ctl(object):
                 self.q_upload.put(None)
                 return
 
+            chunksz = up2k_chunksize(file.size)
             upath = file.abs.decode("utf-8", "replace")
             if not VT100:
                 upath = upath.lstrip("\\?")
@@ -1236,9 +1249,14 @@ class Ctl(object):
                     file.up_c -= len(hs)
                     for cid in hs:
                         sz = file.kchunks[cid][1]
+                        self.up_br -= sz
                         self.up_b -= sz
                         file.up_b -= sz
 
+                if hs and not file.up_b:
+                    # first hs of this file; is this an upload resume?
+                    file.up_b = chunksz * max(0, len(file.kchunks) - len(hs))
+
                 file.ucids = hs
 
             if not hs:
@@ -1252,7 +1270,7 @@ class Ctl(object):
                         c1 = c2 = ""
 
                     spd_h = humansize(file.size / file.t_hash, True)
-                    if file.up_b:
+                    if file.up_c:
                         t_up = file.t1_up - file.t0_up
                         spd_u = humansize(file.size / t_up, True)
 
@@ -1262,13 +1280,12 @@ class Ctl(object):
                         t = "   found %s %s(%.2fs,%s/s)%s"
                         print(t % (upath, c1, file.t_hash, spd_h, c2))
                 else:
-                    kw = "uploaded" if file.up_b else "   found"
+                    kw = "uploaded" if file.up_c else "   found"
                     print("{0} {1}".format(kw, upath))
 
                 self._check_if_done()
                 continue
 
-            chunksz = up2k_chunksize(file.size)
             njoin = (self.ar.sz * 1024 * 1024) // chunksz
             cs = hs[:]
             while cs:
@@ -1365,7 +1382,7 @@ def main():
     cores = (os.cpu_count() if hasattr(os, "cpu_count") else 0) or 2
     hcores = min(cores, 3)  # 4% faster than 4+ on py3.9 @ r5-4500U
 
-    ver = "{0}  v{1}  https://youtu.be/BIcOO6TLKaY".format(S_BUILD_DT, S_VERSION)
+    ver = "{0}, v{1}".format(S_BUILD_DT, S_VERSION)
     if "--version" in sys.argv:
         print(ver)
         return
@@ -1403,6 +1420,7 @@ source file/folder selection uses rsync syntax, meaning that:
 
     ap = app.add_argument_group("file-ID calculator; enable with url '-' to list warks (file identifiers) instead of upload/search")
     ap.add_argument("--wsalt", type=unicode, metavar="S", default="hunter2", help="salt to use when creating warks; must match server config")
+    ap.add_argument("--chs", action="store_true", help="verbose (print the hash/offset of each chunk in each file)")
     ap.add_argument("--jw", action="store_true", help="just identifier+filepath, not mtime/size too")
 
     ap = app.add_argument_group("performance tweaks")