copyparty 1.12.1__py3-none-any.whl → 1.13.0__py3-none-any.whl

copyparty/up2k.py

@@ -88,6 +88,9 @@ CV_EXTS = set(zsg.split(","))
 HINT_HISTPATH = "you could try moving the database to another location (preferably an SSD or NVME drive) using either the --hist argument (global option for all volumes), or the hist volflag (just for this volume)"
 
 
+VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1}
+
+
 class Dbw(object):
     def __init__(self, c , n , t )  :
         self.c = c
@@ -133,6 +136,7 @@ class Up2k(object):
         self.need_rescan  = set()
         self.db_act = 0.0
 
+        self.reg_mutex = threading.Lock()
         self.registry     = {}
         self.flags    = {}
         self.droppable   = {}
@@ -140,7 +144,7 @@ class Up2k(object):
         self.volsize   = {}
         self.volstate   = {}
         self.vol_act   = {}
-        self.busy_aps  = set()
+        self.busy_aps   = {}
         self.dupesched     = {}
         self.snap_prev    = {}
 
@@ -197,11 +201,15 @@ class Up2k(object):
         Daemon(self.deferred_init, "up2k-deferred-init")
 
     def reload(self, rescan_all_vols )  :
-        """mutex me"""
+        """mutex(main) me"""
         self.log("reload #{} scheduled".format(self.gid + 1))
         all_vols = self.asrv.vfs.all_vols
 
-        scan_vols = [k for k, v in all_vols.items() if v.realpath not in self.registry]
+        with self.reg_mutex:
+            scan_vols = [
+                k for k, v in all_vols.items() if v.realpath not in self.registry
+            ]
+
         if rescan_all_vols:
             scan_vols = list(all_vols.keys())
 
@@ -214,7 +222,7 @@ class Up2k(object):
         if self.stop:
             # up-mt consistency not guaranteed if init is interrupted;
             # drop caches for a full scan on next boot
-            with self.mutex:
+            with self.mutex, self.reg_mutex:
                 self._drop_caches()
 
             if self.pp:
@@ -280,10 +288,27 @@ class Up2k(object):
                 min(1000 * 24 * 60 * 60 - 1, time.time() - self.db_act)
             ),
         }
-        return json.dumps(ret, indent=4)
+        return json.dumps(ret, separators=(",\n", ": "))
+
+    def find_job_by_ap(self, ptop , ap )  :
+        try:
+            if ANYWIN:
+                ap = ap.replace("\\", "/")
+
+            vp = ap[len(ptop) :].strip("/")
+            dn, fn = vsplit(vp)
+            with self.reg_mutex:
+                tab2 = self.registry[ptop]
+                for job in tab2.values():
+                    if job["prel"] == dn and job["name"] == fn:
+                        return json.dumps(job, separators=(",\n", ": "))
+        except:
+            pass
+
+        return "{}"
 
     def get_unfinished_by_user(self, uname, ip)  :
-        if PY2 or not self.mutex.acquire(timeout=2):
+        if PY2 or not self.reg_mutex.acquire(timeout=2):
             return '[{"timeout":1}]'
 
         ret      = []
@@ -312,17 +337,25 @@ class Up2k(object):
                     )
                     ret.append(zt5)
         finally:
-            self.mutex.release()
+            self.reg_mutex.release()
+
+        if ANYWIN:
+            ret = [(x[0], x[1].replace("\\", "/"), x[2], x[3], x[4]) for x in ret]
 
         ret.sort(reverse=True)
         ret2 = [
-            {"at": at, "vp": "/" + vp, "pd": 100 - ((nn * 100) // (nh or 1)), "sz": sz}
+            {
+                "at": at,
+                "vp": "/" + quotep(vp),
+                "pd": 100 - ((nn * 100) // (nh or 1)),
+                "sz": sz,
+            }
             for (at, vp, sz, nn, nh) in ret
         ]
-        return json.dumps(ret2, indent=0)
+        return json.dumps(ret2, separators=(",\n", ": "))
 
     def get_unfinished(self)  :
-        if PY2 or not self.mutex.acquire(timeout=0.5):
+        if PY2 or not self.reg_mutex.acquire(timeout=0.5):
             return ""
 
         ret    = {}
@@ -344,17 +377,17 @@ class Up2k(object):
 
                 ret[ptop] = (nbytes, nfiles)
         finally:
-            self.mutex.release()
+            self.reg_mutex.release()
 
-        return json.dumps(ret, indent=4)
+        return json.dumps(ret, separators=(",\n", ": "))
 
     def get_volsize(self, ptop )   :
-        with self.mutex:
+        with self.reg_mutex:
             return self._get_volsize(ptop)
 
     def get_volsizes(self, ptops )   :
         ret = []
-        with self.mutex:
+        with self.reg_mutex:
             for ptop in ptops:
                 ret.append(self._get_volsize(ptop))
 
@@ -382,7 +415,7 @@ class Up2k(object):
     def _rescan(
         self, all_vols  , scan_vols , wait , fscan 
     )  :
-        """mutex me"""
+        """mutex(main) me"""
         if not wait and self.pp:
             return "cannot initiate; scan is already in progress"
 
@@ -664,7 +697,7 @@ class Up2k(object):
                 self.log(msg, c=3)
 
         live_vols = []
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             # only need to protect register_vpath but all in one go feels right
             for vol in vols:
                 try:
@@ -706,7 +739,7 @@ class Up2k(object):
 
         if self.args.re_dhash or [zv for zv in vols if "e2tsr" in zv.flags]:
             self.args.re_dhash = False
-            with self.mutex:
+            with self.mutex, self.reg_mutex:
                 self._drop_caches()
 
         for vol in vols:
@@ -783,7 +816,9 @@ class Up2k(object):
             self.volstate[vol.vpath] = "online (mtp soon)"
 
         for vol in need_vac:
-            reg = self.register_vpath(vol.realpath, vol.flags)
+            with self.mutex, self.reg_mutex:
+                reg = self.register_vpath(vol.realpath, vol.flags)
+
             assert reg
             cur, _ = reg
             with self.mutex:
@@ -797,7 +832,9 @@ class Up2k(object):
             if vol.flags["dbd"] == "acid":
                 continue
 
-            reg = self.register_vpath(vol.realpath, vol.flags)
+            with self.mutex, self.reg_mutex:
+                reg = self.register_vpath(vol.realpath, vol.flags)
+
             try:
                 assert reg
                 cur, db_path = reg
@@ -844,6 +881,7 @@ class Up2k(object):
     def register_vpath(
         self, ptop , flags  
     )   :
+        """mutex(main,reg) me"""
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))
@@ -866,7 +904,7 @@ class Up2k(object):
         ft = "\033[0;32m{}{:.0}"
         ff = "\033[0;35m{}{:.0}"
         fv = "\033[0;36m{}:\033[90m{}"
-        fx = set(("html_head", "rm_re_t", "rm_re_r"))
+        fx = set(("html_head", "rm_re_t", "rm_re_r", "mv_re_t", "mv_re_r"))
         fd = vf_bmap()
         fd.update(vf_cmap())
         fd.update(vf_vmap())
@@ -1027,7 +1065,9 @@ class Up2k(object):
         dev = cst.st_dev if vol.flags.get("xdev") else 0
 
         with self.mutex:
-            reg = self.register_vpath(top, vol.flags)
+            with self.reg_mutex:
+                reg = self.register_vpath(top, vol.flags)
+
             assert reg and self.pp
             cur, db_path = reg
 
@@ -1624,7 +1664,7 @@ class Up2k(object):
 
     def _build_tags_index(self, vol )    :
         ptop = vol.realpath
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             reg = self.register_vpath(ptop, vol.flags)
 
         assert reg and self.pp
@@ -1645,6 +1685,7 @@ class Up2k(object):
         return ret
 
     def _drop_caches(self)  :
+        """mutex(main,reg) me"""
         self.log("dropping caches for a full filesystem scan")
         for vol in self.asrv.vfs.all_vols.values():
             reg = self.register_vpath(vol.realpath, vol.flags)
@@ -1820,7 +1861,7 @@ class Up2k(object):
         params  ,
         flt ,
     )   :
-        """mutex me"""
+        """mutex(main) me"""
         n = 0
         c2 = cur.connection.cursor()
         tf = tempfile.SpooledTemporaryFile(1024 * 1024 * 8, "w+b", prefix="cpp-tq-")
@@ -2154,7 +2195,7 @@ class Up2k(object):
         ip ,
         at ,
     )  :
-        """will mutex"""
+        """will mutex(main)"""
         assert self.mtag
 
         try:
@@ -2186,7 +2227,7 @@ class Up2k(object):
         abspath ,
         tags   ,
     )  :
-        """mutex me"""
+        """mutex(main) me"""
         assert self.mtag
 
         if not bos.path.isfile(abspath):
@@ -2471,28 +2512,36 @@ class Up2k(object):
 
         cur.connection.commit()
 
-    def _job_volchk(self, cj  )  :
-        if not self.register_vpath(cj["ptop"], cj["vcfg"]):
-            if cj["ptop"] not in self.registry:
-                raise Pebkac(410, "location unavailable")
-
-    def handle_json(self, cj  , busy_aps )   :
+    def handle_json(
+        self, cj  , busy_aps  
+    )   :
+        # busy_aps is u2fh (always undefined if -j0) so this is safe
         self.busy_aps = busy_aps
+        got_lock = False
         try:
             # bit expensive; 3.9=10x 3.11=2x
             if self.mutex.acquire(timeout=10):
-                self._job_volchk(cj)
-                self.mutex.release()
+                got_lock = True
+                with self.reg_mutex:
+                    return self._handle_json(cj)
             else:
                 t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
                 raise Pebkac(503, t.format(self.blocked or "[unknown]"))
         except TypeError:
             if not PY2:
                 raise
-            with self.mutex:
-                self._job_volchk(cj)
+            with self.mutex, self.reg_mutex:
+                return self._handle_json(cj)
+        finally:
+            if got_lock:
+                self.mutex.release()
 
+    def _handle_json(self, cj  )   :
         ptop = cj["ptop"]
+        if not self.register_vpath(ptop, cj["vcfg"]):
+            if ptop not in self.registry:
+                raise Pebkac(410, "location unavailable")
+
         cj["name"] = sanitize_fn(cj["name"], "", [".prologue.html", ".epilogue.html"])
         cj["poke"] = now = self.db_act = self.vol_act[ptop] = time.time()
         wark = self._get_wark(cj)
@@ -2507,7 +2556,7 @@ class Up2k(object):
         # refuse out-of-order / multithreaded uploading if sprs False
         sprs = self.fstab.get(pdir) != "ng"
 
-        with self.mutex:
+        if True:
             jcur = self.cur.get(ptop)
             reg = self.registry[ptop]
             vfs = self.asrv.vfs.all_vols[cj["vtop"]]
@@ -2945,7 +2994,7 @@ class Up2k(object):
     def handle_chunk(
         self, ptop , wark , chash 
     )      :
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
             if not job:
@@ -2988,7 +3037,7 @@ class Up2k(object):
         return chunksize, ofs, path, job["lmod"], job["sprs"]
 
     def release_chunk(self, ptop , wark , chash )  :
-        with self.mutex:
+        with self.reg_mutex:
             job = self.registry[ptop].get(wark)
             if job:
                 job["busy"].pop(chash, None)
@@ -2996,7 +3045,7 @@ class Up2k(object):
         return True
 
     def confirm_chunk(self, ptop , wark , chash )   :
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             try:
                 job = self.registry[ptop][wark]
@@ -3019,16 +3068,16 @@ class Up2k(object):
 
             if self.args.nw:
                 self.regdrop(ptop, wark)
-                return ret, dst
 
         return ret, dst
 
     def finish_upload(self, ptop , wark , busy_aps )  :
         self.busy_aps = busy_aps
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             self._finish_upload(ptop, wark)
 
     def _finish_upload(self, ptop , wark )  :
+        """mutex(main,reg) me"""
         try:
             job = self.registry[ptop][wark]
             pdir = djoin(job["ptop"], job["prel"])
@@ -3041,12 +3090,11 @@ class Up2k(object):
             t = "finish_upload {} with remaining chunks {}"
             raise Pebkac(500, t.format(wark, job["need"]))
 
-        # self.log("--- " + wark + "  " + dst + " finish_upload atomic " + dst, 4)
-        atomic_move(src, dst)
-
         upt = job.get("at") or time.time()
         vflags = self.flags[ptop]
 
+        atomic_move(self.log, src, dst, vflags)
+
         times = (int(time.time()), int(job["lmod"]))
         self.log(
             "no more chunks, setting times {} ({}) on {}".format(
@@ -3102,6 +3150,7 @@ class Up2k(object):
             cur.connection.commit()
 
     def regdrop(self, ptop , wark )  :
+        """mutex(main,reg) me"""
         olds = self.droppable[ptop]
         if wark:
             olds.append(wark)
@@ -3196,16 +3245,23 @@ class Up2k(object):
         at ,
         skip_xau  = False,
     )  :
+        """mutex(main) me"""
         self.db_rm(db, rd, fn, sz)
 
+        if not ip:
+            db_ip = ""
+        else:
+            # plugins may expect this to look like an actual IP
+            db_ip = "1.1.1.1" if self.args.no_db_ip else ip
+
         sql = "insert into up values (?,?,?,?,?,?,?)"
-        v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
+        v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
         try:
             db.execute(sql, v)
         except:
             assert self.mem_cur
             rd, fn = s3enc(self.mem_cur, rd, fn)
-            v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
+            v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
             db.execute(sql, v)
 
         self.volsize[db] += sz
@@ -3309,7 +3365,7 @@ class Up2k(object):
             vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
             vn, rem = vn.get_dbv(rem)
             ptop = vn.realpath
-            with self.mutex:
+            with self.mutex, self.reg_mutex:
                 abrt_cfg = self.flags.get(ptop, {}).get("u2abort", 1)
                 addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
                 user = (uname or "\n") if abrt_cfg in (1, 3) else ""
@@ -3317,7 +3373,10 @@ class Up2k(object):
                 for wark, job in reg.items():
                     if (user and user != job["user"]) or (addr and addr != job["addr"]):
                         continue
-                    if djoin(job["prel"], job["name"]) == rem:
+                    jrem = djoin(job["prel"], job["name"])
+                    if ANYWIN:
+                        jrem = jrem.replace("\\", "/")
+                    if jrem == rem:
                         if job["ptop"] != ptop:
                             t = "job.ptop [%s] != vol.ptop [%s] ??"
                             raise Exception(t % (job["ptop"] != ptop))
@@ -3413,7 +3472,7 @@ class Up2k(object):
                         continue
 
                 n_files += 1
-                with self.mutex:
+                with self.mutex, self.reg_mutex:
                     cur = None
                     try:
                         ptop = dbv.realpath
@@ -3531,6 +3590,7 @@ class Up2k(object):
     def _mv_file(
         self, uname , svp , dvp , curs 
     )  :
+        """mutex(main) me;  will mutex(reg)"""
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
         svn, srem = svn.get_dbv(srem)
 
@@ -3611,7 +3671,9 @@ class Up2k(object):
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
-            has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
+            with self.reg_mutex:
+                has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
+
             if not is_xvol:
                 has_dupes = self._relink(w, svn.realpath, srem, dabs)
 
@@ -3650,7 +3712,7 @@ class Up2k(object):
                 self._symlink(dlink, dabs, dvn.flags, lmod=ftime)
                 wunlink(self.log, sabs, svn.flags)
             else:
-                atomic_move(sabs, dabs)
+                atomic_move(self.log, sabs, dabs, svn.flags)
 
         except OSError as ex:
             if ex.errno != errno.EXDEV:
@@ -3741,7 +3803,10 @@ class Up2k(object):
         drop_tags ,
         sz ,
     )  :
-        """forgets file in db, fixes symlinks, does not delete"""
+        """
+        mutex(main,reg) me
+        forgets file in db, fixes symlinks, does not delete
+        """
         srd, sfn = vsplit(vrem)
         has_dupes = False
         self.log("forgetting {}".format(vrem))
@@ -3827,8 +3892,7 @@ class Up2k(object):
             self.log("linkswap [{}] and [{}]".format(sabs, slabs))
             mt = bos.path.getmtime(slabs, False)
             flags = self.flags.get(ptop) or {}
-            wunlink(self.log, slabs, flags)
-            bos.rename(sabs, slabs)
+            atomic_move(self.log, sabs, slabs, flags)
             bos.utime(slabs, (int(time.time()), int(mt)), False)
             self._symlink(slabs, sabs, flags, False)
             full[slabs] = (ptop, rem)
@@ -4067,7 +4131,7 @@ class Up2k(object):
                 self.do_snapshot()
 
     def do_snapshot(self)  :
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             for k, reg in self.registry.items():
                 self._snap_reg(k, reg)
 
@@ -4135,11 +4199,11 @@ class Up2k(object):
 
         path2 = "{}.{}".format(path, os.getpid())
         body = {"droppable": self.droppable[ptop], "registry": reg}
-        j = json.dumps(body, indent=2, sort_keys=True).encode("utf-8")
+        j = json.dumps(body, sort_keys=True, separators=(",\n", ": ")).encode("utf-8")
         with gzip.GzipFile(path2, "wb") as f:
             f.write(j)
 
-        atomic_move(path2, path)
+        atomic_move(self.log, path2, path, VF_CAREFUL)
 
         self.log("snap: {} |{}|".format(path, len(reg.keys())))
         self.snap_prev[ptop] = etag
@@ -4208,7 +4272,7 @@ class Up2k(object):
                 raise Exception("invalid hash task")
 
             try:
-                if not self._hash_t(task):
+                if not self._hash_t(task) and self.stop:
                     return
             except Exception as ex:
                 self.log("failed to hash %s: %s" % (task, ex), 1)
@@ -4218,7 +4282,7 @@ class Up2k(object):
     )  :
         ptop, vtop, flags, rd, fn, ip, at, usr, skip_xau = task
         # self.log("hashq {} pop {}/{}/{}".format(self.n_hashq, ptop, rd, fn))
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             if not self.register_vpath(ptop, flags):
                 return True
 
@@ -4236,7 +4300,7 @@ class Up2k(object):
 
             wark = up2k_wark_from_hashlist(self.salt, inf.st_size, hashes)
 
-        with self.mutex:
+        with self.mutex, self.reg_mutex:
             self.idx_wark(
                 self.flags[ptop],
                 rd,