copyparty 1.13.5__py3-none-any.whl → 1.13.7__py3-none-any.whl

copyparty/up2k.py

@@ -28,8 +28,8 @@ from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
-    VF_CAREFUL,
     SYMTIME,
+    VF_CAREFUL,
     Daemon,
     MTHash,
     Pebkac,
@@ -46,6 +46,7 @@ from .util import (
     hidedir,
     humansize,
     min_ex,
+    pathmod,
     quotep,
     rand_name,
     ren_open,
@@ -162,6 +163,7 @@ class Up2k(object):
         self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)")
         self.xiu_busy = False  # currently running hook
         self.xiu_asleep = True  # needs rescan_cond poke to schedule self
+        self.fx_backlog     = []
 
         self.cur   = {}
         self.mem_cur = None
@@ -542,7 +544,7 @@ class Up2k(object):
                     nrm += 1
 
             if nrm:
-                self.log("{} files graduated in {}".format(nrm, vp))
+                self.log("%d files graduated in /%s" % (nrm, vp))
 
             if timeout < 10:
                 continue
@@ -2541,7 +2543,7 @@ class Up2k(object):
             if self.mutex.acquire(timeout=10):
                 got_lock = True
                 with self.reg_mutex:
-                    return self._handle_json(cj)
+                    ret = self._handle_json(cj)
             else:
                 t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..."
                 raise Pebkac(503, t.format(self.blocked or "[unknown]"))
@@ -2549,11 +2551,16 @@ class Up2k(object):
             if not PY2:
                 raise
             with self.mutex, self.reg_mutex:
-                return self._handle_json(cj)
+                ret = self._handle_json(cj)
         finally:
             if got_lock:
                 self.mutex.release()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
+        return ret
+
     def _handle_json(self, cj  )   :
         ptop = cj["ptop"]
         if not self.register_vpath(ptop, cj["vcfg"]):
@@ -2747,7 +2754,7 @@ class Up2k(object):
                         job = deepcopy(job)
                         job["wark"] = wark
                         job["at"] = cj.get("at") or time.time()
-                        for k in "lmod ptop vtop prel host user addr".split():
+                        for k in "lmod ptop vtop prel name host user addr".split():
                             job[k] = cj.get(k) or ""
 
                         pdir = djoin(cj["ptop"], cj["prel"])
@@ -2755,28 +2762,43 @@ class Up2k(object):
                             job["name"] = rand_name(
                                 pdir, cj["name"], vfs.flags["nrand"]
                             )
-                        else:
-                            job["name"] = self._untaken(pdir, cj, now)
 
                         dst = djoin(job["ptop"], job["prel"], job["name"])
                         xbu = vfs.flags.get("xbu")
-                        if xbu and not runhook(
-                            self.log,
-                            xbu,  # type: ignore
-                            dst,
-                            job["vtop"],
-                            job["host"],
-                            job["user"],
-                            self.asrv.vfs.get_perms(job["vtop"], job["user"]),
-                            job["lmod"],
-                            job["size"],
-                            job["addr"],
-                            job["at"],
-                            "",
-                        ):
-                            t = "upload blocked by xbu server config: {}".format(dst)
-                            self.log(t, 1)
-                            raise Pebkac(403, t)
+                        if xbu:
+                            vp = djoin(job["vtop"], job["prel"], job["name"])
+                            hr = runhook(
+                                self.log,
+                                None,
+                                self,
+                                "xbu.up2k.dupe",
+                                xbu,  # type: ignore
+                                dst,
+                                vp,
+                                job["host"],
+                                job["user"],
+                                self.asrv.vfs.get_perms(job["vtop"], job["user"]),
+                                job["lmod"],
+                                job["size"],
+                                job["addr"],
+                                job["at"],
+                                "",
+                            )
+                            if not hr:
+                                t = "upload blocked by xbu server config: %s" % (dst,)
+                                self.log(t, 1)
+                                raise Pebkac(403, t)
+                            if hr.get("reloc"):
+                                x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"])
+                                if x:
+                                    pdir, _, job["name"], (vfs, rem) = x
+                                    dst = os.path.join(pdir, job["name"])
+                                    job["ptop"] = vfs.realpath
+                                    job["vtop"] = vfs.vpath
+                                    job["prel"] = rem
+                                    bos.makedirs(pdir)
+
+                        job["name"] = self._untaken(pdir, job, now)
 
                         if not self.args.nw:
                             dvf   = vfs.flags
@@ -2848,11 +2870,11 @@ class Up2k(object):
                 # one chunk may occur multiple times in a file;
                 # filter to unique values for the list of missing chunks
                 # (preserve order to reduce disk thrashing)
-                lut = {}
+                lut = set()
                 for k in cj["hash"]:
                     if k not in lut:
                         job["need"].append(k)
-                        lut[k] = 1
+                        lut.add(k)
 
                 try:
                     self._new_upload(job)
@@ -3012,7 +3034,7 @@ class Up2k(object):
 
     def handle_chunks(
         self, ptop , wark , chashes 
-    )      :
+    )       :
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
@@ -3021,12 +3043,37 @@ class Up2k(object):
                 self.log("unknown wark [{}], known: {}".format(wark, known))
                 raise Pebkac(400, "unknown wark" + SSEELOG)
 
+            if len(chashes) > 1 and len(chashes[1]) < 44:
+                # first hash is full-length; expand remaining ones
+                uniq = []
+                lut = set()
+                for chash in job["hash"]:
+                    if chash not in lut:
+                        uniq.append(chash)
+                        lut.add(chash)
+                try:
+                    nchunk = uniq.index(chashes[0])
+                except:
+                    raise Pebkac(400, "unknown chunk0 [%s]" % (chashes[0]))
+                expanded = [chashes[0]]
+                for prefix in chashes[1:]:
+                    nchunk += 1
+                    chash = uniq[nchunk]
+                    if not chash.startswith(prefix):
+                        t = "next sibling chunk does not start with expected prefix [%s]: [%s]"
+                        raise Pebkac(400, t % (prefix, chash))
+                    expanded.append(chash)
+                chashes = expanded
+
             for chash in chashes:
                 if chash not in job["need"]:
                     msg = "chash = {} , need:\n".format(chash)
                     msg += "\n".join(job["need"])
                     self.log(msg)
-                    raise Pebkac(400, "already got that (%s) but thanks??" % (chash,))
+                    t = "already got that (%s) but thanks??"
+                    if chash not in job["hash"]:
+                        t = "unknown chunk wtf: %s"
+                    raise Pebkac(400, t % (chash,))
 
                 if chash in job["busy"]:
                     nh = len(job["hash"])
@@ -3034,9 +3081,11 @@ class Up2k(object):
                     t = "that chunk is already being written to:\n  {}\n  {} {}/{}\n  {}"
                     raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
 
+            assert chash  # type: ignore
             chunksize = up2k_chunksize(job["size"])
 
             coffsets = []
+            nchunks = []
             for chash in chashes:
                 nchunk = [n for n, v in enumerate(job["hash"]) if v == chash]
                 if not nchunk:
@@ -3044,6 +3093,7 @@ class Up2k(object):
 
                 ofs = [chunksize * x for x in nchunk]
                 coffsets.append(ofs)
+                nchunks.append(nchunk)
 
             for ofs1, ofs2 in zip(coffsets, coffsets[1:]):
                 gap = (ofs2[0] - ofs1[0]) - chunksize
@@ -3055,16 +3105,16 @@ class Up2k(object):
 
             if not job["sprs"]:
                 cur_sz = bos.path.getsize(path)
-                if ofs[0] > cur_sz:
+                if coffsets[0][0] > cur_sz:
                     t = "please upload sequentially using one thread;\nserver filesystem does not support sparse files.\n  file: {}\n  chunk: {}\n  cofs: {}\n  flen: {}"
-                    t = t.format(job["name"], nchunk[0], ofs[0], cur_sz)
+                    t = t.format(job["name"], nchunks[0][0], coffsets[0][0], cur_sz)
                     raise Pebkac(400, t)
 
             job["busy"][chash] = 1
 
         job["poke"] = time.time()
 
-        return chunksize, coffsets, path, job["lmod"], job["sprs"]
+        return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
 
     def release_chunks(self, ptop , wark , chashes )  :
         with self.reg_mutex:
@@ -3111,6 +3161,9 @@ class Up2k(object):
         with self.mutex, self.reg_mutex:
             self._finish_upload(ptop, wark)
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
     def _finish_upload(self, ptop , wark )  :
         """mutex(main,reg) me"""
         try:
@@ -3304,25 +3357,30 @@ class Up2k(object):
 
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
-        if xau and not runhook(
-            self.log,
-            xau,
-            dst,
-            djoin(vtop, rd, fn),
-            host,
-            usr,
-            self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
-            int(ts),
-            sz,
-            ip,
-            at or time.time(),
-            "",
-        ):
-            t = "upload blocked by xau server config"
-            self.log(t, 1)
-            wunlink(self.log, dst, vflags)
-            self.registry[ptop].pop(wark, None)
-            raise Pebkac(403, t)
+        if xau:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xau.up2k",
+                xau,
+                dst,
+                djoin(vtop, rd, fn),
+                host,
+                usr,
+                self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr),
+                ts,
+                sz,
+                ip,
+                at or time.time(),
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xau server config"
+                self.log(t, 1)
+                wunlink(self.log, dst, vflags)
+                self.registry[ptop].pop(wark, None)
+                raise Pebkac(403, t)
 
         xiu = vflags.get("xiu")
         if xiu:
@@ -3506,6 +3564,9 @@ class Up2k(object):
                 if xbd:
                     if not runhook(
                         self.log,
+                        None,
+                        self,
+                        "xbd",
                         xbd,
                         abspath,
                         vpath,
@@ -3515,7 +3576,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     ):
                         t = "delete blocked by xbd server config: {}"
@@ -3540,6 +3601,9 @@ class Up2k(object):
                 if xad:
                     runhook(
                         self.log,
+                        None,
+                        self,
+                        "xad",
                         xad,
                         abspath,
                         vpath,
@@ -3549,7 +3613,7 @@ class Up2k(object):
                         stl.st_mtime,
                         st.st_size,
                         ip,
-                        0,
+                        time.time(),
                         "",
                     )
 
@@ -3565,7 +3629,7 @@ class Up2k(object):
 
         return n_files, ok + ok2, ng + ng2
 
-    def handle_mv(self, uname , svp , dvp )  :
+    def handle_mv(self, uname , ip , svp , dvp )  :
         if svp == dvp or dvp.startswith(svp + "/"):
             raise Pebkac(400, "mv: cannot move parent into subfolder")
 
@@ -3582,7 +3646,7 @@ class Up2k(object):
         if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
             with self.mutex:
                 try:
-                    ret = self._mv_file(uname, svp, dvp, curs)
+                    ret = self._mv_file(uname, ip, svp, dvp, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3615,7 +3679,7 @@ class Up2k(object):
                             raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp))
 
                         dvpf = dvp + svpf[len(svp) :]
-                        self._mv_file(uname, svpf, dvpf, curs)
+                        self._mv_file(uname, ip, svpf, dvpf, curs)
                 finally:
                     for v in curs:
                         v.connection.commit()
@@ -3640,7 +3704,7 @@ class Up2k(object):
         return "k"
 
     def _mv_file(
-        self, uname , svp , dvp , curs 
+        self, uname , ip , svp , dvp , curs 
     )  :
         """mutex(main) me;  will mutex(reg)"""
         svn, srem = self.asrv.vfs.get(svp, uname, True, False, True)
@@ -3674,21 +3738,27 @@ class Up2k(object):
             except:
                 pass  # broken symlink; keep as-is
 
+        ftime = stl.st_mtime
+        fsize = st.st_size
+
         xbr = svn.flags.get("xbr")
         xar = dvn.flags.get("xar")
         if xbr:
             if not runhook(
                 self.log,
+                None,
+                self,
+                "xbr",
                 xbr,
                 sabs,
                 svp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(svp, uname),
-                stl.st_mtime,
-                st.st_size,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             ):
                 t = "move blocked by xbr server config: {}".format(svp)
@@ -3716,16 +3786,19 @@ class Up2k(object):
             if xar:
                 runhook(
                     self.log,
+                    None,
+                    self,
+                    "xar.ln",
                     xar,
                     dabs,
                     dvp,
                     "",
                     uname,
                     self.asrv.vfs.get_perms(dvp, uname),
-                    0,
-                    0,
-                    "",
-                    0,
+                    ftime,
+                    fsize,
+                    ip,
+                    time.time(),
                     "",
                 )
 
@@ -3734,13 +3807,6 @@ class Up2k(object):
         c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem)
         c2 = self.cur.get(dvn.realpath)
 
-        if ftime_ is None:
-            ftime = stl.st_mtime
-            fsize = st.st_size
-        else:
-            ftime = ftime_
-            fsize = fsize_ or 0
-
         has_dupes = False
         if w:
             assert c1
@@ -3748,7 +3814,9 @@ class Up2k(object):
                 self._copy_tags(c1, c2, w)
 
             with self.reg_mutex:
-                has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize)
+                has_dupes = self._forget_file(
+                    svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
+                )
 
             if not is_xvol:
                 has_dupes = self._relink(w, svn.realpath, srem, dabs)
@@ -3818,7 +3886,7 @@ class Up2k(object):
 
             if is_link:
                 try:
-                    times = (int(time.time()), int(stl.st_mtime))
+                    times = (int(time.time()), int(ftime))
                     bos.utime(dabs, times, False)
                 except:
                     pass
@@ -3828,16 +3896,19 @@ class Up2k(object):
         if xar:
             runhook(
                 self.log,
+                None,
+                self,
+                "xar.mv",
                 xar,
                 dabs,
                 dvp,
                 "",
                 uname,
                 self.asrv.vfs.get_perms(dvp, uname),
-                0,
-                0,
-                "",
-                0,
+                ftime,
+                fsize,
+                ip,
+                time.time(),
                 "",
             )
 
@@ -4121,23 +4192,35 @@ class Up2k(object):
         xbu = self.flags[job["ptop"]].get("xbu")
         ap_chk = djoin(pdir, job["name"])
         vp_chk = djoin(job["vtop"], job["prel"], job["name"])
-        if xbu and not runhook(
-            self.log,
-            xbu,
-            ap_chk,
-            vp_chk,
-            job["host"],
-            job["user"],
-            self.asrv.vfs.get_perms(vp_chk, job["user"]),
-            int(job["lmod"]),
-            job["size"],
-            job["addr"],
-            int(job["t0"]),
-            "",
-        ):
-            t = "upload blocked by xbu server config: {}".format(vp_chk)
-            self.log(t, 1)
-            raise Pebkac(403, t)
+        if xbu:
+            hr = runhook(
+                self.log,
+                None,
+                self,
+                "xbu.up2k",
+                xbu,
+                ap_chk,
+                vp_chk,
+                job["host"],
+                job["user"],
+                self.asrv.vfs.get_perms(vp_chk, job["user"]),
+                job["lmod"],
+                job["size"],
+                job["addr"],
+                job["t0"],
+                "",
+            )
+            if not hr:
+                t = "upload blocked by xbu server config: {}".format(vp_chk)
+                self.log(t, 1)
+                raise Pebkac(403, t)
+            if hr.get("reloc"):
+                x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"])
+                if x:
+                    pdir, _, job["name"], (vfs, rem) = x
+                    job["ptop"] = vfs.realpath
+                    job["vtop"] = vfs.vpath
+                    job["prel"] = rem
 
         tnam = job["name"] + ".PARTIAL"
         if self.args.dotpart:
@@ -4411,6 +4494,9 @@ class Up2k(object):
             with self.rescan_cond:
                 self.rescan_cond.notify_all()
 
+        if self.fx_backlog:
+            self.do_fx_backlog()
+
         return True
 
     def hash_file(
@@ -4442,6 +4528,48 @@ class Up2k(object):
             self.hashq.put(zt)
             self.n_hashq += 1
 
+    def do_fx_backlog(self):
+        with self.mutex, self.reg_mutex:
+            todo = self.fx_backlog
+            self.fx_backlog = []
+        for act, hr, req_vp in todo:
+            self.hook_fx(act, hr, req_vp)
+
+    def hook_fx(self, act , hr  , req_vp )  :
+        bad = [k for k in hr if k != "vp"]
+        if bad:
+            t = "got unsupported key in %s from hook: %s"
+            raise Exception(t % (act, bad))
+
+        for fvp in hr.get("vp") or []:
+            # expect vpath including filename; either absolute
+            # or relative to the client's vpath (request url)
+            if fvp.startswith("/"):
+                fvp, fn = vsplit(fvp[1:])
+                fvp = "/" + fvp
+            else:
+                fvp, fn = vsplit(fvp)
+
+            x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn})
+            if not x:
+                t = "hook_fx(%s): failed to resolve %s based on %s"
+                self.log(t % (act, fvp, req_vp))
+                continue
+
+            ap, rd, fn, (vn, rem) = x
+            vp = vjoin(rd, fn)
+            if not vp:
+                raise Exception("hook_fx: blank vp from pathmod")
+
+            if act == "idx":
+                rd = rd[len(vn.vpath) :].strip("/")
+                self.hash_file(
+                    vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True
+                )
+
+            if act == "del":
+                self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False)
+
     def shutdown(self)  :
         self.stop = True