copyparty 1.15.10__py3-none-any.whl → 1.16.1__py3-none-any.whl

copyparty/httpcli.py

@@ -181,6 +181,7 @@ class HttpCli(object):
         self.rem = " "
         self.vpath = " "
         self.vpaths = " "
+        self.dl_id = ""
         self.gctx = " "  # additional context for garda
         self.trailing_slash = True
         self.uname = " "
@@ -632,7 +633,7 @@ class HttpCli(object):
             avn.can_access("", self.uname) if avn else [False] * 8
         )
         self.avn = avn
-        self.vn = vn
+        self.vn = vn  # note: do not dbv due to walk/zipgen
         self.rem = rem
 
         self.s.settimeout(self.args.s_tbody or None)
@@ -721,6 +722,11 @@ class HttpCli(object):
             except Pebkac:
                 return False
 
+        finally:
+            if self.dl_id:
+                self.conn.hsrv.dli.pop(self.dl_id, None)
+                self.conn.hsrv.dls.pop(self.dl_id, None)
+
     def dip(self)  :
         if self.args.plain_ip:
             return self.ip.replace(":", ".")
@@ -1191,6 +1197,9 @@ class HttpCli(object):
             if "move" in self.uparam:
                 return self.handle_mv()
 
+            if "copy" in self.uparam:
+                return self.handle_cp()
+
         if not self.vpath and self.ouparam:
             if "reload" in self.uparam:
                 return self.handle_reload()
@@ -1210,6 +1219,9 @@ class HttpCli(object):
             if "shares" in self.uparam:
                 return self.tx_shares()
 
+            if "dls" in self.uparam:
+                return self.tx_dls()
+
         if "h" in self.uparam:
             return self.tx_mounts()
 
@@ -1445,6 +1457,7 @@ class HttpCli(object):
                 not self.args.no_scandir,
                 [[True, False]],
                 lstat="davrt" not in vn.flags,
+                throw=True,
             )
             if not self.can_read:
                 vfs_ls = []
@@ -1782,6 +1795,9 @@ class HttpCli(object):
         if "move" in self.uparam:
             return self.handle_mv()
 
+        if "copy" in self.uparam:
+            return self.handle_cp()
+
         if "delete" in self.uparam:
             return self.handle_rm([])
 
@@ -3665,6 +3681,8 @@ class HttpCli(object):
             self.args.s_wr_sz,
             self.args.s_wr_slp,
             not self.args.no_poll,
+            {},
+            "",
         )
         res.close()
 
@@ -3879,6 +3897,19 @@ class HttpCli(object):
             self.send_headers(length=upper - lower, status=status, mime=mime)
             return True
 
+        dls = self.conn.hsrv.dls
+        if upper - lower > 0x400000:  # 4m
+            now = time.time()
+            self.dl_id = "%s:%s" % (self.ip, self.addr[1])
+            dls[self.dl_id] = (now, 0)
+            self.conn.hsrv.dli[self.dl_id] = (
+                now,
+                upper - lower,
+                self.vn,
+                self.vpath,
+                self.uname,
+            )
+
         if ptop is not None:
             return self.tx_pipe(
                 ptop, req_path, ap_data, job, lower, upper, status, mime, logmsg
@@ -3898,6 +3929,8 @@ class HttpCli(object):
                 self.args.s_wr_sz,
                 self.args.s_wr_slp,
                 not self.args.no_poll,
+                dls,
+                self.dl_id,
             )
 
         if remains > 0:
@@ -4048,6 +4081,8 @@ class HttpCli(object):
                     wr_sz,
                     wr_slp,
                     not self.args.no_poll,
+                    self.conn.hsrv.dls,
+                    self.dl_id,
                 )
 
         spd = self._spd((upper - lower) - remains)
@@ -4133,6 +4168,18 @@ class HttpCli(object):
                 self.log("transcoding to [{}]".format(cfmt))
                 fgen = gfilter(fgen, self.thumbcli, self.uname, vpath, cfmt)
 
+        now = time.time()
+        self.dl_id = "%s:%s" % (self.ip, self.addr[1])
+        self.conn.hsrv.dli[self.dl_id] = (
+            now,
+            0,
+            self.vn,
+            "%s :%s" % (self.vpath, ext),
+            self.uname,
+        )
+        dls = self.conn.hsrv.dls
+        dls[self.dl_id] = (time.time(), 0)
+
         bgen = packer(
             self.log,
             self.asrv,
@@ -4141,6 +4188,7 @@ class HttpCli(object):
             pre_crc="crc" in uarg,
             cmp=uarg if cancmp or uarg == "pax" else "",
         )
+        n = 0
         bsent = 0
         for buf in bgen.gen():
             if not buf:
@@ -4154,6 +4202,11 @@ class HttpCli(object):
                 bgen.stop()
                 break
 
+            n += 1
+            if n >= 4:
+                n = 0
+                dls[self.dl_id] = (time.time(), bsent)
+
         spd = self._spd(bsent)
         self.log("{},  {}".format(logmsg, spd))
         return True
@@ -4410,6 +4463,32 @@ class HttpCli(object):
             }
 
 
+        dls = dl_list = []
+        if self.conn.hsrv.tdls:
+            zi = self.args.dl_list
+            if zi == 2 or (zi == 1 and self.avol):
+                dl_list = self.get_dls()
+        for t0, t1, sent, sz, vp, dl_id, uname in dl_list:
+            rem = sz - sent
+            td = max(0.1, now - t0)
+            rd, fn = vsplit(vp)
+            if not rd:
+                rd = "/"
+            erd = quotep(rd)
+            rds = rd.replace("/", " / ")
+            spd = humansize(sent / td, True) + "/s"
+            hsent = humansize(sent, True)
+            idle = s2hms(now - t1, True)
+            usr = "%s @%s" % (dl_id, uname) if dl_id else uname
+            if sz and sent and td:
+                eta = s2hms((sz - sent) / (sent / td), True)
+                perc = int(100 * sent / sz)
+            else:
+                eta = perc = "--"
+
+            fn = html_escape(fn) if fn else self.conn.hsrv.iiam
+            dls.append((perc, hsent, spd, eta, idle, usr, erd, rds, fn))
+
         fmt = self.uparam.get("ls", "")
         if not fmt and (self.ua.startswith("curl/") or self.ua.startswith("fetch")):
             fmt = "v"
@@ -4431,6 +4510,12 @@ class HttpCli(object):
                     txt += "\n%s" % (", ".join((str(x) for x in zt)),)
                 txt += "\n"
 
+            if dls:
+                txt += "\n\nactive downloads:"
+                for zt in dls:
+                    txt += "\n%s" % (", ".join((str(x) for x in zt)),)
+                txt += "\n"
+
             if rvol:
                 txt += "\nyou can browse:"
                 for v in rvol:
@@ -4454,6 +4539,7 @@ class HttpCli(object):
             avol=avol,
             in_shr=self.args.shr and self.vpath.startswith(self.args.shr1),
             vstate=vstate,
+            dls=dls,
             ups=ups,
             scanning=vs["scanning"],
             hashq=vs["hashq"],
@@ -4516,8 +4602,14 @@ class HttpCli(object):
 
         t = t.format(self.args.SR)
         qv = quotep(self.vpaths) + self.ourlq()
-        in_shr = self.args.shr and self.vpath.startswith(self.args.shr1)
-        html = self.j2s("splash", this=self, qvpath=qv, in_shr=in_shr, msg=t)
+        html = self.j2s(
+            "splash",
+            this=self,
+            qvpath=qv,
+            msg=t,
+            in_shr=self.args.shr and self.vpath.startswith(self.args.shr1),
+            ahttps="" if self.is_https else "https://" + self.host + self.req,
+        )
         self.reply(html.encode("utf-8"), status=rc)
         return True
 
@@ -4565,7 +4657,7 @@ class HttpCli(object):
         if self.args.no_reload:
             raise Pebkac(403, "the reload feature is disabled in server config")
 
-        x = self.conn.hsrv.broker.ask("reload")
+        x = self.conn.hsrv.broker.ask("reload", True, True)
         return self.redirect("", "?h", x.get(), "return to", False)
 
     def tx_stack(self)  :
@@ -4668,6 +4760,40 @@ class HttpCli(object):
         ret["a"] = dirs
         return ret
 
+    def get_dls(self)  :
+        ret = []
+        dls = self.conn.hsrv.tdls
+        for dl_id, (t0, sz, vn, vp, uname) in self.conn.hsrv.tdli.items():
+            t1, sent = dls[dl_id]
+            if sent > 0x100000:  # 1m; buffers 2~4
+                sent -= 0x100000
+            if self.uname not in vn.axs.uread:
+                vp = ""
+            elif self.uname not in vn.axs.udot and (vp.startswith(".") or "/." in vp):
+                vp = ""
+            if self.uname not in vn.axs.uadmin:
+                dl_id = uname = ""
+
+            ret.append([t0, t1, sent, sz, vp, dl_id, uname])
+        return ret
+
+    def tx_dls(self)  :
+        ret = [
+            {
+                "t0": x[0],
+                "t1": x[1],
+                "sent": x[2],
+                "size": x[3],
+                "path": x[4],
+                "conn": x[5],
+                "uname": x[6],
+            }
+            for x in self.get_dls()
+        ]
+        zs = json.dumps(ret, separators=(",\n", ": "))
+        self.reply(zs.encode("utf-8", "replace"), mime="application/json")
+        return True
+
     def tx_ups(self)  :
         idx = self.conn.get_u2idx()
         if not idx or not hasattr(idx, "p_end"):
@@ -4852,7 +4978,7 @@ class HttpCli(object):
 
         cur.connection.commit()
         if reload:
-            self.conn.hsrv.broker.ask("_reload_blocking", False, False).get()
+            self.conn.hsrv.broker.ask("reload", False, False).get()
             self.conn.hsrv.broker.ask("up2k.wake_rescanner").get()
 
         self.redirect(self.args.SRS + "?shares")
@@ -4943,7 +5069,7 @@ class HttpCli(object):
             cur.execute(q, (skey, fn))
 
         cur.connection.commit()
-        self.conn.hsrv.broker.ask("_reload_blocking", False, False).get()
+        self.conn.hsrv.broker.ask("reload", False, False).get()
         self.conn.hsrv.broker.ask("up2k.wake_rescanner").get()
 
         fn = quotep(fns[0]) if len(fns) == 1 else ""
@@ -4994,16 +5120,39 @@ class HttpCli(object):
         return self._mv(self.vpath, dst.lstrip("/"))
 
     def _mv(self, vsrc , vdst )  :
-        if not self.can_move:
-            raise Pebkac(403, "not allowed for user " + self.uname)
-
         if self.args.no_mv:
             raise Pebkac(403, "the rename/move feature is disabled in server config")
 
+        self.asrv.vfs.get(vsrc, self.uname, True, False, True)
+        self.asrv.vfs.get(vdst, self.uname, False, True)
+
         x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst)
         self.loud_reply(x.get(), status=201)
         return True
 
+    def handle_cp(self)  :
+        # full path of new loc (incl filename)
+        dst = self.uparam.get("copy")
+
+        if self.is_vproxied and dst and dst.startswith(self.args.SR):
+            dst = dst[len(self.args.RS) :]
+
+        if not dst:
+            raise Pebkac(400, "need dst vpath")
+
+        return self._cp(self.vpath, dst.lstrip("/"))
+
+    def _cp(self, vsrc , vdst )  :
+        if self.args.no_cp:
+            raise Pebkac(403, "the copy feature is disabled in server config")
+
+        self.asrv.vfs.get(vsrc, self.uname, True, False)
+        self.asrv.vfs.get(vdst, self.uname, False, True)
+
+        x = self.conn.hsrv.broker.ask("up2k.handle_cp", self.uname, self.ip, vsrc, vdst)
+        self.loud_reply(x.get(), status=201)
+        return True
+
     def tx_ls(self, ls  )  :
         dirs = ls["dirs"]
         files = ls["files"]
@@ -5433,6 +5582,7 @@ class HttpCli(object):
             not self.args.no_scandir,
             [[True, False], [False, True]],
             lstat="lt" in self.uparam,
+            throw=True,
         )
         stats = {k: v for k, v in vfs_ls}
         ls_names = [x[0] for x in vfs_ls]

copyparty/httpsrv.py

@@ -81,6 +81,7 @@ from .util import (
 )
 
 if TYPE_CHECKING:
+    from .authsrv import VFS
     from .broker_util import BrokerCli
     from .ssdp import SSDPr
 
@@ -127,6 +128,12 @@ class HttpSrv(object):
         self.bans   = {}
         self.aclose   = {}
 
+        dli       = {}  # info
+        dls    = {}  # state
+        self.dli = self.tdli = dli
+        self.dls = self.tdls = dls
+        self.iiam = '<img src="%s.cpr/iiam.gif" />' % (self.args.SRS,)
+
         self.bound   = set()
         self.name = "hsrv" + nsuf
         self.mutex = threading.Lock()
@@ -201,6 +208,9 @@ class HttpSrv(object):
             self.start_threads(4)
 
         if nid:
+            self.tdli = {}
+            self.tdls = {}
+
             if self.args.stackmon:
                 start_stackmon(self.args.stackmon, nid)
 
@@ -573,3 +583,32 @@ class HttpSrv(object):
                 ident += "a"
 
             self.u2idx_free[ident] = u2idx
+
+    def read_dls(
+        self,
+    ):  
+                
+    
+        """
+        mp-broker asking for local dl-info + dl-state;
+        reduce overhead by sending just the vfs vpath
+        """
+        dli = {k: (a, b, c.vpath, d, e) for k, (a, b, c, d, e) in self.dli.items()}
+        return (dli, self.dls)
+
+    def write_dls(
+        self,
+        sdli      ,
+        dls   ,
+    )  :
+        """
+        mp-broker pushing total dl-info + dl-state;
+        swap out the vfs vpath with the vfs node
+        """
+        dli       = {}
+        for k, (a, b, c, d, e) in sdli.items():
+            vn = self.asrv.vfs.all_nodes[c]
+            dli[k] = (a, b, vn, d, e)
+
+        self.tdli = dli
+        self.tdls = dls

copyparty/metrics.py

@@ -72,6 +72,9 @@ class Metrics(object):
         v = "{:.3f}".format(self.hsrv.t0)
         addug("cpp_boot_unixtime", "seconds", v, t)
 
+        t = "number of active downloads"
+        addg("cpp_active_dl", str(len(self.hsrv.tdls)), t)
+
         t = "number of open http(s) client connections"
         addg("cpp_http_conns", str(self.hsrv.ncli), t)
 

copyparty/mtag.py

@@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals
 import argparse
 import json
 import os
+import re
 import shutil
 import subprocess as sp
 import sys
@@ -56,6 +57,9 @@ def have_ff(scmd )  :
 HAVE_FFMPEG = not os.environ.get("PRTY_NO_FFMPEG") and have_ff("ffmpeg")
 HAVE_FFPROBE = not os.environ.get("PRTY_NO_FFPROBE") and have_ff("ffprobe")
 
+CBZ_PICS = set("png jpg jpeg gif bmp tga tif tiff webp avif".split())
+CBZ_01 = re.compile(r"(^|[^0-9v])0+[01]\b")
+
 
 class MParser(object):
     def __init__(self, cmdline )  :
@@ -120,6 +124,7 @@ def au_unpk(
     log , fmt_map  , abspath , vn  = None
 )  :
     ret = ""
+    maxsz = 1024 * 1024 * 64
     try:
         ext = abspath.split(".")[-1].lower()
         au, pk = fmt_map[ext].split(".")
@@ -142,17 +147,41 @@ def au_unpk(
             zf = zipfile.ZipFile(abspath, "r")
             zil = zf.infolist()
             zil = [x for x in zil if x.filename.lower().split(".")[-1] == au]
+            if not zil:
+                raise Exception("no audio inside zip")
             fi = zf.open(zil[0])
 
+        elif pk == "cbz":
+            import zipfile
+
+            zf = zipfile.ZipFile(abspath, "r")
+            znil = [(x.filename.lower(), x) for x in zf.infolist()]
+            nf = len(znil)
+            znil = [x for x in znil if x[0].split(".")[-1] in CBZ_PICS]
+            znil = [x for x in znil if "cover" in x[0]] or znil
+            znil = [x for x in znil if CBZ_01.search(x[0])] or znil
+            t = "cbz: %d files, %d hits" % (nf, len(znil))
+            if znil:
+                t += ", using " + znil[0][1].filename
+            log(t)
+            if not znil:
+                raise Exception("no images inside cbz")
+            fi = zf.open(znil[0][1])
+
         else:
             raise Exception("unknown compression %s" % (pk,))
 
+        fsz = 0
         with os.fdopen(fd, "wb") as fo:
             while True:
                 buf = fi.read(32768)
                 if not buf:
                     break
 
+                fsz += len(buf)
+                if fsz > maxsz:
+                    raise Exception("zipbomb defused")
+
                 fo.write(buf)
 
         return ret

copyparty/pwhash.py

@@ -24,17 +24,13 @@ class PWHash(object):
     def __init__(self, args ):
         self.args = args
 
-        try:
-            alg, ac = args.ah_alg.split(",")
-        except:
-            alg = args.ah_alg
-            ac = {}
-
+        zsl = args.ah_alg.split(",")
+        alg = zsl[0]
         if alg == "none":
             alg = ""
 
         self.alg = alg
-        self.ac = ac
+        self.ac = zsl[1:]
         if not alg:
             self.on = False
             self.hash = unicode
@@ -90,17 +86,23 @@ class PWHash(object):
         its = 2
         blksz = 8
         para = 4
+        ramcap = 0  # openssl 1.1 = 32 MiB
         try:
             cost = 2 << int(self.ac[0])
             its = int(self.ac[1])
             blksz = int(self.ac[2])
             para = int(self.ac[3])
+            ramcap = int(self.ac[4]) * 1024 * 1024
         except:
             pass
 
+        cfg = {"salt": self.salt, "n": cost, "r": blksz, "p": para, "dklen": 24}
+        if ramcap:
+            cfg["maxmem"] = ramcap
+
         ret = plain.encode("utf-8")
         for _ in range(its):
-            ret = hashlib.scrypt(ret, salt=self.salt, n=cost, r=blksz, p=para, dklen=24)
+            ret = hashlib.scrypt(ret, **cfg)
 
         return "+" + base64.urlsafe_b64encode(ret).decode("utf-8")
 

copyparty/svchub.py

@@ -106,7 +106,7 @@ class SvcHub(object):
         self.stopping = False
         self.stopped = False
         self.reload_req = False
-        self.reloading = 0
+        self.reload_mutex = threading.Lock()
         self.stop_cond = threading.Condition()
         self.nsigs = 3
         self.retcode = 0
@@ -205,6 +205,15 @@ class SvcHub(object):
             t = "WARNING: --s-rd-sz (%d) is larger than --iobuf (%d); this may lead to reduced performance"
             self.log("root", t % (args.s_rd_sz, args.iobuf), 3)
 
+        zs = ""
+        if args.th_ram_max < 0.22:
+            zs = "generate thumbnails"
+        elif args.th_ram_max < 1:
+            zs = "generate audio waveforms or spectrograms"
+        if zs:
+            t = "WARNING: --th-ram-max is very small (%.2f GiB); will not be able to %s"
+            self.log("root", t % (args.th_ram_max, zs), 3)
+
         if args.chpw and args.idp_h_usr:
             t = "ERROR: user-changeable passwords is incompatible with IdP/identity-providers; you must disable either --chpw or --idp-h-usr"
             self.log("root", t, 1)
@@ -994,41 +1003,18 @@ class SvcHub(object):
             except:
                 self.log("root", "ssdp startup failed;\n" + min_ex(), 3)
 
-    def reload(self)  :
-        with self.up2k.mutex:
-            if self.reloading:
-                return "cannot reload; already in progress"
-            self.reloading = 1
-
-        Daemon(self._reload, "reloading")
-        return "reload initiated"
-
-    def _reload(self, rescan_all_vols  = True, up2k  = True)  :
-        with self.up2k.mutex:
-            if self.reloading != 1:
-                return
-            self.reloading = 2
+    def reload(self, rescan_all_vols , up2k )  :
+        t = "config has been reloaded"
+        with self.reload_mutex:
             self.log("root", "reloading config")
             self.asrv.reload(9 if up2k else 4)
             if up2k:
                 self.up2k.reload(rescan_all_vols)
+                t += "; volumes are now reinitializing"
             else:
                 self.log("root", "reload done")
             self.broker.reload()
-            self.reloading = 0
-
-    def _reload_blocking(self, rescan_all_vols  = True, up2k  = True)  :
-        while True:
-            with self.up2k.mutex:
-                if self.reloading < 2:
-                    self.reloading = 1
-                    break
-            time.sleep(0.05)
-
-        # try to handle multiple pending IdP reloads at once:
-        time.sleep(0.2)
-
-        self._reload(rescan_all_vols=rescan_all_vols, up2k=up2k)
+        return t
 
     def _reload_sessions(self)  :
         with self.asrv.mutex:
@@ -1042,7 +1028,7 @@ class SvcHub(object):
 
             if self.reload_req:
                 self.reload_req = False
-                self.reload()
+                self.reload(True, True)
 
         self.shutdown()
 

copyparty/szip.py

@@ -94,7 +94,7 @@ def gen_hdr(
 
     # spec says to put zeros when !crc if bit3 (streaming)
     # however infozip does actual sz and it even works on winxp
-    # (same reasning for z64 extradata later)
+    # (same reasoning for z64 extradata later)
     vsz = 0xFFFFFFFF if z64 else sz
     ret += spack(b"<LL", vsz, vsz)
 

copyparty/tcpsrv.py

@@ -368,7 +368,7 @@ class TcpSrv(object):
             if self.args.q:
                 print(msg)
 
-            self.hub.broker.say("listen", srv)
+            self.hub.broker.say("httpsrv.listen", srv)
 
         self.srv = srvs
         self.bound = bound
@@ -376,7 +376,7 @@ class TcpSrv(object):
         self._distribute_netdevs()
 
     def _distribute_netdevs(self):
-        self.hub.broker.say("set_netdevs", self.netdevs)
+        self.hub.broker.say("httpsrv.set_netdevs", self.netdevs)
         self.hub.start_zeroconf()
         gencert(self.log, self.args, self.netdevs)
         self.hub.restart_ftpd()

copyparty/tftpd.py

@@ -266,6 +266,7 @@ class Tftpd(object):
             "*",
             not self.args.no_scandir,
             [[True, False]],
+            throw=True,
         )
         dnames = set([x[0] for x in vfs_ls if stat.S_ISDIR(x[1].st_mode)])
         dirs1 = [(v.st_mtime, v.st_size, k + "/") for k, v in vfs_ls if k in dnames]

copyparty/th_srv.py

@@ -20,7 +20,6 @@ from .util import (
     FFMPEG_URL,
     Cooldown,
     Daemon,
-    Pebkac,
     afsenc,
     fsenc,
     min_ex,
@@ -161,6 +160,7 @@ class ThumbSrv(object):
         self.ram   = {}
         self.memcond = threading.Condition(self.mutex)
         self.stopping = False
+        self.rm_nullthumbs = True  # forget failed conversions on startup
         self.nthr = max(1, self.args.th_mt)
 
         self.q     = Queue(self.nthr * 4)
@@ -858,7 +858,6 @@ class ThumbSrv(object):
     def cleaner(self)  :
         interval = self.args.th_clean
         while True:
-            time.sleep(interval)
             ndirs = 0
             for vol, histpath in self.asrv.vfs.histtab.items():
                 if histpath.startswith(vol):
@@ -872,6 +871,8 @@ class ThumbSrv(object):
                     self.log("\033[Jcln err in %s: %r" % (histpath, ex), 3)
 
             self.log("\033[Jcln ok; rm {} dirs".format(ndirs))
+            self.rm_nullthumbs = False
+            time.sleep(interval)
 
     def clean(self, histpath )  :
         ret = 0
@@ -892,7 +893,9 @@ class ThumbSrv(object):
         prev_b64 = None
         prev_fp = ""
         try:
-            t1 = statdir(self.log_func, not self.args.no_scandir, False, thumbpath)
+            t1 = statdir(
+                self.log_func, not self.args.no_scandir, False, thumbpath, False
+            )
             ents = sorted(list(t1))
         except:
             return 0
@@ -933,6 +936,10 @@ class ThumbSrv(object):
 
                 continue
 
+            if self.rm_nullthumbs and not inf.st_size:
+                bos.unlink(fp)
+                continue
+
             if b64 == prev_b64:
                 self.log("rm replaced [{}]".format(fp))
                 bos.unlink(prev_fp)