copyparty 1.15.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

copyparty/httpcli.py

@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import argparse  # typechk
-import base64
 import calendar
 import copy
 import errno
@@ -58,6 +57,7 @@ from .util import (
     absreal,
     alltrace,
     atomic_move,
+    b64dec,
     exclude_dotfiles,
     formatdate,
     fsenc,
@@ -87,6 +87,7 @@ from .util import (
     relchk,
     ren_open,
     runhook,
+    s2hms,
     s3enc,
     sanitize_fn,
     sanitize_vpath,
@@ -123,7 +124,6 @@ class HttpCli(object):
     """
 
     def __init__(self, conn )  :
-        assert conn.sr
 
         self.t0 = time.time()
         self.conn = conn
@@ -498,7 +498,7 @@ class HttpCli(object):
         ):
             try:
                 zb = zso.split(" ")[1].encode("ascii")
-                zs = base64.b64decode(zb).decode("utf-8")
+                zs = b64dec(zb).decode("utf-8")
                 # try "pwd", "x:pwd", "pwd:x"
                 for bauth in [zs] + zs.split(":", 1)[::-1]:
                     if bauth in self.asrv.sesa:
@@ -1391,7 +1391,6 @@ class HttpCli(object):
         xroot = mkenod("D:orz")
         xroot.insert(0, parse_xml(txt))
         xprop = xroot.find(r"./{DAV:}propertyupdate/{DAV:}set/{DAV:}prop")
-        assert xprop
         for ze in xprop:
             ze.clear()
 
@@ -1399,12 +1398,10 @@ class HttpCli(object):
         xroot = parse_xml(txt)
 
         el = xroot.find(r"./{DAV:}response")
-        assert el
         e2 = mktnod("D:href", quotep(self.args.SRS + self.vpath))
         el.insert(0, e2)
 
         el = xroot.find(r"./{DAV:}response/{DAV:}propstat")
-        assert el
         el.insert(0, xprop)
 
         ret = '<?xml version="1.0" encoding="{}"?>\n'.format(uenc)
@@ -1788,7 +1785,6 @@ class HttpCli(object):
             fn = os.devnull
 
         params.update(open_ka)
-        assert fn
 
         if not self.args.nw:
             if rnd:
@@ -1860,10 +1856,12 @@ class HttpCli(object):
                 # small toctou, but better than clobbering a hardlink
                 wunlink(self.log, path, vfs.flags)
 
-        with ren_open(fn, *open_a, **params) as zfw:
-            f, fn = zfw["orz"]
+        f, fn = ren_open(fn, *open_a, **params)
+        try:
             path = os.path.join(fdir, fn)
             post_sz, sha_hex, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
+        finally:
+            f.close()
 
         if lim:
             lim.nup(self.ip)
@@ -1902,8 +1900,8 @@ class HttpCli(object):
                     fn2 = fn.rsplit(".", 1)[0] + "." + ext
 
                 params["suffix"] = suffix[:-4]
-                with ren_open(fn, *open_a, **params) as zfw:
-                    f, fn = zfw["orz"]
+                f, fn2 = ren_open(fn2, *open_a, **params)
+                f.close()
 
                 path2 = os.path.join(fdir, fn2)
                 atomic_move(self.log, path, path2, vfs.flags)
@@ -2097,7 +2095,6 @@ class HttpCli(object):
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
     def handle_zip_post(self)  :
-        assert self.parser
         try:
             k = next(x for x in self.uparam if x in ("zip", "tar"))
         except:
@@ -2297,11 +2294,16 @@ class HttpCli(object):
         vfs, _ = self.asrv.vfs.get(self.vpath, self.uname, False, True)
         ptop = (vfs.dbv or vfs).realpath
 
-        x = self.conn.hsrv.broker.ask("up2k.handle_chunks", ptop, wark, chashes)
+        broker = self.conn.hsrv.broker
+        x = broker.ask("up2k.handle_chunks", ptop, wark, chashes)
         response = x.get()
         chashes, chunksize, cstarts, path, lastmod, sprs = response
         maxsize = chunksize * len(chashes)
         cstart0 = cstarts[0]
+        locked = chashes  # remaining chunks to be received in this request
+        written = []  # chunks written to disk, but not yet released by up2k
+        num_left = -1  # num chunks left according to most recent up2k release
+        treport = time.time()  # ratelimit up2k reporting to reduce overhead
 
         try:
             if self.args.nw:
@@ -2347,11 +2349,8 @@ class HttpCli(object):
                     remains -= chunksize
 
                     if len(cstart) > 1 and path != os.devnull:
-                        self.log(
-                            "clone {} to {}".format(
-                                cstart[0], " & ".join(unicode(x) for x in cstart[1:])
-                            )
-                        )
+                        t = " & ".join(unicode(x) for x in cstart[1:])
+                        self.log("clone %s to %s" % (cstart[0], t))
                         ofs = 0
                         while ofs < chunksize:
                             bufsz = max(4 * 1024 * 1024, self.args.iobuf)
@@ -2366,6 +2365,25 @@ class HttpCli(object):
 
                         self.log("clone {} done".format(cstart[0]))
 
+                    # be quick to keep the tcp winsize scale;
+                    # if we can't confirm rn then that's fine
+                    written.append(chash)
+                    now = time.time()
+                    if now - treport < 1:
+                        continue
+                    treport = now
+                    x = broker.ask("up2k.fast_confirm_chunks", ptop, wark, written)
+                    num_left, t = x.get()
+                    if num_left < -1:
+                        self.loud_reply(t, status=500)
+                        locked = written = []
+                        return False
+                    elif num_left >= 0:
+                        t = "got %d more chunks, %d left"
+                        self.log(t % (len(written), num_left), 6)
+                        locked = locked[len(written) :]
+                        written = []
+
                 if not fpool:
                     f.close()
                 else:
@@ -2376,25 +2394,25 @@ class HttpCli(object):
                 f.close()
                 raise
         finally:
-            x = self.conn.hsrv.broker.ask("up2k.release_chunks", ptop, wark, chashes)
-            x.get()  # block client until released
+            if locked:
+                # now block until all chunks released+confirmed
+                x = broker.ask("up2k.confirm_chunks", ptop, wark, locked)
+                num_left, t = x.get()
+                if num_left < 0:
+                    self.loud_reply(t, status=500)
+                    return False
+                t = "got %d more chunks, %d left"
+                self.log(t % (len(locked), num_left), 6)
 
-        x = self.conn.hsrv.broker.ask("up2k.confirm_chunks", ptop, wark, chashes)
-        ztis = x.get()
-        try:
-            num_left, fin_path = ztis
-        except:
-            self.loud_reply(ztis, status=500)
-            return False
+        if num_left < 0:
+            raise Pebkac(500, "unconfirmed; see serverlog")
 
         if not num_left and fpool:
             with self.u2mutex:
                 self.u2fh.close(path)
 
         if not num_left and not self.args.nw:
-            self.conn.hsrv.broker.ask(
-                "up2k.finish_upload", ptop, wark, self.u2fh.aps
-            ).get()
+            broker.ask("up2k.finish_upload", ptop, wark, self.u2fh.aps).get()
 
         cinf = self.headers.get("x-up2k-stat", "")
 
@@ -2404,7 +2422,6 @@ class HttpCli(object):
         return True
 
     def handle_chpw(self)  :
-        assert self.parser
         pwd = self.parser.require("pw", 64)
         self.parser.drop()
 
@@ -2421,7 +2438,6 @@ class HttpCli(object):
         return True
 
     def handle_login(self)  :
-        assert self.parser
         pwd = self.parser.require("cppwd", 64)
         try:
             uhash = self.parser.require("uhash", 256)
@@ -2449,7 +2465,6 @@ class HttpCli(object):
         return True
 
     def handle_logout(self)  :
-        assert self.parser
         self.parser.drop()
 
         self.log("logout " + self.uname)
@@ -2478,7 +2493,7 @@ class HttpCli(object):
                 logpwd = ""
             elif self.args.log_badpwd == 2:
                 zb = hashlib.sha512(pwd.encode("utf-8", "replace")).digest()
-                logpwd = "%" + base64.b64encode(zb[:12]).decode("utf-8")
+                logpwd = "%" + ub64enc(zb[:12]).decode("ascii")
 
             if pwd != "x":
                 self.log("invalid password: {}".format(logpwd), 3)
@@ -2503,7 +2518,6 @@ class HttpCli(object):
         return dur > 0, msg
 
     def handle_mkdir(self)  :
-        assert self.parser
         new_dir = self.parser.require("name", 512)
         self.parser.drop()
 
@@ -2549,7 +2563,6 @@ class HttpCli(object):
         return True
 
     def handle_new_md(self)  :
-        assert self.parser
         new_file = self.parser.require("name", 512)
         self.parser.drop()
 
@@ -2715,8 +2728,8 @@ class HttpCli(object):
                     bos.makedirs(fdir)
 
                     # reserve destination filename
-                    with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
-                        fname = zfw["orz"][1]
+                    f, fname = ren_open(fname, "wb", fdir=fdir, suffix=suffix)
+                    f.close()
 
                     tnam = fname + ".PARTIAL"
                     if self.args.dotpart:
@@ -2739,8 +2752,8 @@ class HttpCli(object):
                         v2 = lim.dfv - lim.dfl
                         max_sz = min(v1, v2) if v1 and v2 else v1 or v2
 
-                    with ren_open(tnam, "wb", self.args.iobuf, **open_args) as zfw:
-                        f, tnam = zfw["orz"]
+                    f, tnam = ren_open(tnam, "wb", self.args.iobuf, **open_args)
+                    try:
                         tabspath = os.path.join(fdir, tnam)
                         self.log("writing to {}".format(tabspath))
                         sz, sha_hex, sha_b64 = hashcopy(
@@ -2748,6 +2761,8 @@ class HttpCli(object):
                         )
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
+                    finally:
+                        f.close()
 
                     if lim:
                         lim.nup(self.ip)
@@ -2957,7 +2972,6 @@ class HttpCli(object):
         return True
 
     def handle_text_upload(self)  :
-        assert self.parser
         try:
             cli_lastmod3 = int(self.parser.require("lastmod", 16))
         except:
@@ -3042,7 +3056,6 @@ class HttpCli(object):
                 pass
             wrename(self.log, fp, os.path.join(mdir, ".hist", mfile2), vfs.flags)
 
-        assert self.parser.gen
         p_field, _, p_data = next(self.parser.gen)
         if p_field != "body":
             raise Pebkac(400, "expected body, got {}".format(p_field))
@@ -3143,7 +3156,6 @@ class HttpCli(object):
                 # some browser append "; length=573"
                 cli_lastmod = cli_lastmod.split(";")[0].strip()
                 cli_dt = parsedate(cli_lastmod)
-                assert cli_dt
                 cli_ts = calendar.timegm(cli_dt)
                 return file_lastmod, int(file_ts) > int(cli_ts)
             except Exception as ex:
@@ -3911,6 +3923,9 @@ class HttpCli(object):
         vp = re.sub(r"[<>&$?`\"']", "_", self.uparam["hc"] or "").lstrip("/")
         pw = pw.replace(" ", "%20")
         vp = vp.replace(" ", "%20")
+        if pw in self.asrv.sesa:
+            pw = "pwd"
+
         html = self.j2s(
             "svcs",
             args=self.args,
@@ -3935,11 +3950,30 @@ class HttpCli(object):
             for y in [self.rvol, self.wvol, self.avol]
         ]
 
-        if self.avol and not self.args.no_rescan:
-            x = self.conn.hsrv.broker.ask("up2k.get_state")
+        ups = []
+        now = time.time()
+        get_vst = self.avol and not self.args.no_rescan
+        get_ups = self.rvol and not self.args.no_up_list and self.uname or ""
+        if get_vst or get_ups:
+            x = self.conn.hsrv.broker.ask("up2k.get_state", get_vst, get_ups)
             vs = json.loads(x.get())
             vstate = {("/" + k).rstrip("/") + "/": v for k, v in vs["volstate"].items()}
-        else:
+            try:
+                for rem, sz, t0, poke, vp in vs["ups"]:
+                    fdone = max(0.001, 1 - rem)
+                    td = max(0.1, now - t0)
+                    rd, fn = vsplit(vp.replace(os.sep, "/"))
+                    if not rd:
+                        rd = "/"
+                    erd = quotep(rd)
+                    rds = rd.replace("/", " / ")
+                    spd = humansize(sz * fdone / td, True) + "/s"
+                    eta = s2hms((td / fdone) - td, True)
+                    idle = s2hms(now - poke, True)
+                    ups.append((int(100 * fdone), spd, eta, idle, erd, rds, fn))
+            except Exception as ex:
+                self.log("failed to list upload progress: %r" % (ex,), 1)
+        if not get_vst:
             vstate = {}
             vs = {
                 "scanning": None,
@@ -3964,6 +3998,12 @@ class HttpCli(object):
                 for k in ["scanning", "hashq", "tagq", "mtpq", "dbwt"]:
                     txt += " {}({})".format(k, vs[k])
 
+            if ups:
+                txt += "\n\nincoming files:"
+                for zt in ups:
+                    txt += "\n%s" % (", ".join((str(x) for x in zt)),)
+                txt += "\n"
+
             if rvol:
                 txt += "\nyou can browse:"
                 for v in rvol:
@@ -3987,6 +4027,7 @@ class HttpCli(object):
             avol=avol,
             in_shr=self.args.shr and self.vpath.startswith(self.args.shr[1:]),
             vstate=vstate,
+            ups=ups,
             scanning=vs["scanning"],
             hashq=vs["hashq"],
             tagq=vs["tagq"],
@@ -5091,7 +5132,6 @@ class HttpCli(object):
                 dirs.append(item)
             else:
                 files.append(item)
-                item["rd"] = rem
 
         if is_dk and not vf.get("dks"):
             dirs = []
@@ -5114,16 +5154,9 @@ class HttpCli(object):
         add_up_at = ".up_at" in mte
         is_admin = self.can_admin
         tagset  = set()
-        for fe in files:
+        rd = vrem
+        for fe in files if icur else []:
             fn = fe["name"]
-            rd = fe["rd"]
-            del fe["rd"]
-            if not icur:
-                continue
-
-            if vn != dbv:
-                _, rd = vn.get_dbv(rd)
-
             erd_efn = (rd, fn)
             q = "select mt.k, mt.v from up inner join mt on mt.w = substr(up.w,1,16) where up.rd = ? and up.fn = ? and +mt.k != 'x'"
             try:
@@ -5165,13 +5198,25 @@ class HttpCli(object):
             fe["tags"] = tags
 
         if icur:
+            for fe in dirs:
+                fe["tags"] = ODict()
+
             lmte = list(mte)
             if self.can_admin:
                 lmte.extend(("up_ip", ".up_at"))
 
+            if "nodirsz" not in vf:
+                tagset.add(".files")
+                vdir = "%s/" % (rd,) if rd else ""
+                q = "select sz, nf from ds where rd=? limit 1"
+                for fe in dirs:
+                    try:
+                        hit = icur.execute(q, (vdir + fe["name"],)).fetchone()
+                        (fe["sz"], fe["tags"][".files"]) = hit
+                    except:
+                        pass  # 404 or mojibake
+
             taglist = [k for k in lmte if k in tagset]
-            for fe in dirs:
-                fe["tags"] = ODict()
         else:
             taglist = list(tagset)
 
@@ -5315,7 +5360,7 @@ class HttpCli(object):
                 fmt = vn.flags.get("og_th", "j")
                 th_base = ujoin(url_base, quotep(thumb))
                 query = "th=%s&cache" % (fmt,)
-                query = ub64enc(query.encode("utf-8")).decode("utf-8")
+                query = ub64enc(query.encode("utf-8")).decode("ascii")
                 # discord looks at file extension, not content-type...
                 query += "/th.jpg" if "j" in fmt else "/th.webp"
                 j2a["og_thumb"] = "%s/.uqe/%s" % (th_base, query)
@@ -5324,7 +5369,7 @@ class HttpCli(object):
             j2a["og_file"] = file
             if og_fn:
                 og_fn_q = quotep(og_fn)
-                query = ub64enc(b"raw").decode("utf-8")
+                query = ub64enc(b"raw").decode("ascii")
                 query += "/%s" % (og_fn_q,)
                 j2a["og_url"] = ujoin(url_base, og_fn_q)
                 j2a["og_raw"] = j2a["og_url"] + "/.uqe/" + query

copyparty/httpconn.py

@@ -187,7 +187,6 @@ class HttpConn(object):
 
                 if self.args.ssl_dbg and hasattr(self.s, "shared_ciphers"):
                     ciphers = self.s.shared_ciphers()
-                    assert ciphers
                     overlap = [str(y[::-1]) for y in ciphers]
                     self.log("TLS cipher overlap:" + "\n".join(overlap))
                     for k, v in [

copyparty/httpsrv.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import base64
 import math
 import os
 import re
@@ -75,6 +74,7 @@ from .util import (
     spack,
     start_log_thrs,
     start_stackmon,
+    ub64enc,
 )
 
 if TYPE_CHECKING:
@@ -234,7 +234,6 @@ class HttpSrv(object):
         if self.args.log_htp:
             self.log(self.name, "workers -= {} = {}".format(n, self.tp_nthr), 6)
 
-        assert self.tp_q
         for _ in range(n):
             self.tp_q.put(None)
 
@@ -428,7 +427,6 @@ class HttpSrv(object):
         )
 
     def thr_poolw(self)  :
-        assert self.tp_q
         while True:
             task = self.tp_q.get()
             if not task:
@@ -540,8 +538,8 @@ class HttpSrv(object):
             except:
                 pass
 
-            v = base64.urlsafe_b64encode(spack(b">xxL", int(v)))
-            self.cb_v = v.decode("ascii")[-4:]
+            # spack gives 4 lsb, take 3 lsb, get 4 ch
+            self.cb_v = ub64enc(spack(b">L", int(v))[1:]).decode("ascii")
             self.cb_ts = time.time()
             return self.cb_v
 

copyparty/svchub.py

@@ -2,7 +2,6 @@
 from __future__ import print_function, unicode_literals
 
 import argparse
-import base64
 import errno
 import gzip
 import logging
@@ -61,6 +60,7 @@ from .util import (
     pybin,
     start_log_thrs,
     start_stackmon,
+    ub64enc,
 )
 
 if TYPE_CHECKING:
@@ -413,8 +413,6 @@ class SvcHub(object):
             r"insert into kv values ('sver', 1)",
         ]
 
-        assert db  # type: ignore
-        assert cur  # type: ignore
         if create:
             for cmd in sch:
                 cur.execute(cmd)
@@ -482,8 +480,6 @@ class SvcHub(object):
             r"create index sh_t1 on sh(t1)",
         ]
 
-        assert db  # type: ignore
-        assert cur  # type: ignore
         if create:
             dver = 2
             modified = True
@@ -1291,5 +1287,5 @@ class SvcHub(object):
         zs = "{}\n{}".format(VERSIONS, alltrace())
         zb = zs.encode("utf-8", "replace")
         zb = gzip.compress(zb)
-        zs = base64.b64encode(zb).decode("ascii")
+        zs = ub64enc(zb).decode("ascii")
         self.log("stacks", zs)

copyparty/th_srv.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import base64
 import hashlib
 import logging
 import os
@@ -27,6 +26,7 @@ from .util import (
     min_ex,
     runcmd,
     statdir,
+    ub64enc,
     vsplit,
     wrename,
     wunlink,
@@ -106,6 +106,9 @@ except:
     HAVE_VIPS = False
 
 
+th_dir_cache = {}
+
+
 def thumb_path(histpath , rem , mtime , fmt , ffa )  :
     # base16 = 16 = 256
     # b64-lc = 38 = 1444
@@ -119,14 +122,20 @@ def thumb_path(histpath , rem , mtime , fmt , ffa )  :
     if ext in ffa and fmt[:2] in ("wf", "jf"):
         fmt = fmt.replace("f", "")
 
-    rd += "\n" + fmt
-    h = hashlib.sha512(afsenc(rd)).digest()
-    b64 = base64.urlsafe_b64encode(h).decode("ascii")[:24]
-    rd = ("%s/%s/" % (b64[:2], b64[2:4])).lower() + b64
+    dcache = th_dir_cache
+    rd_key = rd + "\n" + fmt
+    rd = dcache.get(rd_key)
+    if not rd:
+        h = hashlib.sha512(afsenc(rd_key)).digest()
+        b64 = ub64enc(h).decode("ascii")[:24]
+        rd = ("%s/%s/" % (b64[:2], b64[2:4])).lower() + b64
+        if len(dcache) > 9001:
+            dcache.clear()
+        dcache[rd_key] = rd
 
     # could keep original filenames but this is safer re pathlen
     h = hashlib.sha512(afsenc(fn)).digest()
-    fn = base64.urlsafe_b64encode(h).decode("ascii")[:24]
+    fn = ub64enc(h).decode("ascii")[:24]
 
     if fmt in ("opus", "caf", "mp3"):
         cat = "ac"
@@ -476,7 +485,6 @@ class ThumbSrv(object):
                 if c == crops[-1]:
                     raise
 
-        assert img  # type: ignore
         img.write_to_file(tpath, Q=40)
 
     def conv_ffmpeg(self, abspath , tpath , fmt , vn )  :

copyparty/u2idx.py

@@ -101,7 +101,6 @@ class U2idx(object):
         if not HAVE_SQLITE3 or not self.args.shr:
             return None
 
-        assert sqlite3  # type: ignore
 
         db = sqlite3.connect(self.args.shr_db, timeout=2, check_same_thread=False)
         cur = db.cursor()
@@ -117,7 +116,6 @@ class U2idx(object):
         if not HAVE_SQLITE3 or "e2d" not in vn.flags:
             return None
 
-        assert sqlite3  # type: ignore
 
         ptop = vn.realpath
         histpath = self.asrv.vfs.histtab.get(ptop)
@@ -464,5 +462,4 @@ class U2idx(object):
                 return
 
         if identifier == self.active_id:
-            assert self.active_cur
             self.active_cur.connection.interrupt()