copyparty 1.15.0__py3-none-any.whl → 1.15.2__py3-none-any.whl

copyparty/up2k.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import base64
 import errno
 import gzip
 import hashlib
@@ -61,6 +60,7 @@ from .util import (
     sfsenc,
     spack,
     statdir,
+    ub64enc,
     unhumanize,
     vjoin,
     vsplit,
@@ -265,19 +265,29 @@ class Up2k(object):
             if not self.stop:
                 self.log("uploads are now possible", 2)
 
-    def get_state(self)  :
+    def get_state(self, get_q , uname )  :
         mtpq   = 0
+        ups = []
+        up_en = not self.args.no_up_list
         q = "select count(w) from mt where k = 't:mtp'"
         got_lock = False if PY2 else self.mutex.acquire(timeout=0.5)
         if got_lock:
-            for cur in self.cur.values():
-                try:
-                    mtpq += cur.execute(q).fetchone()[0]
-                except:
-                    pass
-            self.mutex.release()
+            try:
+                for cur in self.cur.values() if get_q else []:
+                    try:
+                        mtpq += cur.execute(q).fetchone()[0]
+                    except:
+                        pass
+                if uname and up_en:
+                    ups = self._active_uploads(uname)
+            finally:
+                self.mutex.release()
         else:
             mtpq = "(?)"
+            if up_en:
+                ups = [(0, 0, time.time(), "cannot show list (server too busy)")]
+
+        ups.sort(reverse=True)
 
         ret = {
             "volstate": self.volstate,
@@ -285,6 +295,7 @@ class Up2k(object):
             "hashq": self.n_hashq,
             "tagq": self.n_tagq,
             "mtpq": mtpq,
+            "ups": ups,
             "dbwu": "{:.2f}".format(self.db_act),
             "dbwt": "{:.2f}".format(
                 min(1000 * 24 * 60 * 60 - 1, time.time() - self.db_act)
@@ -292,6 +303,32 @@ class Up2k(object):
         }
         return json.dumps(ret, separators=(",\n", ": "))
 
+    def _active_uploads(self, uname )     :
+        ret = []
+        for vtop in self.asrv.vfs.aread[uname]:
+            vfs = self.asrv.vfs.all_vols.get(vtop)
+            if not vfs:  # dbv only
+                continue
+            ptop = vfs.realpath
+            tab = self.registry.get(ptop)
+            if not tab:
+                continue
+            for job in tab.values():
+                ineed = len(job["need"])
+                ihash = len(job["hash"])
+                if ineed == ihash or not ineed:
+                    continue
+
+                zt = (
+                    ineed / ihash,
+                    job["size"],
+                    int(job["t0"]),
+                    int(job["poke"]),
+                    djoin(vtop, job["prel"], job["name"]),
+                )
+                ret.append(zt)
+        return ret
+
     def find_job_by_ap(self, ptop , ap )  :
         try:
             if ANYWIN:
@@ -572,7 +609,6 @@ class Up2k(object):
         return timeout
 
     def _check_shares(self)  :
-        assert sqlite3  # type: ignore
 
         now = time.time()
         timeout = now + 9001
@@ -893,7 +929,6 @@ class Up2k(object):
             with self.mutex, self.reg_mutex:
                 reg = self.register_vpath(vol.realpath, vol.flags)
 
-            assert reg
             cur, _ = reg
             with self.mutex:
                 cur.connection.commit()
@@ -910,7 +945,6 @@ class Up2k(object):
                 reg = self.register_vpath(vol.realpath, vol.flags)
 
             try:
-                assert reg
                 cur, db_path = reg
                 if bos.path.getsize(db_path + "-wal") < 1024 * 1024 * 5:
                     continue
@@ -1116,7 +1150,7 @@ class Up2k(object):
         zsl = [x[len(prefix) :] for x in zsl]
         zsl.sort()
         zb = hashlib.sha1("\n".join(zsl).encode("utf-8", "replace")).digest()
-        vcfg = base64.urlsafe_b64encode(zb[:18]).decode("ascii")
+        vcfg = ub64enc(zb[:18]).decode("ascii")
 
         c = cur.execute("select v from kv where k = 'volcfg'")
         try:
@@ -1145,7 +1179,6 @@ class Up2k(object):
             with self.reg_mutex:
                 reg = self.register_vpath(top, vol.flags)
 
-            assert reg and self.pp
             cur, db_path = reg
 
             db = Dbw(cur, 0, time.time())
@@ -1164,6 +1197,10 @@ class Up2k(object):
                 # ~/.wine/dosdevices/z:/ and such
                 excl.extend(("/dev", "/proc", "/run", "/sys"))
 
+            if self.args.re_dirsz:
+                db.c.execute("delete from ds")
+                db.n += 1
+
             rtop = absreal(top)
             n_add = n_rm = 0
             try:
@@ -1172,7 +1209,7 @@ class Up2k(object):
                     self.log(t % (vol.vpath, rtop), 6)
                     return True, False
 
-                n_add = self._build_dir(
+                n_add, _, _ = self._build_dir(
                     db,
                     top,
                     set(excl),
@@ -1246,17 +1283,18 @@ class Up2k(object):
         cst ,
         dev ,
         xvol ,
-    )  :
+    )    :
         if xvol and not rcdir.startswith(top):
             self.log("skip xvol: [{}] -> [{}]".format(cdir, rcdir), 6)
-            return 0
+            return 0, 0, 0
 
         if rcdir in seen:
             t = "bailing from symlink loop,\n  prev: {}\n  curr: {}\n  from: {}"
             self.log(t.format(seen[-1], rcdir, cdir), 3)
-            return 0
+            return 0, 0, 0
 
-        ret = 0
+        # total-files-added, total-num-files, recursive-size
+        tfa = tnf = rsz = 0
         seen = seen + [rcdir]
         unreg  = []
         files    = []
@@ -1266,22 +1304,24 @@ class Up2k(object):
         th_cvd = self.args.th_coversd
         th_cvds = self.args.th_coversd_set
 
-        assert self.pp and self.mem_cur
         self.pp.msg = "a%d %s" % (self.pp.n, cdir)
 
         rd = cdir[len(top) :].strip("/")
         if WINDOWS:
             rd = rd.replace("\\", "/").strip("/")
 
+        rds = rd + "/" if rd else ""
+        cdirs = cdir + os.sep
+
         g = statdir(self.log_func, not self.args.no_scandir, True, cdir)
         gl = sorted(g)
         partials = set([x[0] for x in gl if "PARTIAL" in x[0]])
         for iname, inf in gl:
             if self.stop:
-                return -1
+                return -1, 0, 0
 
-            rp = vjoin(rd, iname)
-            abspath = os.path.join(cdir, iname)
+            rp = rds + iname
+            abspath = cdirs + iname
 
             if rei and rei.search(abspath):
                 unreg.append(rp)
@@ -1315,7 +1355,7 @@ class Up2k(object):
                     continue
                 # self.log(" dir: {}".format(abspath))
                 try:
-                    ret += self._build_dir(
+                    i1, i2, i3 = self._build_dir(
                         db,
                         top,
                         excl,
@@ -1330,6 +1370,9 @@ class Up2k(object):
                         dev,
                         xvol,
                     )
+                    tfa += i1
+                    tnf += i2
+                    rsz += i3
                 except:
                     t = "failed to index subdir [{}]:\n{}"
                     self.log(t.format(abspath, min_ex()), c=1)
@@ -1348,6 +1391,7 @@ class Up2k(object):
                     # placeholder for unfinished upload
                     continue
 
+                rsz += sz
                 files.append((sz, lmod, iname))
                 liname = iname.lower()
                 if (
@@ -1369,6 +1413,18 @@ class Up2k(object):
                 ):
                     cv = iname
 
+        if not self.args.no_dirsz:
+            tnf += len(files)
+            q = "select sz, nf from ds where rd=? limit 1"
+            try:
+                db_sz, db_nf = db.c.execute(q, (rd,)).fetchone() or (-1, -1)
+                if rsz != db_sz or tnf != db_nf:
+                    db.c.execute("delete from ds where rd=?", (rd,))
+                    db.c.execute("insert into ds values (?,?,?)", (rd, rsz, tnf))
+                    db.n += 1
+            except:
+                pass  # mojibake rd
+
         # folder of 1000 files = ~1 MiB RAM best-case (tiny filenames);
         # free up stuff we're done with before dhashing
         gl = []
@@ -1382,7 +1438,7 @@ class Up2k(object):
 
             zh.update(cv.encode("utf-8", "replace"))
             zh.update(spack(b"<d", cst.st_mtime))
-            dhash = base64.urlsafe_b64encode(zh.digest()[:12]).decode("ascii")
+            dhash = ub64enc(zh.digest()[:12]).decode("ascii")
             sql = "select d from dh where d=? and +h=?"
             try:
                 c = db.c.execute(sql, (rd, dhash))
@@ -1392,7 +1448,7 @@ class Up2k(object):
                 c = db.c.execute(sql, (drd, dhash))
 
             if c.fetchone():
-                return ret
+                return tfa, tnf, rsz
 
         if cv and rd:
             # mojibake not supported (for performance / simplicity):
@@ -1409,10 +1465,10 @@ class Up2k(object):
         seen_files = set([x[2] for x in files])  # for dropcheck
         for sz, lmod, fn in files:
             if self.stop:
-                return -1
+                return -1, 0, 0
 
-            rp = vjoin(rd, fn)
-            abspath = os.path.join(cdir, fn)
+            rp = rds + fn
+            abspath = cdirs + fn
             nohash = reh.search(abspath) if reh else False
 
             sql = "select w, mt, sz, ip, at from up where rd = ? and fn = ?"
@@ -1442,7 +1498,7 @@ class Up2k(object):
                 )
                 self.log(t)
                 self.db_rm(db.c, rd, fn, 0)
-                ret += 1
+                tfa += 1
                 db.n += 1
                 in_db = []
             else:
@@ -1467,7 +1523,7 @@ class Up2k(object):
                     continue
 
                 if not hashes:
-                    return -1
+                    return -1, 0, 0
 
                 wark = up2k_wark_from_hashlist(self.salt, sz, hashes)
 
@@ -1478,7 +1534,7 @@ class Up2k(object):
             # skip upload hooks by not providing vflags
             self.db_add(db.c, {}, rd, fn, lmod, sz, "", "", wark, "", "", ip, at)
             db.n += 1
-            ret += 1
+            tfa += 1
             td = time.time() - db.t
             if db.n >= 4096 or td >= 60:
                 self.log("commit {} new files".format(db.n))
@@ -1491,33 +1547,37 @@ class Up2k(object):
             db.c.execute("insert into dh values (?,?)", (drd, dhash))  # type: ignore
 
         if self.stop:
-            return -1
+            return -1, 0, 0
 
         # drop shadowed folders
         for sh_rd in unreg:
             n = 0
-            q = "select count(w) from up where (rd = ? or rd like ?||'%') and at == 0"
+            q = "select count(w) from up where (rd=? or rd like ?||'/%') and +at == 0"
             for sh_erd in [sh_rd, "//" + w8b64enc(sh_rd)]:
                 try:
-                    n = db.c.execute(q, (sh_erd, sh_erd + "/")).fetchone()[0]
+                    erd_erd = (sh_erd, sh_erd)
+                    n = db.c.execute(q, erd_erd).fetchone()[0]
                     break
                 except:
                     pass
 
+
             if n:
                 t = "forgetting {} shadowed autoindexed files in [{}] > [{}]"
                 self.log(t.format(n, top, sh_rd))
-                assert sh_erd  # type: ignore
 
-                q = "delete from dh where (d = ? or d like ?||'%')"
-                db.c.execute(q, (sh_erd, sh_erd + "/"))
+                q = "delete from dh where (d = ? or d like ?||'/%')"
+                db.c.execute(q, erd_erd)
+
+                q = "delete from up where (rd=? or rd like ?||'/%') and +at == 0"
+                db.c.execute(q, erd_erd)
+                tfa += n
 
-                q = "delete from up where (rd = ? or rd like ?||'%') and at == 0"
-                db.c.execute(q, (sh_erd, sh_erd + "/"))
-                ret += n
+            q = "delete from ds where (rd=? or rd like ?||'/%')"
+            db.c.execute(q, erd_erd)
 
         if n4g:
-            return ret
+            return tfa, tnf, rsz
 
         # drop missing files
         q = "select fn from up where rd = ?"
@@ -1535,7 +1595,7 @@ class Up2k(object):
         if n_rm:
             self.log("forgot {} deleted files".format(n_rm))
 
-        return ret
+        return tfa, tnf, rsz
 
     def _drop_lost(self, cur , top , excl )  :
         rm = []
@@ -1610,7 +1670,7 @@ class Up2k(object):
 
         # then covers
         n_rm3 = 0
-        qu = "select 1 from up where rd=? and +fn=? limit 1"
+        qu = "select 1 from up where rd=? and fn=? limit 1"
         q = "delete from cv where rd=? and dn=? and +fn=?"
         for crd, cdn, fn in cur.execute("select * from cv"):
             urd = vjoin(crd, cdn)
@@ -1753,13 +1813,13 @@ class Up2k(object):
             return 0
 
         with self.mutex:
+            q = "update up set w=?, sz=?, mt=? where rd=? and fn=?"
             for rd, fn, w, sz, mt in rewark:
-                q = "update up set w = ?, sz = ?, mt = ? where rd = ? and fn = ? limit 1"
                 cur.execute(q, (w, sz, int(mt), rd, fn))
 
-            for _, _, w in f404:
-                q = "delete from up where w = ? limit 1"
-                cur.execute(q, (w,))
+            if f404:
+                q = "delete from up where rd=? and fn=? and +w=?"
+                cur.executemany(q, f404)
 
             cur.connection.commit()
 
@@ -2226,7 +2286,6 @@ class Up2k(object):
         # mp.pool.ThreadPool and concurrent.futures.ThreadPoolExecutor
         # both do crazy runahead so lets reinvent another wheel
         nw = max(1, self.args.mtag_mt)
-        assert self.mtag
         if not self.mpool_used:
             self.mpool_used = True
             self.log("using {}x {}".format(nw, self.mtag.backend))
@@ -2300,7 +2359,6 @@ class Up2k(object):
         at ,
     )  :
         """will mutex(main)"""
-        assert self.mtag
 
         try:
             st = bos.stat(abspath)
@@ -2332,7 +2390,6 @@ class Up2k(object):
         tags   ,
     )  :
         """mutex(main) me"""
-        assert self.mtag
 
         if not bos.path.isfile(abspath):
             return 0
@@ -2388,7 +2445,7 @@ class Up2k(object):
     def _log_sqlite_incompat(self, db_path, t0)  :
         txt = t0 or ""
         digest = hashlib.sha512(db_path.encode("utf-8", "replace")).digest()
-        stackname = base64.urlsafe_b64encode(digest[:9]).decode("utf-8")
+        stackname = ub64enc(digest[:9]).decode("ascii")
         stackpath = os.path.join(E.cfg, "stack-%s.txt" % (stackname,))
 
         t = "  the filesystem at %s may not support locking, or is otherwise incompatible with sqlite\n\n  %s\n\n"
@@ -2431,12 +2488,11 @@ class Up2k(object):
                 self.log("WARN: failed to upgrade from v4", 3)
 
         if ver == DB_VER:
-            try:
-                self._add_cv_tab(cur)
-                self._add_xiu_tab(cur)
-                self._add_dhash_tab(cur)
-            except:
-                pass
+            self._add_dhash_tab(cur)
+            self._add_xiu_tab(cur)
+            self._add_cv_tab(cur)
+            self._add_idx_up_vp(cur, db_path)
+            self._add_ds_tab(cur)
 
             try:
                 nfiles = next(cur.execute("select count(w) from up"))[0]
@@ -2533,9 +2589,10 @@ class Up2k(object):
 
         for cmd in [
             r"create table up (w text, mt int, sz int, rd text, fn text, ip text, at int)",
-            r"create index up_rd on up(rd)",
+            r"create index up_vp on up(rd, fn)",
             r"create index up_fn on up(fn)",
             r"create index up_ip on up(ip)",
+            r"create index up_at on up(at)",
             idx,
             r"create table mt (w text, k text, v int)",
             r"create index mt_w on mt(w)",
@@ -2549,6 +2606,7 @@ class Up2k(object):
         self._add_dhash_tab(cur)
         self._add_xiu_tab(cur)
         self._add_cv_tab(cur)
+        self._add_ds_tab(cur)
         self.log("created DB at {}".format(db_path))
         return cur
 
@@ -2565,6 +2623,12 @@ class Up2k(object):
 
     def _add_dhash_tab(self, cur )  :
         # v5 -> v5a
+        try:
+            cur.execute("select d, h from dh limit 1").fetchone()
+            return
+        except:
+            pass
+
         for cmd in [
             r"create table dh (d text, h text)",
             r"create index dh_d on dh(d)",
@@ -2618,6 +2682,40 @@ class Up2k(object):
 
         cur.connection.commit()
 
+    def _add_idx_up_vp(self, cur , db_path )  :
+        # v5c -> v5d
+        try:
+            cur.execute("drop index up_rd")
+        except:
+            return
+
+        for cmd in [
+            r"create index up_vp on up(rd, fn)",
+            r"create index up_at on up(at)",
+        ]:
+            self.log("upgrading db [%s]: %s" % (db_path, cmd[:18]))
+            cur.execute(cmd)
+
+        self.log("upgrading db [%s]: writing to disk..." % (db_path,))
+        cur.connection.commit()
+        cur.execute("vacuum")
+
+    def _add_ds_tab(self, cur )  :
+        # v5d -> v5e
+        try:
+            cur.execute("select rd, sz from ds limit 1").fetchone()
+            return
+        except:
+            pass
+
+        for cmd in [
+            r"create table ds (rd text, sz int, nf int)",
+            r"create index ds_rd on ds(rd)",
+        ]:
+            cur.execute(cmd)
+
+        cur.connection.commit()
+
     def wake_rescanner(self):
         with self.rescan_cond:
             self.rescan_cond.notify_all()
@@ -2802,7 +2900,6 @@ class Up2k(object):
 
                     c2 = cur
 
-                assert c2
                 c2.connection.commit()
 
             cur = jcur
@@ -2843,7 +2940,7 @@ class Up2k(object):
                     self.log(t)
                     del reg[wark]
 
-                elif inc_ap != orig_ap and not data_ok:
+                elif inc_ap != orig_ap and not data_ok and "done" in reg[wark]:
                     self.log("asserting contents of %s" % (orig_ap,))
                     dhashes, _ = self._hashlist_from_file(orig_ap)
                     dwark = up2k_wark_from_hashlist(self.salt, st.st_size, dhashes)
@@ -2907,9 +3004,12 @@ class Up2k(object):
                         job = deepcopy(job)
                         job["wark"] = wark
                         job["at"] = cj.get("at") or time.time()
-                        zs = "lmod ptop vtop prel name host user addr poke"
+                        zs = "vtop ptop prel name lmod host user addr poke"
                         for k in zs.split():
                             job[k] = cj.get(k) or ""
+                        for k in ("life", "replace"):
+                            if k in cj:
+                                job[k] = cj[k]
 
                         pdir = djoin(cj["ptop"], cj["prel"])
                         if rand:
@@ -3010,18 +3110,8 @@ class Up2k(object):
                     "busy": {},
                 }
                 # client-provided, sanitized by _get_wark: name, size, lmod
-                for k in [
-                    "host",
-                    "user",
-                    "addr",
-                    "vtop",
-                    "ptop",
-                    "prel",
-                    "name",
-                    "size",
-                    "lmod",
-                    "poke",
-                ]:
+                zs = "vtop ptop prel name size lmod host user addr poke"
+                for k in zs.split():
                     job[k] = cj[k]
 
                 for k in ["life", "replace"]:
@@ -3104,7 +3194,22 @@ class Up2k(object):
         fp = djoin(fdir, fname)
         if job.get("replace") and bos.path.exists(fp):
             self.log("replacing existing file at {}".format(fp))
-            wunlink(self.log, fp, self.flags.get(job["ptop"]) or {})
+            cur = None
+            ptop = job["ptop"]
+            vf = self.flags.get(ptop) or {}
+            st = bos.stat(fp)
+            try:
+                vrel = vjoin(job["prel"], fname)
+                xlink = bool(vf.get("xlink"))
+                cur, wark, _, _, _, _ = self._find_from_vpath(ptop, vrel)
+                self._forget_file(ptop, vrel, cur, wark, True, st.st_size, xlink)
+            except Exception as ex:
+                self.log("skipping replace-relink: %r" % (ex,))
+            finally:
+                if cur:
+                    cur.connection.commit()
+
+            wunlink(self.log, fp, vf)
 
         if self.args.plain_ip:
             dip = ip.replace(":", ".")
@@ -3112,8 +3217,9 @@ class Up2k(object):
             dip = self.hub.iphash.s(ip)
 
         suffix = "-%.6f-%s" % (ts, dip)
-        with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw:
-            return zfw["orz"][1]
+        f, ret = ren_open(fname, "wb", fdir=fdir, suffix=suffix)
+        f.close()
+        return ret
 
     def _symlink(
         self,
@@ -3260,7 +3366,6 @@ class Up2k(object):
                     t = "that chunk is already being written to:\n  {}\n  {} {}/{}\n  {}"
                     raise Pebkac(400, t.format(wark, chash, idx, nh, job["name"]))
 
-            assert chash  # type: ignore
             chunksize = up2k_chunksize(job["size"])
 
             coffsets = []
@@ -3295,19 +3400,30 @@ class Up2k(object):
 
         return chashes, chunksize, coffsets, path, job["lmod"], job["sprs"]
 
-    def release_chunks(self, ptop , wark , chashes )  :
-        with self.reg_mutex:
-            job = self.registry[ptop].get(wark)
-            if job:
-                for chash in chashes:
-                    job["busy"].pop(chash, None)
-
-        return True
+    def fast_confirm_chunks(
+        self, ptop , wark , chashes 
+    )   :
+        if not self.mutex.acquire(False):
+            return -1, ""
+        if not self.reg_mutex.acquire(False):
+            self.mutex.release()
+            return -1, ""
+        try:
+            return self._confirm_chunks(ptop, wark, chashes)
+        finally:
+            self.reg_mutex.release()
+            self.mutex.release()
 
     def confirm_chunks(
         self, ptop , wark , chashes 
     )   :
         with self.mutex, self.reg_mutex:
+            return self._confirm_chunks(ptop, wark, chashes)
+
+    def _confirm_chunks(
+        self, ptop , wark , chashes 
+    )   :
+        if True:
             self.db_act = self.vol_act[ptop] = time.time()
             try:
                 job = self.registry[ptop][wark]
@@ -3315,7 +3431,7 @@ class Up2k(object):
                 src = djoin(pdir, job["tnam"])
                 dst = djoin(pdir, job["name"])
             except Exception as ex:
-                return "confirm_chunk, wark(%r)" % (ex,)  # type: ignore
+                return -2, "confirm_chunk, wark(%r)" % (ex,)  # type: ignore
 
             for chash in chashes:
                 job["busy"].pop(chash, None)
@@ -3324,7 +3440,7 @@ class Up2k(object):
                 for chash in chashes:
                     job["need"].remove(chash)
             except Exception as ex:
-                return "confirm_chunk, chash(%s) %r" % (chash, ex)  # type: ignore
+                return -2, "confirm_chunk, chash(%s) %r" % (chash, ex)  # type: ignore
 
             ret = len(job["need"])
             if ret > 0:
@@ -3473,7 +3589,6 @@ class Up2k(object):
             cur.connection.commit()
         except Exception as ex:
             x = self.register_vpath(ptop, {})
-            assert x
             db_ex_chk(self.log, ex, x[1])
             raise
 
@@ -3488,7 +3603,6 @@ class Up2k(object):
         try:
             r = db.execute(sql, (rd, fn))
         except:
-            assert self.mem_cur
             r = db.execute(sql, s3enc(self.mem_cur, rd, fn))
 
         if r.rowcount:
@@ -3526,7 +3640,6 @@ class Up2k(object):
         try:
             db.execute(sql, v)
         except:
-            assert self.mem_cur
             rd, fn = s3enc(self.mem_cur, rd, fn)
             v = (wark, int(ts), sz, rd, fn, db_ip, int(at or 0))
             db.execute(sql, v)
@@ -3574,7 +3687,6 @@ class Up2k(object):
                 try:
                     db.execute(q, (cd, wark[:16], rd, fn))
                 except:
-                    assert self.mem_cur
                     rd, fn = s3enc(self.mem_cur, rd, fn)
                     db.execute(q, (cd, wark[:16], rd, fn))
 
@@ -3607,6 +3719,19 @@ class Up2k(object):
                 except:
                     pass
 
+        if "nodirsz" not in vflags:
+            try:
+                q = "update ds set nf=nf+1, sz=sz+? where rd=?"
+                q2 = "insert into ds values(?,?,1)"
+                while True:
+                    if not db.execute(q, (sz, rd)).rowcount:
+                        db.execute(q2, (rd, sz))
+                    if not rd:
+                        break
+                    rd = rd.rsplit("/", 1)[0] if "/" in rd else ""
+            except:
+                pass
+
     def handle_rm(
         self,
         uname ,
@@ -3991,7 +4116,6 @@ class Up2k(object):
 
         has_dupes = False
         if w:
-            assert c1
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
@@ -4129,7 +4253,6 @@ class Up2k(object):
         try:
             c = cur.execute(q, (rd, fn))
         except:
-            assert self.mem_cur
             c = cur.execute(q, s3enc(self.mem_cur, rd, fn))
 
         hit = c.fetchone()
@@ -4372,8 +4495,7 @@ class Up2k(object):
                     rem -= len(buf)
 
                 digest = hashobj.digest()[:33]
-                digest = base64.urlsafe_b64encode(digest)
-                ret.append(digest.decode("utf-8"))
+                ret.append(ub64enc(digest).decode("ascii"))
 
         return ret, st
 
@@ -4445,8 +4567,8 @@ class Up2k(object):
             dip = self.hub.iphash.s(job["addr"])
 
         suffix = "-%.6f-%s" % (job["t0"], dip)
-        with ren_open(tnam, "wb", fdir=pdir, suffix=suffix) as zfw:
-            f, job["tnam"] = zfw["orz"]
+        f, job["tnam"] = ren_open(tnam, "wb", fdir=pdir, suffix=suffix)
+        try:
             abspath = djoin(pdir, job["tnam"])
             sprs = job["sprs"]
             sz = job["size"]
@@ -4493,6 +4615,8 @@ class Up2k(object):
             if job["hash"] and sprs:
                 f.seek(sz - 1)
                 f.write(b"e")
+        finally:
+            f.close()
 
         if not job["hash"]:
             self._finish_upload(job["ptop"], job["wark"])
@@ -4835,11 +4959,10 @@ def up2k_wark_from_hashlist(salt , filesize , hashes )  :
     vstr = "\n".join(values)
 
     wark = hashlib.sha512(vstr.encode("utf-8")).digest()[:33]
-    wark = base64.urlsafe_b64encode(wark)
-    return wark.decode("ascii")
+    return ub64enc(wark).decode("ascii")
 
 
 def up2k_wark_from_metadata(salt , sz , lastmod , rd , fn )  :
     ret = sfsenc("%s\n%d\n%d\n%s\n%s" % (salt, lastmod, sz, rd, fn))
-    ret = base64.urlsafe_b64encode(hashlib.sha512(ret).digest())
+    ret = ub64enc(hashlib.sha512(ret).digest())
     return ("#%s" % (ret.decode("ascii"),))[:44]