copyparty 1.7.6__py3-none-any.whl → 1.8.1__py3-none-any.whl

copyparty/httpcli.py

@@ -58,6 +58,7 @@ from .util import (
     html_escape,
     humansize,
     ipnorm,
+    loadpy,
     min_ex,
     quotep,
     rand_name,
@@ -169,13 +170,16 @@ class HttpCli(object):
     def log(self, msg , c   = 0)  :
         ptn = self.asrv.re_pwd
         if ptn and ptn.search(msg):
-            msg = ptn.sub(self.unpwd, msg)
+            if self.asrv.ah.on:
+                msg = ptn.sub("\033[7m pw \033[27m", msg)
+            else:
+                msg = ptn.sub(self.unpwd, msg)
 
         self.log_func(self.log_src, msg, c)
 
     def unpwd(self, m )  :
-        a, b = m.groups()
-        return "=\033[7m {} \033[27m{}".format(self.asrv.iacct[a], b)
+        a, b, c = m.groups()
+        return "{}\033[7m {} \033[27m{}".format(a, self.asrv.iacct[b], c)
 
     def _check_nonfatal(self, ex , post )  :
         if post:
@@ -379,13 +383,14 @@ class HttpCli(object):
                 zs = base64.b64decode(zb).decode("utf-8")
                 # try "pwd", "x:pwd", "pwd:x"
                 for bauth in [zs] + zs.split(":", 1)[::-1]:
-                    if self.asrv.iacct.get(bauth):
+                    hpw = self.asrv.ah.hash(bauth)
+                    if self.asrv.iacct.get(hpw):
                         break
             except:
                 pass
 
         self.pw = uparam.get("pw") or self.headers.get("pw") or bauth or cookie_pw
-        self.uname = self.asrv.iacct.get(self.pw) or "*"
+        self.uname = self.asrv.iacct.get(self.asrv.ah.hash(self.pw)) or "*"
         self.rvol = self.asrv.vfs.aread[self.uname]
         self.wvol = self.asrv.vfs.awrite[self.uname]
         self.mvol = self.asrv.vfs.amove[self.uname]
@@ -759,11 +764,27 @@ class HttpCli(object):
             return True
 
         if not self.can_read and not self.can_write and not self.can_get:
-            if self.vpath:
-                self.log("inaccessible: [{}]".format(self.vpath))
-                return self.tx_404(True)
+            t = "@{} has no access to [{}]"
+            self.log(t.format(self.uname, self.vpath))
+
+            if self.avn and "on403" in self.avn.flags:
+                vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False)
+                ret = self.on40x(vn.flags["on403"], vn, rem)
+                if ret == "true":
+                    return True
+                elif ret == "false":
+                    return False
+                elif ret == "allow":
+                    self.log("plugin override; access permitted")
+                    self.can_read = self.can_write = self.can_move = True
+                    self.can_delete = self.can_get = self.can_upget = True
+                else:
+                    return self.tx_404(True)
+            else:
+                if self.vpath:
+                    return self.tx_404(True)
 
-            self.uparam["h"] = ""
+                self.uparam["h"] = ""
 
         if "tree" in self.uparam:
             return self.tx_tree()
@@ -1354,7 +1375,9 @@ class HttpCli(object):
         lim = vfs.get_dbv(rem)[0].lim
         fdir = vfs.canonical(rem)
         if lim:
-            fdir, rem = lim.all(self.ip, rem, remains, fdir)
+            fdir, rem = lim.all(
+                self.ip, rem, remains, vfs.realpath, fdir, self.conn.hsrv.broker
+            )
 
         fn = None
         if rem and not self.trailing_slash and not bos.path.isdir(fdir):
@@ -1487,6 +1510,7 @@ class HttpCli(object):
             lim.bup(self.ip, post_sz)
             try:
                 lim.chk_sz(post_sz)
+                lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, post_sz)
             except:
                 bos.unlink(path)
                 raise
@@ -1961,7 +1985,7 @@ class HttpCli(object):
         return True
 
     def get_pwd_cookie(self, pwd )  :
-        if pwd in self.asrv.iacct:
+        if self.asrv.ah.hash(pwd) in self.asrv.iacct:
             msg = "login ok"
             dur = int(60 * 60 * self.args.logout)
         else:
@@ -2097,7 +2121,9 @@ class HttpCli(object):
         lim = vfs.get_dbv(rem)[0].lim
         fdir_base = vfs.canonical(rem)
         if lim:
-            fdir_base, rem = lim.all(self.ip, rem, -1, fdir_base)
+            fdir_base, rem = lim.all(
+                self.ip, rem, -1, vfs.realpath, fdir_base, self.conn.hsrv.broker
+            )
             upload_vpath = "{}/{}".format(vfs.vpath, rem).strip("/")
             if not nullwrite:
                 bos.makedirs(fdir_base)
@@ -2190,6 +2216,7 @@ class HttpCli(object):
                         try:
                             lim.chk_df(tabspath, sz, True)
                             lim.chk_sz(sz)
+                            lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
                             lim.chk_bup(self.ip)
                             lim.chk_nup(self.ip)
                         except:
@@ -2365,7 +2392,7 @@ class HttpCli(object):
         fp = vfs.canonical(rp)
         lim = vfs.get_dbv(rem)[0].lim
         if lim:
-            fp, rp = lim.all(self.ip, rp, clen, fp)
+            fp, rp = lim.all(self.ip, rp, clen, vfs.realpath, fp, self.conn.hsrv.broker)
             bos.makedirs(fp)
 
         fp = os.path.join(fp, fn)
@@ -2436,6 +2463,25 @@ class HttpCli(object):
         if p_field != "body":
             raise Pebkac(400, "expected body, got {}".format(p_field))
 
+        xbu = vfs.flags.get("xbu")
+        if xbu:
+            if not runhook(
+                self.log,
+                xbu,
+                fp,
+                self.vpath,
+                self.host,
+                self.uname,
+                time.time(),
+                0,
+                self.ip,
+                time.time(),
+                "",
+            ):
+                t = "save blocked by xbu server config"
+                self.log(t, 1)
+                raise Pebkac(403, t)
+
         if bos.path.exists(fp):
             bos.unlink(fp)
 
@@ -2447,6 +2493,7 @@ class HttpCli(object):
             lim.bup(self.ip, sz)
             try:
                 lim.chk_sz(sz)
+                lim.chk_vsz(self.conn.hsrv.broker, vfs.realpath, sz)
             except:
                 bos.unlink(fp)
                 raise
@@ -2455,6 +2502,39 @@ class HttpCli(object):
         new_lastmod3 = int(new_lastmod * 1000)
         sha512 = sha512[:56]
 
+        xau = vfs.flags.get("xau")
+        if xau and not runhook(
+            self.log,
+            xau,
+            fp,
+            self.vpath,
+            self.host,
+            self.uname,
+            new_lastmod,
+            sz,
+            self.ip,
+            new_lastmod,
+            "",
+        ):
+            t = "save blocked by xau server config"
+            self.log(t, 1)
+            os.unlink(fp)
+            raise Pebkac(403, t)
+
+        vfs, rem = vfs.get_dbv(rem)
+        self.conn.hsrv.broker.say(
+            "up2k.hash_file",
+            vfs.realpath,
+            vfs.vpath,
+            vfs.flags,
+            vsplit(rem)[0],
+            fn,
+            self.ip,
+            new_lastmod,
+            self.uname,
+            True,
+        )
+
         response = json.dumps(
             {"ok": True, "lastmod": new_lastmod3, "size": sz, "sha512": sha512}
         )
@@ -2989,6 +3069,20 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"), status=rc)
         return True
 
+    def on40x(self, mods , vn , rem )  :
+        for mpath in mods:
+            try:
+                mod = loadpy(mpath, self.args.hot_handlers)
+            except Exception as ex:
+                self.log("import failed: {!r}".format(ex))
+                continue
+
+            ret = mod.main(self, vn, rem)
+            if ret:
+                return ret.lower()
+
+        return ""  # unhandled / fallthrough
+
     def scanvol(self)  :
         if not self.can_read or not self.can_write:
             raise Pebkac(403, "not allowed for user " + self.uname)
@@ -3306,7 +3400,21 @@ class HttpCli(object):
         try:
             st = bos.stat(abspath)
         except:
-            return self.tx_404()
+            if "on404" not in vn.flags:
+                return self.tx_404()
+
+            ret = self.on40x(vn.flags["on404"], vn, rem)
+            if ret == "true":
+                return True
+            elif ret == "false":
+                return False
+            elif ret == "retry":
+                try:
+                    st = bos.stat(abspath)
+                except:
+                    return self.tx_404()
+            else:
+                return self.tx_404()
 
         if rem.startswith(".hist/up2k.") or (
             rem.endswith("/dir.txt") and rem.startswith(".hist/th/")

copyparty/ico.py

@@ -17,7 +17,9 @@ class Ico(object):
     def get(self, ext , as_thumb , chrome )   :
         """placeholder to make thumbnails not break"""
 
-        zb = hashlib.sha1(ext.encode("utf-8")).digest()[2:4]
+        bext = ext.encode("ascii", "replace")
+        ext = bext.decode("utf-8")
+        zb = hashlib.sha1(bext).digest()[2:4]
         if PY2:
             zb = [ord(x) for x in zb]
 
@@ -33,7 +35,7 @@ class Ico(object):
             h = int(100 / (float(sw) / float(sh)))
             w = 100
 
-        if chrome and as_thumb:
+        if chrome:
             # cannot handle more than ~2000 unique SVGs
             if HAVE_PIL:
                 # svg: 3s, cache: 6s, this: 8s
@@ -43,8 +45,19 @@ class Ico(object):
                 w = 64
                 img = Image.new("RGB", (w, h), "#" + c[:6])
                 pb = ImageDraw.Draw(img)
-                tw, th = pb.textsize(ext)
-                pb.text(((w - tw) // 2, (h - th) // 2), ext, fill="#" + c[6:])
+                try:
+                    _, _, tw, th = pb.textbbox((0, 0), ext)
+                except:
+                    tw, th = pb.textsize(ext)
+
+                tw += len(ext)
+                cw = tw // len(ext)
+                x = ((w - tw) // 2) - (cw * 2) // 3
+                fill = "#" + c[6:]
+                for ch in ext:
+                    pb.text((x, (h - th) // 2), " %s " % (ch,), fill=fill)
+                    x += cw
+
                 img = img.resize((w * 3, h * 3), Image.NEAREST)
 
                 buf = BytesIO()

copyparty/pwhash.py

@@ -0,0 +1,145 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import argparse
+import base64
+import hashlib
+import sys
+import threading
+
+from .__init__ import unicode
+
+
+class PWHash(object):
+    def __init__(self, args ):
+        self.args = args
+
+        try:
+            alg, ac = args.ah_alg.split(",")
+        except:
+            alg = args.ah_alg
+            ac = {}
+
+        if alg == "none":
+            alg = ""
+
+        self.alg = alg
+        self.ac = ac
+        if not alg:
+            self.on = False
+            self.hash = unicode
+            return
+
+        self.on = True
+        self.salt = args.ah_salt.encode("utf-8")
+        self.cache   = {}
+        self.mutex = threading.Lock()
+        self.hash = self._cache_hash
+
+        if alg == "sha2":
+            self._hash = self._gen_sha2
+        elif alg == "scrypt":
+            self._hash = self._gen_scrypt
+        elif alg == "argon2":
+            self._hash = self._gen_argon2
+        else:
+            t = "unsupported password hashing algorithm [{}], must be one of these: argon2 scrypt sha2 none"
+            raise Exception(t.format(alg))
+
+    def _cache_hash(self, plain )  :
+        with self.mutex:
+            try:
+                return self.cache[plain]
+            except:
+                pass
+
+            if not plain:
+                return ""
+
+            if len(plain) > 255:
+                raise Exception("password too long")
+
+            if len(self.cache) > 9000:
+                self.cache = {}
+
+            ret = self._hash(plain)
+            self.cache[plain] = ret
+            return ret
+
+    def _gen_sha2(self, plain )  :
+        its = int(self.ac[0]) if self.ac else 424242
+        bplain = plain.encode("utf-8")
+        ret = b"\n"
+        for _ in range(its):
+            ret = hashlib.sha512(self.salt + bplain + ret).digest()
+
+        return "+" + base64.urlsafe_b64encode(ret[:24]).decode("utf-8")
+
+    def _gen_scrypt(self, plain )  :
+        cost = 2 << 13
+        its = 2
+        blksz = 8
+        para = 4
+        try:
+            cost = 2 << int(self.ac[0])
+            its = int(self.ac[1])
+            blksz = int(self.ac[2])
+            para = int(self.ac[3])
+        except:
+            pass
+
+        ret = plain.encode("utf-8")
+        for _ in range(its):
+            ret = hashlib.scrypt(ret, salt=self.salt, n=cost, r=blksz, p=para, dklen=24)
+
+        return "+" + base64.urlsafe_b64encode(ret).decode("utf-8")
+
+    def _gen_argon2(self, plain )  :
+        from argon2.low_level import Type as ArgonType
+        from argon2.low_level import hash_secret
+
+        time_cost = 3
+        mem_cost = 256
+        parallelism = 4
+        version = 19
+        try:
+            time_cost = int(self.ac[0])
+            mem_cost = int(self.ac[1])
+            parallelism = int(self.ac[2])
+            version = int(self.ac[3])
+        except:
+            pass
+
+        bplain = plain.encode("utf-8")
+
+        bret = hash_secret(
+            secret=bplain,
+            salt=self.salt,
+            time_cost=time_cost,
+            memory_cost=mem_cost * 1024,
+            parallelism=parallelism,
+            hash_len=24,
+            type=ArgonType.ID,
+            version=version,
+        )
+        ret = bret.split(b"$")[-1].decode("utf-8")
+        return "+" + ret.replace("/", "_").replace("+", "-")
+
+    def stdin(self)  :
+        while True:
+            ln = sys.stdin.readline().strip()
+            if not ln:
+                break
+            print(self.hash(ln))
+
+    def cli(self)  :
+        import getpass
+
+        while True:
+            p1 = getpass.getpass("password> ")
+            p2 = getpass.getpass("again or just hit ENTER> ")
+            if p2 and p1 != p2:
+                print("\033[31minputs don't match; try again\033[0m", file=sys.stderr)
+                continue
+            print(self.hash(p1))
+            print()

copyparty/u2idx.py

@@ -66,7 +66,7 @@ class U2idx(object):
 
         fsize = body["size"]
         fhash = body["hash"]
-        wark = up2k_wark_from_hashlist(self.args.salt, fsize, fhash)
+        wark = up2k_wark_from_hashlist(self.args.warksalt, fsize, fhash)
 
         uq = "substr(w,1,16) = ? and w = ?"
         uv   = [wark[:16], wark]

copyparty/up2k.py

@@ -41,6 +41,7 @@ from .util import (
     gen_filekey,
     gen_filekey_dbg,
     hidedir,
+    humansize,
     min_ex,
     quotep,
     rand_name,
@@ -56,6 +57,7 @@ from .util import (
     sfsenc,
     spack,
     statdir,
+    unhumanize,
     vjoin,
     vsplit,
     w8b64dec,
@@ -107,7 +109,7 @@ class Up2k(object):
         self.args = hub.args
         self.log_func = hub.log
 
-        self.salt = self.args.salt
+        self.salt = self.args.warksalt
         self.r_hash = re.compile("^[0-9a-zA-Z_-]{44}$")
 
         self.gid = 0
@@ -122,6 +124,8 @@ class Up2k(object):
         self.registry     = {}
         self.flags    = {}
         self.droppable   = {}
+        self.volnfiles   = {}
+        self.volsize   = {}
         self.volstate   = {}
         self.vol_act   = {}
         self.busy_aps  = set()
@@ -258,6 +262,20 @@ class Up2k(object):
         }
         return json.dumps(ret, indent=4)
 
+    def get_volsize(self, ptop )   :
+        with self.mutex:
+            return self._get_volsize(ptop)
+
+    def _get_volsize(self, ptop )   :
+        cur = self.cur[ptop]
+        nbytes = self.volsize[cur]
+        nfiles = self.volnfiles[cur]
+        for j in list(self.registry.get(ptop, {}).values()):
+            nbytes += j["size"]
+            nfiles += 1
+
+        return (nbytes, nfiles)
+
     def rescan(
         self, all_vols  , scan_vols , wait , fscan 
     )  :
@@ -807,6 +825,8 @@ class Up2k(object):
         try:
             cur = self._open_db(db_path)
             self.cur[ptop] = cur
+            self.volsize[cur] = 0
+            self.volnfiles[cur] = 0
 
             # speeds measured uploading 520 small files on a WD20SPZX (SMR 2.5" 5400rpm 4kb)
             dbd = flags["dbd"]
@@ -914,6 +934,24 @@ class Up2k(object):
 
             db.c.connection.commit()
 
+            if vol.flags.get("vmaxb") or vol.flags.get("vmaxn"):
+                zs = "select count(sz), sum(sz) from up"
+                vn, vb = db.c.execute(zs).fetchone()
+                vb = vb or 0
+                vb += vn * 2048
+                self.volsize[db.c] = vb
+                self.volnfiles[db.c] = vn
+                vmaxb = unhumanize(vol.flags.get("vmaxb") or "0")
+                vmaxn = unhumanize(vol.flags.get("vmaxn") or "0")
+                t = "{} / {}  ( {} / {} files) in {}".format(
+                    humansize(vb, True),
+                    humansize(vmaxb, True),
+                    humansize(vn, True).rstrip("B"),
+                    humansize(vmaxn, True).rstrip("B"),
+                    vol.realpath,
+                )
+                self.log(t)
+
             return True, bool(n_add or n_rm or do_vac)
 
     def _build_dir(
@@ -1089,7 +1127,7 @@ class Up2k(object):
                         top, rp, dts, lmod, dsz, sz
                     )
                     self.log(t)
-                    self.db_rm(db.c, rd, fn)
+                    self.db_rm(db.c, rd, fn, 0)
                     ret += 1
                     db.n += 1
                     in_db = []
@@ -1172,7 +1210,7 @@ class Up2k(object):
         rm_files = [x for x in hits if x not in seen_files]
         n_rm = len(rm_files)
         for fn in rm_files:
-            self.db_rm(db.c, rd, fn)
+            self.db_rm(db.c, rd, fn, 0)
 
         if n_rm:
             self.log("forgot {} deleted files".format(n_rm))
@@ -2281,7 +2319,9 @@ class Up2k(object):
             if lost:
                 c2 = None
                 for cur, dp_dir, dp_fn in lost:
-                    self.db_rm(cur, dp_dir, dp_fn)
+                    t = "forgetting deleted file: /{}"
+                    self.log(t.format(vjoin(vjoin(vfs.vpath, dp_dir), dp_fn)))
+                    self.db_rm(cur, dp_dir, dp_fn, cj["size"])
                     if c2 and c2 != cur:
                         c2.connection.commit()
 
@@ -2415,7 +2455,14 @@ class Up2k(object):
 
                 if vfs.lim:
                     ap2, cj["prel"] = vfs.lim.all(
-                        cj["addr"], cj["prel"], cj["size"], ap1, reg
+                        cj["addr"],
+                        cj["prel"],
+                        cj["size"],
+                        cj["ptop"],
+                        ap1,
+                        self.hub.broker,
+                        reg,
+                        "up2k._get_volsize",
                     )
                     bos.makedirs(ap2)
                     vfs.lim.nup(cj["addr"])
@@ -2733,7 +2780,7 @@ class Up2k(object):
 
             self._symlink(dst, d2, self.flags[ptop], lmod=lmod)
             if cur:
-                self.db_rm(cur, rd, fn)
+                self.db_rm(cur, rd, fn, job["size"])
                 self.db_add(cur, vflags, rd, fn, lmod, *z2[3:])
 
         if cur:
@@ -2776,7 +2823,7 @@ class Up2k(object):
 
         self.db_act = self.vol_act[ptop] = time.time()
         try:
-            self.db_rm(cur, rd, fn)
+            self.db_rm(cur, rd, fn, sz)
             self.db_add(
                 cur,
                 vflags,
@@ -2806,13 +2853,17 @@ class Up2k(object):
 
         return True
 
-    def db_rm(self, db , rd , fn )  :
+    def db_rm(self, db , rd , fn , sz )  :
         sql = "delete from up where rd = ? and fn = ?"
         try:
-            db.execute(sql, (rd, fn))
+            r = db.execute(sql, (rd, fn))
         except:
             assert self.mem_cur
-            db.execute(sql, s3enc(self.mem_cur, rd, fn))
+            r = db.execute(sql, s3enc(self.mem_cur, rd, fn))
+
+        if r.rowcount:
+            self.volsize[db] -= sz
+            self.volnfiles[db] -= 1
 
     def db_add(
         self,
@@ -2841,6 +2892,9 @@ class Up2k(object):
             v = (wark, int(ts), sz, rd, fn, ip or "", int(at or 0))
             db.execute(sql, v)
 
+        self.volsize[db] += sz
+        self.volnfiles[db] += 1
+
         xau = False if skip_xau else vflags.get("xau")
         dst = djoin(ptop, rd, fn)
         if xau and not runhook(
@@ -2988,12 +3042,12 @@ class Up2k(object):
                         break
 
                 abspath = djoin(adir, fn)
+                st = bos.stat(abspath)
                 volpath = "{}/{}".format(vrem, fn).strip("/")
                 vpath = "{}/{}".format(dbv.vpath, volpath).strip("/")
                 self.log("rm {}\n  {}".format(vpath, abspath))
                 _ = dbv.get(volpath, uname, *permsets[0])
                 if xbd:
-                    st = bos.stat(abspath)
                     if not runhook(
                         self.log,
                         xbd,
@@ -3017,14 +3071,26 @@ class Up2k(object):
                     try:
                         ptop = dbv.realpath
                         cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath)
-                        self._forget_file(ptop, volpath, cur, wark, True)
+                        self._forget_file(ptop, volpath, cur, wark, True, st.st_size)
                     finally:
                         if cur:
                             cur.connection.commit()
 
                 bos.unlink(abspath)
                 if xad:
-                    runhook(self.log, xad, abspath, vpath, "", uname, 0, 0, ip, 0, "")
+                    runhook(
+                        self.log,
+                        xad,
+                        abspath,
+                        vpath,
+                        "",
+                        uname,
+                        st.st_mtime,
+                        st.st_size,
+                        ip,
+                        0,
+                        "",
+                    )
 
         if is_dir:
             ok, ng = rmdirs(self.log_func, scandir, True, atop, 1)
@@ -3200,7 +3266,7 @@ class Up2k(object):
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
-            self._forget_file(svn.realpath, srem, c1, w, c1 != c2)
+            self._forget_file(svn.realpath, srem, c1, w, c1 != c2, fsize)
             self._relink(w, svn.realpath, srem, dabs)
             curs.add(c1)
 
@@ -3276,6 +3342,7 @@ class Up2k(object):
         cur ,
         wark ,
         drop_tags ,
+        sz ,
     )  :
         """forgets file in db, fixes symlinks, does not delete"""
         srd, sfn = vsplit(vrem)
@@ -3290,7 +3357,7 @@ class Up2k(object):
                 q = "delete from mt where w=?"
                 cur.execute(q, (wark[:16],))
 
-            self.db_rm(cur, srd, sfn)
+            self.db_rm(cur, srd, sfn, sz)
 
         reg = self.registry.get(ptop)
         if reg: