copyparty 1.14.4__py3-none-any.whl → 1.15.1__py3-none-any.whl

copyparty/__main__.py

@@ -986,9 +986,10 @@ def add_upload(ap):
     ap2.add_argument("--reg-cap", metavar="N", type=int, default=38400, help="max number of uploads to keep in memory when running without \033[33m-e2d\033[0m; roughly 1 MiB RAM per 600")
     ap2.add_argument("--no-fpool", action="store_true", help="disable file-handle pooling -- instead, repeatedly close and reopen files during upload (bad idea to enable this on windows and/or cow filesystems)")
     ap2.add_argument("--use-fpool", action="store_true", help="force file-handle pooling, even when it might be dangerous (multiprocessing, filesystems lacking sparse-files support, ...)")
-    ap2.add_argument("--hardlink", action="store_true", help="prefer hardlinks instead of symlinks when possible (within same filesystem) (volflag=hardlink)")
-    ap2.add_argument("--never-symlink", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=neversymlink)")
-    ap2.add_argument("--no-dedup", action="store_true", help="disable symlink/hardlink creation; copy file contents instead (volflag=copydupes)")
+    ap2.add_argument("--dedup", action="store_true", help="enable symlink-based upload deduplication (volflag=dedup)")
+    ap2.add_argument("--safe-dedup", metavar="N", type=int, default=50, help="how careful to be when deduplicating files; [\033[32m1\033[0m] = just verify the filesize, [\033[32m50\033[0m] = verify file contents have not been altered (volflag=safededup)")
+    ap2.add_argument("--hardlink", action="store_true", help="enable hardlink-based dedup; will fallback on symlinks when that is impossible (across filesystems) (volflag=hardlink)")
+    ap2.add_argument("--hardlink-only", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=hardlinkonly)")
     ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
     ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
     ap2.add_argument("--snap-wri", metavar="SEC", type=int, default=300, help="write upload state to ./hist/up2k.snap every \033[33mSEC\033[0m seconds; allows resuming incomplete uploads after a server crash")
@@ -1060,6 +1061,7 @@ def add_cert(ap, cert_path):
 
 
 def add_auth(ap):
+    ses_db = os.path.join(E.cfg, "sessions.db")
     ap2 = ap.add_argument_group('IdP / identity provider / user authentication options')
     ap2.add_argument("--idp-h-usr", metavar="HN", type=u, default="", help="bypass the copyparty authentication checks and assume the request-header \033[33mHN\033[0m contains the username of the requesting user (for use with authentik/oauth/...)\n\033[1;31mWARNING:\033[0m if you enable this, make sure clients are unable to specify this header themselves; must be washed away and replaced by a reverse-proxy")
     ap2.add_argument("--idp-h-grp", metavar="HN", type=u, default="", help="assume the request-header \033[33mHN\033[0m contains the groupname of the requesting user; can be referenced in config files for group-based access control")
@@ -1067,6 +1069,9 @@ def add_auth(ap):
     ap2.add_argument("--idp-gsep", metavar="RE", type=u, default="|:;+,", help="if there are multiple groups in \033[33m--idp-h-grp\033[0m, they are separated by one of the characters in \033[33mRE\033[0m")
     ap2.add_argument("--no-bauth", action="store_true", help="disable basic-authentication support; do not accept passwords from the 'Authenticate' header at all. NOTE: This breaks support for the android app")
     ap2.add_argument("--bauth-last", action="store_true", help="keeps basic-authentication enabled, but only as a last-resort; if a cookie is also provided then the cookie wins")
+    ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")
+    ap2.add_argument("--ses-len", metavar="CHARS", type=int, default=20, help="session key length; default is 120 bits ((20//4)*4*6)")
+    ap2.add_argument("--no-ses", action="store_true", help="disable sessions; use plaintext passwords in cookies")
 
 
 def add_chpw(ap):
@@ -1279,6 +1284,7 @@ def add_logging(ap):
     ap2.add_argument("--ansi", action="store_true", help="force colors; overrides environment-variable NO_COLOR")
     ap2.add_argument("--no-logflush", action="store_true", help="don't flush the logfile after each write; tiny bit faster")
     ap2.add_argument("--no-voldump", action="store_true", help="do not list volumes and permissions on startup")
+    ap2.add_argument("--log-utc", action="store_true", help="do not use local timezone; assume the TZ env-var is UTC (tiny bit faster)")
     ap2.add_argument("--log-tdec", metavar="N", type=int, default=3, help="timestamp resolution / number of timestamp decimals")
     ap2.add_argument("--log-badpwd", metavar="N", type=int, default=1, help="log failed login attempt passwords: 0=terse, 1=plaintext, 2=hashed")
     ap2.add_argument("--log-conn", action="store_true", help="debug: print tcp-server msgs")
@@ -1337,7 +1343,7 @@ def add_transcoding(ap):
 def add_db_general(ap, hcores):
     noidx = APPLESAN_TXT if MACOS else ""
     ap2 = ap.add_argument_group('general db options')
-    ap2.add_argument("-e2d", action="store_true", help="enable up2k database, making files searchable + enables upload deduplication")
+    ap2.add_argument("-e2d", action="store_true", help="enable up2k database; this enables file search, upload-undo, improves deduplication")
     ap2.add_argument("-e2ds", action="store_true", help="scan writable folders for new files on startup; sets \033[33m-e2d\033[0m")
     ap2.add_argument("-e2dsa", action="store_true", help="scans all folders on startup; sets \033[33m-e2ds\033[0m")
     ap2.add_argument("-e2v", action="store_true", help="verify file integrity; rehash all files and compare with db")
@@ -1350,7 +1356,7 @@ def add_db_general(ap, hcores):
     ap2.add_argument("--re-dhash", action="store_true", help="force a cache rebuild on startup; enable this once if it gets out of sync (should never be necessary)")
     ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice -- only useful for offloading uploads to a cloud service or something (volflag=noforget)")
     ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see \033[33m--help-dbd\033[0m (volflag=dbd)")
-    ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")
+    ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (probably buggy, not recommended) (volflag=xlink)")
     ap2.add_argument("--hash-mt", metavar="CORES", type=int, default=hcores, help="num cpu cores to use for file hashing; set 0 or 1 for single-core hashing")
     ap2.add_argument("--re-maxage", metavar="SEC", type=int, default=0, help="rescan filesystem for changes every \033[33mSEC\033[0m seconds; 0=off (volflag=scan)")
     ap2.add_argument("--db-act", metavar="SEC", type=float, default=10.0, help="defer any scheduled volume reindexing until \033[33mSEC\033[0m seconds after last db write (uploads, renames, ...)")
@@ -1463,6 +1469,7 @@ def run_argparse(
 )  :
     ap = argparse.ArgumentParser(
         formatter_class=formatter,
+        usage=argparse.SUPPRESS,
         prog="copyparty",
         description="http file sharing hub v{} ({})".format(S_VERSION, S_BUILD_DT),
     )
@@ -1613,6 +1620,7 @@ def main(argv  = None, rsrc  = None)  :
         ("--hdr-au-usr", "--idp-h-usr"),
         ("--idp-h-sep", "--idp-gsep"),
         ("--th-no-crop", "--th-crop=n"),
+        ("--never-symlink", "--hardlink-only"),
     ]
     for dk, nk in deprecated:
         idx = -1
@@ -1637,7 +1645,7 @@ def main(argv  = None, rsrc  = None)  :
             argv.extend(["--qr"])
             if ANYWIN or not os.geteuid():
                 # win10 allows symlinks if admin; can be unexpected
-                argv.extend(["-p80,443,3923", "--ign-ebind", "--no-dedup"])
+                argv.extend(["-p80,443,3923", "--ign-ebind"])
     except:
         pass
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 14, 4)
-CODENAME = "one step forward"
-BUILD_DT = (2024, 9, 2)
+VERSION = (1, 15, 1)
+CODENAME = "fill the drives"
+BUILD_DT = (2024, 9, 9)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -833,8 +833,10 @@ class AuthSrv(object):
 
         # fwd-decl
         self.vfs = VFS(log_func, "", "", AXS(), {})
-        self.acct   = {}
-        self.iacct   = {}
+        self.acct   = {}  # uname->pw
+        self.iacct   = {}  # pw->uname
+        self.ases   = {}  # uname->session
+        self.sesa   = {}  # session->uname
         self.defpw   = {}
         self.grps   = {}
         self.re_pwd  = None
@@ -1884,6 +1886,11 @@ class AuthSrv(object):
             if len(zs) == 3:  # fc5 => ffcc55
                 vol.flags["tcolor"] = "".join([x * 2 for x in zs])
 
+            if vol.flags.get("neversymlink"):
+                vol.flags["hardlinkonly"] = True  # was renamed
+            if vol.flags.get("hardlinkonly"):
+                vol.flags["hardlink"] = True
+
             for k1, k2 in IMPLICATIONS:
                 if k1 in vol.flags:
                     vol.flags[k2] = True
@@ -1988,9 +1995,6 @@ class AuthSrv(object):
                     for x in drop:
                         vol.flags.pop(x)
 
-            if vol.flags.get("neversymlink") and not vol.flags.get("hardlink"):
-                vol.flags["copydupes"] = True
-
             # verify tags mentioned by -mt[mp] are used by -mte
             local_mtp = {}
             local_only_mtp = {}
@@ -2069,6 +2073,8 @@ class AuthSrv(object):
 
         have_e2d = False
         have_e2t = False
+        have_dedup = False
+        unsafe_dedup = []
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
             if not self.warn_anonwrite or verbosity < 5:
@@ -2101,6 +2107,11 @@ class AuthSrv(object):
             if "e2t" in zv.flags:
                 have_e2t = True
 
+            if "dedup" in zv.flags:
+                have_dedup = True
+                if "e2d" not in zv.flags and "hardlink" not in zv.flags:
+                    unsafe_dedup.append("/" + zv.vpath)
+
             t += "\n"
 
         if self.warn_anonwrite and verbosity > 4:
@@ -2113,10 +2124,17 @@ class AuthSrv(object):
                     self.log("\n\033[{}\033[0m\n".format(t))
 
                 if not have_e2t:
-                    t = "hint: argument -e2ts enables multimedia indexing (artist/title/...)"
+                    t = "hint: enable multimedia indexing (artist/title/...) with argument -e2ts"
                     self.log(t, 6)
             else:
-                t = "hint: argument -e2dsa enables searching, upload-undo, and better deduplication"
+                t = "hint: enable searching and upload-undo with argument -e2dsa"
+                self.log(t, 6)
+
+            if unsafe_dedup:
+                t = "WARNING: symlink-based deduplication is enabled for some volumes, but without indexing. Please enable -e2dsa and/or --hardlink to avoid problems when moving/renaming files. Affected volumes: %s"
+                self.log(t % (", ".join(unsafe_dedup)), 3)
+            elif not have_dedup:
+                t = "hint: enable upload deduplication with --dedup (but see readme for consequences)"
                 self.log(t, 6)
 
             zv, _ = vfs.get("/", "*", False, False)
@@ -2158,8 +2176,11 @@ class AuthSrv(object):
         self.grps = grps
         self.iacct = {v: k for k, v in acct.items()}
 
+        self.load_sessions()
+
         self.re_pwd = None
         pwds = [re.escape(x) for x in self.iacct.keys()]
+        pwds.extend(list(self.sesa))
         if pwds:
             if self.ah.on:
                 zs = r"(\[H\] pw:.*|[?&]pw=)([^&]+)"
@@ -2234,6 +2255,72 @@ class AuthSrv(object):
             cur.close()
             db.close()
 
+    def load_sessions(self, quiet=False)  :
+        # mutex me
+        if self.args.no_ses:
+            self.ases = {}
+            self.sesa = {}
+            return
+
+        import sqlite3
+
+        ases = {}
+        blen = (self.args.ses_len // 4) * 4  # 3 bytes in 4 chars
+        blen = (blen * 3) // 4  # bytes needed for ses_len chars
+
+        db = sqlite3.connect(self.args.ses_db)
+        cur = db.cursor()
+
+        for uname, sid in cur.execute("select un, si from us"):
+            if uname in self.acct:
+                ases[uname] = sid
+
+        n = []
+        q = "insert into us values (?,?,?)"
+        for uname in self.acct:
+            if uname not in ases:
+                sid = ub64enc(os.urandom(blen)).decode("utf-8")
+                cur.execute(q, (uname, sid, int(time.time())))
+                ases[uname] = sid
+                n.append(uname)
+
+        if n:
+            db.commit()
+
+        cur.close()
+        db.close()
+
+        self.ases = ases
+        self.sesa = {v: k for k, v in ases.items()}
+        if n and not quiet:
+            t = ", ".join(n[:3])
+            if len(n) > 3:
+                t += "..."
+            self.log("added %d new sessions (%s)" % (len(n), t))
+
+    def forget_session(self, broker , uname )  :
+        with self.mutex:
+            self._forget_session(uname)
+
+        if broker:
+            broker.ask("_reload_sessions").get()
+
+    def _forget_session(self, uname )  :
+        if self.args.no_ses:
+            return
+
+        import sqlite3
+
+        db = sqlite3.connect(self.args.ses_db)
+        cur = db.cursor()
+        cur.execute("delete from us where un = ?", (uname,))
+        db.commit()
+        cur.close()
+        db.close()
+
+        self.sesa.pop(self.ases.get(uname, ""), "")
+        self.ases.pop(uname, "")
+
     def chpw(self, broker , uname, pw)   :
         if not self.args.chpw:
             return False, "feature disabled in server config"
@@ -2253,7 +2340,7 @@ class AuthSrv(object):
         if hpw == self.acct[uname]:
             return False, "that's already your password my dude"
 
-        if hpw in self.iacct:
+        if hpw in self.iacct or hpw in self.sesa:
             return False, "password is taken"
 
         with self.mutex:

copyparty/broker_mp.py

@@ -72,6 +72,10 @@ class BrokerMp(object):
         for _, proc in enumerate(self.procs):
             proc.q_pend.put((0, "reload", []))
 
+    def reload_sessions(self)  :
+        for _, proc in enumerate(self.procs):
+            proc.q_pend.put((0, "reload_sessions", []))
+
     def collector(self, proc )  :
         """receive message from hub in other process"""
         while True:

copyparty/broker_mpw.py

@@ -88,6 +88,10 @@ class MpWorker(BrokerCli):
                 self.asrv.reload()
                 self.logw("mpw.asrv reloaded")
 
+            elif dest == "reload_sessions":
+                with self.asrv.mutex:
+                    self.asrv.load_sessions()
+
             elif dest == "listen":
                 self.httpsrv.listen(args[0], args[1])
 

copyparty/broker_thr.py

@@ -30,6 +30,7 @@ class BrokerThr(BrokerCli):
         self.iphash = HMaccas(os.path.join(self.args.E.cfg, "iphash"), 8)
         self.httpsrv = HttpSrv(self, None)
         self.reload = self.noop
+        self.reload_sessions = self.noop
 
     def shutdown(self)  :
         # self.log("broker", "shutting down")

copyparty/cfg.py

@@ -12,8 +12,7 @@ def vf_bmap()   :
         "dav_auth": "davauth",
         "dav_rt": "davrt",
         "ed": "dots",
-        "never_symlink": "neversymlink",
-        "no_dedup": "copydupes",
+        "hardlink_only": "hardlinkonly",
         "no_dupe": "nodupe",
         "no_forget": "noforget",
         "no_pipe": "nopipe",
@@ -23,6 +22,7 @@ def vf_bmap()   :
         "no_athumb": "dathumb",
     }
     for k in (
+        "dedup",
         "dotsrch",
         "e2d",
         "e2ds",
@@ -58,6 +58,7 @@ def vf_vmap()   :
         "no_hash": "nohash",
         "no_idx": "noidx",
         "re_maxage": "scan",
+        "safe_dedup": "safededup",
         "th_convt": "convt",
         "th_size": "thsize",
         "th_crop": "crop",
@@ -129,10 +130,11 @@ permdescs = {
 
 flagcats = {
     "uploads, general": {
+        "dedup": "enable symlink-based file deduplication",
+        "hardlink": "enable hardlink-based file deduplication,\nwith fallback on symlinks when that is impossible",
+        "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
+        "safededup": "verify on-disk data before using it for dedup",
         "nodupe": "rejects existing files (instead of symlinking them)",
-        "hardlink": "does dedup with hardlinks instead of symlinks",
-        "neversymlink": "disables symlink fallback; full copy instead",
-        "copydupes": "disables dedup, always saves full copies of dupes",
         "sparse": "force use of sparse files, mainly for s3-backed storage",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",
@@ -159,7 +161,7 @@ flagcats = {
         "lifetime=3600": "uploads are deleted after 1 hour",
     },
     "database, general": {
-        "e2d": "enable database; makes files searchable + enables upload dedup",
+        "e2d": "enable database; makes files searchable + enables upload-undo",
         "e2ds": "scan writable folders for new files on startup; also sets -e2d",
         "e2dsa": "scans all folders for new files on startup; also sets -e2d",
         "e2t": "enable multimedia indexing; makes it possible to search for tags",
@@ -177,7 +179,7 @@ flagcats = {
         "noforget": "don't forget files when deleted from disk",
         "fat32": "avoid excessive reindexing on android sdcardfs",
         "dbd=[acid|swal|wal|yolo]": "database speed-durability tradeoff",
-        "xlink": "cross-volume dupe detection / linking",
+        "xlink": "cross-volume dupe detection / linking (dangerous)",
         "xdev": "do not descend into other filesystems",
         "xvol": "do not follow symlinks leaving the volume root",
         "dotsrch": "show dotfiles in search results",

copyparty/httpcli.py

@@ -201,7 +201,8 @@ class HttpCli(object):
 
     def unpwd(self, m )  :
         a, b, c = m.groups()
-        return "%s\033[7m %s \033[27m%s" % (a, self.asrv.iacct[b], c)
+        uname = self.asrv.iacct.get(b) or self.asrv.sesa.get(b)
+        return "%s\033[7m %s \033[27m%s" % (a, uname, c)
 
     def _check_nonfatal(self, ex , post )  :
         if post:
@@ -500,6 +501,8 @@ class HttpCli(object):
                 zs = base64.b64decode(zb).decode("utf-8")
                 # try "pwd", "x:pwd", "pwd:x"
                 for bauth in [zs] + zs.split(":", 1)[::-1]:
+                    if bauth in self.asrv.sesa:
+                        break
                     hpw = self.asrv.ah.hash(bauth)
                     if self.asrv.iacct.get(hpw):
                         break
@@ -561,7 +564,11 @@ class HttpCli(object):
                 self.uname = "*"
         else:
             self.pw = uparam.get("pw") or self.headers.get("pw") or bauth or cookie_pw
-            self.uname = self.asrv.iacct.get(self.asrv.ah.hash(self.pw)) or "*"
+            self.uname = (
+                self.asrv.sesa.get(self.pw)
+                or self.asrv.iacct.get(self.asrv.ah.hash(self.pw))
+                or "*"
+            )
 
         self.rvol = self.asrv.vfs.aread[self.uname]
         self.wvol = self.asrv.vfs.awrite[self.uname]
@@ -2084,6 +2091,9 @@ class HttpCli(object):
         if act == "chpw":
             return self.handle_chpw()
 
+        if act == "logout":
+            return self.handle_logout()
+
         raise Pebkac(422, 'invalid action "{}"'.format(act))
 
     def handle_zip_post(self)  :
@@ -2405,7 +2415,8 @@ class HttpCli(object):
                 msg = "new password OK"
 
         redir = (self.args.SRS + "?h") if ok else ""
-        html = self.j2s("msg", h1=msg, h2='<a href="/?h">ack</a>', redir=redir)
+        h2 = '<a href="' + self.args.SRS + '?h">ack</a>'
+        html = self.j2s("msg", h1=msg, h2=h2, redir=redir)
         self.reply(html.encode("utf-8"))
         return True
 
@@ -2418,9 +2429,8 @@ class HttpCli(object):
             uhash = ""
         self.parser.drop()
 
-        self.out_headerlist = [
-            x for x in self.out_headerlist if x[0] != "Set-Cookie" or "cppw" != x[1][:4]
-        ]
+        if not pwd:
+            raise Pebkac(422, "password cannot be blank")
 
         dst = self.args.SRS
         if self.vpath:
@@ -2438,9 +2448,27 @@ class HttpCli(object):
         self.reply(html.encode("utf-8"))
         return True
 
+    def handle_logout(self)  :
+        assert self.parser
+        self.parser.drop()
+
+        self.log("logout " + self.uname)
+        self.asrv.forget_session(self.conn.hsrv.broker, self.uname)
+        self.get_pwd_cookie("x")
+
+        dst = self.args.SRS + "?h"
+        h2 = '<a href="' + dst + '">ack</a>'
+        html = self.j2s("msg", h1="ok bye", h2=h2, redir=dst)
+        self.reply(html.encode("utf-8"))
+        return True
+
     def get_pwd_cookie(self, pwd )   :
-        hpwd = self.asrv.ah.hash(pwd)
-        uname = self.asrv.iacct.get(hpwd)
+        uname = self.asrv.sesa.get(pwd)
+        if not uname:
+            hpwd = self.asrv.ah.hash(pwd)
+            uname = self.asrv.iacct.get(hpwd)
+            if uname:
+                pwd = self.asrv.ases.get(uname) or pwd
         if uname:
             msg = "hi " + uname
             dur = int(60 * 60 * self.args.logout)
@@ -2452,8 +2480,9 @@ class HttpCli(object):
                 zb = hashlib.sha512(pwd.encode("utf-8", "replace")).digest()
                 logpwd = "%" + base64.b64encode(zb[:12]).decode("utf-8")
 
-            self.log("invalid password: {}".format(logpwd), 3)
-            self.cbonk(self.conn.hsrv.gpwd, pwd, "pw", "invalid passwords")
+            if pwd != "x":
+                self.log("invalid password: {}".format(logpwd), 3)
+                self.cbonk(self.conn.hsrv.gpwd, pwd, "pw", "invalid passwords")
 
             msg = "naw dude"
             pwd = "x"  # nosec
@@ -2465,10 +2494,11 @@ class HttpCli(object):
             for k in ("cppwd", "cppws") if self.is_https else ("cppwd",):
                 ck = gencookie(k, pwd, self.args.R, False)
                 self.out_headerlist.append(("Set-Cookie", ck))
+            self.out_headers.pop("Set-Cookie", None)  # drop keepalive
         else:
             k = "cppws" if self.is_https else "cppwd"
             ck = gencookie(k, pwd, self.args.R, self.is_https, dur, "; HttpOnly")
-            self.out_headerlist.append(("Set-Cookie", ck))
+            self.out_headers["Set-Cookie"] = ck
 
         return dur > 0, msg