copyparty 1.17.2__py3-none-any.whl → 1.18.1__py3-none-any.whl

copyparty/__init__.py

@@ -77,6 +77,7 @@ web/deps/prismd.css
 web/deps/scp.woff2
 web/deps/sha512.ac.js
 web/deps/sha512.hw.js
+web/idp.html
 web/iiam.gif
 web/md.css
 web/md.html

copyparty/__main__.py

@@ -956,6 +956,7 @@ def add_general(ap, nc, srvname):
     ap2.add_argument("--name", metavar="TXT", type=u, default=srvname, help="server name (displayed topleft in browser and in mDNS)")
     ap2.add_argument("--mime", metavar="EXT=MIME", type=u, action="append", help="map file \033[33mEXT\033[0mension to \033[33mMIME\033[0mtype, for example [\033[32mjpg=image/jpeg\033[0m]")
     ap2.add_argument("--mimes", action="store_true", help="list default mimetype mapping and exit")
+    ap2.add_argument("--rmagic", action="store_true", help="do expensive analysis to improve accuracy of returned mimetypes; will make file-downloads, rss, and webdav slower (volflag=rmagic)")
     ap2.add_argument("--license", action="store_true", help="show licenses and exit")
     ap2.add_argument("--version", action="store_true", help="show versions and exit")
 
@@ -1084,12 +1085,16 @@ def add_cert(ap, cert_path):
 
 
 def add_auth(ap):
+    idp_db = os.path.join(E.cfg, "idp.db")
     ses_db = os.path.join(E.cfg, "sessions.db")
     ap2 = ap.add_argument_group('IdP / identity provider / user authentication options')
     ap2.add_argument("--idp-h-usr", metavar="HN", type=u, default="", help="bypass the copyparty authentication checks if the request-header \033[33mHN\033[0m contains a username to associate the request with (for use with authentik/oauth/...)\n\033[1;31mWARNING:\033[0m if you enable this, make sure clients are unable to specify this header themselves; must be washed away and replaced by a reverse-proxy")
     ap2.add_argument("--idp-h-grp", metavar="HN", type=u, default="", help="assume the request-header \033[33mHN\033[0m contains the groupname of the requesting user; can be referenced in config files for group-based access control")
     ap2.add_argument("--idp-h-key", metavar="HN", type=u, default="", help="optional but recommended safeguard; your reverse-proxy will insert a secret header named \033[33mHN\033[0m into all requests, and the other IdP headers will be ignored if this header is not present")
     ap2.add_argument("--idp-gsep", metavar="RE", type=u, default="|:;+,", help="if there are multiple groups in \033[33m--idp-h-grp\033[0m, they are separated by one of the characters in \033[33mRE\033[0m")
+    ap2.add_argument("--idp-db", metavar="PATH", type=u, default=idp_db, help="where to store the known IdP users/groups (if you run multiple copyparty instances, make sure they use different DBs)")
+    ap2.add_argument("--idp-store", metavar="N", type=int, default=1, help="how to use \033[33m--idp-db\033[0m; [\033[32m0\033[0m] = entirely disable, [\033[32m1\033[0m] = write-only (effectively disabled), [\033[32m2\033[0m] = remember users, [\033[32m3\033[0m] = remember users and groups.\nNOTE: Will remember and restore the IdP-volumes of all users for all eternity if set to 2 or 3, even when user is deleted from your IdP")
+    ap2.add_argument("--idp-adm", metavar="U,U", type=u, default="", help="comma-separated list of users allowed to use /?idp (the cache management UI)")
     ap2.add_argument("--no-bauth", action="store_true", help="disable basic-authentication support; do not accept passwords from the 'Authenticate' header at all. NOTE: This breaks support for the android app")
     ap2.add_argument("--bauth-last", action="store_true", help="keeps basic-authentication enabled, but only as a last-resort; if a cookie is also provided then the cookie wins")
     ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")
@@ -1262,6 +1267,7 @@ def add_optouts(ap):
     ap2.add_argument("--no-tarcmp", action="store_true", help="disable download as compressed tar (?tar=gz, ?tar=bz2, ?tar=xz, ?tar=gz:9, ...)")
     ap2.add_argument("--no-lifetime", action="store_true", help="do not allow clients (or server config) to schedule an upload to be deleted after a given time")
     ap2.add_argument("--no-pipe", action="store_true", help="disable race-the-beam (lockstep download of files which are currently being uploaded) (volflag=nopipe)")
+    ap2.add_argument("--no-tail", action="store_true", help="disable streaming a growing files with ?tail (volflag=notail)")
     ap2.add_argument("--no-db-ip", action="store_true", help="do not write uploader-IP into the database; will also disable unpost, you may want \033[32m--forget-ip\033[0m instead (volflag=no_db_ip)")
 
 
@@ -1391,6 +1397,16 @@ def add_transcoding(ap):
     ap2.add_argument("--ac-maxage", metavar="SEC", type=int, default=86400, help="delete cached transcode output after \033[33mSEC\033[0m seconds")
 
 
+def add_tail(ap):
+    ap2 = ap.add_argument_group('tailing options (realtime streaming of a growing file)')
+    ap2.add_argument("--tail-who", metavar="LVL", type=int, default=2, help="who can tail? [\033[32m0\033[0m]=nobody, [\033[32m1\033[0m]=admins, [\033[32m2\033[0m]=authenticated-with-read-access, [\033[32m3\033[0m]=everyone-with-read-access (volflag=tail_who)")
+    ap2.add_argument("--tail-cmax", metavar="N", type=int, default=64, help="do not allow starting a new tail if more than \033[33mN\033[0m active downloads")
+    ap2.add_argument("--tail-tmax", metavar="SEC", type=float, default=0, help="terminate connection after \033[33mSEC\033[0m seconds; [\033[32m0\033[0m]=never (volflag=tail_tmax)")
+    ap2.add_argument("--tail-rate", metavar="SEC", type=float, default=0.2, help="check for new data every \033[33mSEC\033[0m seconds (volflag=tail_rate)")
+    ap2.add_argument("--tail-ka", metavar="SEC", type=float, default=3.0, help="send a zerobyte if connection is idle for \033[33mSEC\033[0m seconds to prevent disconnect")
+    ap2.add_argument("--tail-fd", metavar="SEC", type=float, default=1.0, help="check if file was replaced (new fd) if idle for \033[33mSEC\033[0m seconds (volflag=tail_fd)")
+
+
 def add_rss(ap):
     ap2 = ap.add_argument_group('RSS options')
     ap2.add_argument("--rss", action="store_true", help="enable RSS output (experimental) (volflag=rss)")
@@ -1486,6 +1502,7 @@ def add_ui(ap, retry):
     ap2.add_argument("--sort", metavar="C,C,C", type=u, default="href", help="default sort order, comma-separated column IDs (see header tooltips), prefix with '-' for descending. Examples: \033[32mhref -href ext sz ts tags/Album tags/.tn\033[0m (volflag=sort)")
     ap2.add_argument("--nsort", action="store_true", help="default-enable natural sort of filenames with leading numbers (volflag=nsort)")
     ap2.add_argument("--hsortn", metavar="N", type=int, default=2, help="number of sorting rules to include in media URLs by default (volflag=hsortn)")
+    ap2.add_argument("--see-dots", action="store_true", help="default-enable seeing dotfiles; only takes effect if user has the necessary permissions")
     ap2.add_argument("--unlist", metavar="REGEX", type=u, default="", help="don't show files matching \033[33mREGEX\033[0m in file list. Purely cosmetic! Does not affect API calls, just the browser. Example: [\033[32m\\.(js|css)$\033[0m] (volflag=unlist)")
     ap2.add_argument("--favico", metavar="TXT", type=u, default="c 000 none" if retry else "🎉 000 none", help="\033[33mfavicon-text\033[0m [ \033[33mforeground\033[0m [ \033[33mbackground\033[0m ] ], set blank to disable")
     ap2.add_argument("--ext-th", metavar="E=VP", type=u, action="append", help="use thumbnail-image \033[33mVP\033[0m for file-extension \033[33mE\033[0m, example: [\033[32mexe=/.res/exe.png\033[0m] (volflag=ext_th)")
@@ -1595,6 +1612,7 @@ def run_argparse(
     add_hooks(ap)
     add_stats(ap)
     add_txt(ap)
+    add_tail(ap)
     add_og(ap)
     add_ui(ap, retry)
     add_admin(ap)

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 17, 2)
-CODENAME = "mixtape.m3u"
-BUILD_DT = (2025, 5, 27)
+VERSION = (1, 18, 1)
+CODENAME = "logtail"
+BUILD_DT = (2025, 7, 7)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -21,6 +21,7 @@ from .util import (
     DEF_MTE,
     DEF_MTH,
     EXTS,
+    HAVE_SQLITE3,
     IMPLICATIONS,
     MIMES,
     SQLITE_VER,
@@ -32,6 +33,7 @@ from .util import (
     afsenc,
     get_df,
     humansize,
+    min_ex,
     odfusion,
     read_utf8,
     relchk,
@@ -44,6 +46,9 @@ from .util import (
     vsplit,
 )
 
+if HAVE_SQLITE3:
+    import sqlite3
+
 if TYPE_CHECKING:
     from .broker_mp import BrokerMp
     from .broker_thr import BrokerThr
@@ -65,7 +70,9 @@ SSEELOG = " ({})".format(SEE_LOG)
 BAD_CFG = "invalid config; {}".format(SEE_LOG)
 SBADCFG = " ({})".format(BAD_CFG)
 
-PTN_U_GRP = re.compile(r"\$\{u%([+-])([^}]+)\}")
+PTN_U_GRP = re.compile(r"\$\{u(%[+-][^}]+)\}")
+PTN_G_GRP = re.compile(r"\$\{g(%[+-][^}]+)\}")
+PTN_SIGIL = re.compile(r"(\${[ug][}%])")
 
 
 class CfgEx(Exception):
@@ -926,6 +933,10 @@ class AuthSrv(object):
                 return False
 
             self.idp_accs[uname] = gnames
+            try:
+                self._update_idp_db(uname, gname)
+            except:
+                self.log("failed to update the --idp-db:\n%s" % (min_ex(),), 3)
 
             t = "reinitializing due to new user from IdP: [%r:%r]"
             self.log(t % (uname, gnames), 3)
@@ -938,6 +949,21 @@ class AuthSrv(object):
         broker.ask("reload", False, True).get()
         return True
 
+    def _update_idp_db(self, uname , gname )  :
+        if not self.args.idp_store:
+            return
+
+
+        db = sqlite3.connect(self.args.idp_db)
+        cur = db.cursor()
+
+        cur.execute("delete from us where un = ?", (uname,))
+        cur.execute("insert into us values (?,?)", (uname, gname))
+
+        db.commit()
+        cur.close()
+        db.close()
+
     def _map_volume_idp(
         self,
         src ,
@@ -958,15 +984,27 @@ class AuthSrv(object):
             un_gn = [("", "")]
 
         for un, gn in un_gn:
-            m = PTN_U_GRP.search(dst0)
-            if m:
-                req, gnc = m.groups()
-                hit = gnc in (un_gns.get(un) or [])
-                if req == "+":
-                    if not hit:
-                        continue
-                elif hit:
+            rejected = False
+            for ptn in [PTN_U_GRP, PTN_G_GRP]:
+                m = ptn.search(dst0)
+                if not m:
                     continue
+                zs = m.group(1)
+                zs = zs.replace(",%+", "\n%+")
+                zs = zs.replace(",%-", "\n%-")
+                for rule in zs.split("\n"):
+                    gnc = rule[2:]
+                    if ptn == PTN_U_GRP:
+                        # is user member of group?
+                        hit = gnc in (un_gns.get(un) or [])
+                    else:
+                        # is it this specific group?
+                        hit = gn == gnc
+
+                    if rule.startswith("%+") != hit:
+                        rejected = True
+            if rejected:
+                continue
 
             # if ap/vp has a user/group placeholder, make sure to keep
             # track so the same user/group is mapped when setting perms;
@@ -981,6 +1019,8 @@ class AuthSrv(object):
 
             src = src1.replace("${g}", gn or "\n")
             dst = dst1.replace("${g}", gn or "\n")
+            src = PTN_G_GRP.sub(gn or "\n", src)
+            dst = PTN_G_GRP.sub(gn or "\n", dst)
             if src == src1 and dst == dst1:
                 gn = ""
 
@@ -1072,6 +1112,7 @@ class AuthSrv(object):
          * any non-zero value from IdP group header
          * otherwise take --grps / [groups]
         """
+        self.load_idp_db(bool(self.idp_accs))
         ret = {un: gns[:] for un, gns in self.idp_accs.items()}
         ret.update({zs: [""] for zs in acct if zs not in ret})
         for gn, uns in grps.items():
@@ -1632,7 +1673,6 @@ class AuthSrv(object):
         shr = enshare[1:-1]
         shrs = enshare[1:]
         if enshare:
-            import sqlite3
 
             zsd = {"d2d": True, "tcolor": self.args.tcolor}
             shv = VFS(self.log_func, "", shr, shr, AXS(), zsd)
@@ -1855,7 +1895,7 @@ class AuthSrv(object):
             is_shr = shr and zv.vpath.split("/")[0] == shr
             if histp and not is_shr and histp in rhisttab:
                 zv2 = rhisttab[histp]
-                t = "invalid config; multiple volumes share the same histpath (database+thumbnails location):\n  histpath: %s\n  volume 1: /%s  [%s]\n  volume 2: %s  [%s]"
+                t = "invalid config; multiple volumes share the same histpath (database+thumbnails location):\n  histpath: %s\n  volume 1: /%s  [%s]\n  volume 2: /%s  [%s]"
                 t = t % (histp, zv2.vpath, zv2.realpath, zv.vpath, zv.realpath)
                 self.log(t, 1)
                 raise Exception(t)
@@ -1869,7 +1909,7 @@ class AuthSrv(object):
             is_shr = shr and zv.vpath.split("/")[0] == shr
             if dbp and not is_shr and dbp in rdbpaths:
                 zv2 = rdbpaths[dbp]
-                t = "invalid config; multiple volumes share the same dbpath (database location):\n  dbpath: %s\n  volume 1: /%s  [%s]\n  volume 2: %s  [%s]"
+                t = "invalid config; multiple volumes share the same dbpath (database location):\n  dbpath: %s\n  volume 1: /%s  [%s]\n  volume 2: /%s  [%s]"
                 t = t % (dbp, zv2.vpath, zv2.realpath, zv.vpath, zv.realpath)
                 self.log(t, 1)
                 raise Exception(t)
@@ -2052,12 +2092,13 @@ class AuthSrv(object):
                 if vf not in vol.flags:
                     vol.flags[vf] = getattr(self.args, ga)
 
-            zs = "forget_ip nrand u2abort u2ow ups_who zip_who"
+            zs = "forget_ip nrand tail_who u2abort u2ow ups_who zip_who"
             for k in zs.split():
                 if k in vol.flags:
                     vol.flags[k] = int(vol.flags[k])
 
-            for k in ("convt",):
+            zs = "convt tail_fd tail_rate tail_tmax"
+            for k in zs.split():
                 if k in vol.flags:
                     vol.flags[k] = float(vol.flags[k])
 
@@ -2386,7 +2427,7 @@ class AuthSrv(object):
             idp_vn, _ = vfs.get(idp_vp, "*", False, False)
             idp_vp0 = idp_vn.vpath0
 
-            sigils = set(re.findall(r"(\${[ug][}%])", idp_vp0))
+            sigils = set(PTN_SIGIL.findall(idp_vp0))
             if len(sigils) > 1:
                 t = '\nWARNING: IdP-volume "/%s" created by "/%s" has multiple IdP placeholders: %s'
                 self.idp_warn.append(t % (idp_vp, idp_vp0, list(sigils)))
@@ -2556,6 +2597,7 @@ class AuthSrv(object):
                 "txt_ext": self.args.textfiles.replace(",", " "),
                 "def_hcols": list(vf.get("mth") or []),
                 "unlist0": vf.get("unlist") or "",
+                "see_dots": self.args.see_dots,
                 "dgrid": "grid" in vf,
                 "dgsel": "gsel" in vf,
                 "dnsort": "nsort" in vf,
@@ -2595,6 +2637,42 @@ class AuthSrv(object):
             zs = str(vol.flags.get("tcolor") or self.args.tcolor)
             vol.flags["tcolor"] = zs.lstrip("#")
 
+    def load_idp_db(self, quiet=False)  :
+        # mutex me
+        level = self.args.idp_store
+        if level < 2 or not self.args.idp_h_usr:
+            return
+
+
+        db = sqlite3.connect(self.args.idp_db)
+        cur = db.cursor()
+        from_cache = cur.execute("select un, gs from us").fetchall()
+        cur.close()
+        db.close()
+
+        self.idp_accs.clear()
+        self.idp_usr_gh.clear()
+
+        gsep = self.args.idp_gsep
+        n = []
+        for uname, gname in from_cache:
+            if level < 3:
+                if uname in self.idp_accs:
+                    continue
+                gname = ""
+            gnames = [x.strip() for x in gsep.split(gname)]
+            gnames.sort()
+
+            # self.idp_usr_gh[uname] = gname
+            self.idp_accs[uname] = gnames
+            n.append(uname)
+
+        if n and not quiet:
+            t = ", ".join(n[:9])
+            if len(n) > 9:
+                t += "..."
+            self.log("found %d IdP users in db (%s)" % (len(n), t))
+
     def load_sessions(self, quiet=False)  :
         # mutex me
         if self.args.no_ses:
@@ -2602,7 +2680,6 @@ class AuthSrv(object):
             self.sesa = {}
             return
 
-        import sqlite3
 
         ases = {}
         blen = (self.args.ses_len // 4) * 4  # 3 bytes in 4 chars
@@ -2649,7 +2726,6 @@ class AuthSrv(object):
         if self.args.no_ses:
             return
 
-        import sqlite3
 
         db = sqlite3.connect(self.args.ses_db)
         cur = db.cursor()

copyparty/cfg.py

@@ -22,6 +22,7 @@ def vf_bmap()   :
         "no_forget": "noforget",
         "no_pipe": "nopipe",
         "no_robots": "norobots",
+        "no_tail": "notail",
         "no_thumb": "dthumb",
         "no_vthumb": "dvthumb",
         "no_athumb": "dathumb",
@@ -51,6 +52,7 @@ def vf_bmap()   :
         "og_no_head",
         "og_s_title",
         "rand",
+        "rmagic",
         "rss",
         "wo_up_readme",
         "xdev",
@@ -101,6 +103,10 @@ def vf_vmap()   :
         "mv_retry",
         "rm_retry",
         "sort",
+        "tail_fd",
+        "tail_rate",
+        "tail_tmax",
+        "tail_who",
         "tcolor",
         "unlist",
         "u2abort",
@@ -304,6 +310,13 @@ flagcats = {
         "exp_md": "placeholders to expand in markdown files; see --help",
         "exp_lg": "placeholders to expand in prologue/epilogue; see --help",
     },
+    "tailing": {
+        "notail": "disable ?tail (download a growing file continuously)",
+        "tail_fd=1": "check if file was replaced (new fd) every 1 sec",
+        "tail_rate=0.2": "check for new data every 0.2 sec",
+        "tail_tmax=30": "kill connection after 30 sec",
+        "tail_who=2": "restrict ?tail access (1=admins,2=authed,3=everyone)",
+    },
     "others": {
         "dots": "allow all users with read-access to\nenable the option to show dotfiles in listings",
         "fk=8": 'generates per-file accesskeys,\nwhich are then required at the "g" permission;\nkeys are invalidated if filesize or inode changes',
@@ -312,6 +325,7 @@ flagcats = {
         "dks": "per-directory accesskeys allow browsing into subdirs",
         "dky": 'allow seeing files (not folders) inside a specific folder\nwith "g" perm, and does not require a valid dirkey to do so',
         "rss": "allow '?rss' URL suffix (experimental)",
+        "rmagic": "expensive analysis for mimetype accuracy",
         "ups_who=2": "restrict viewing the list of recent uploads",
         "zip_who=2": "restrict access to download-as-zip/tar",
         "zipmaxn=9k": "reject download-as-zip if more than 9000 files",