copyparty 1.15.5__py3-none-any.whl → 1.15.7__py3-none-any.whl

copyparty/__main__.py

@@ -772,7 +772,7 @@ def get_sects():
             dedent(
                 """
             specify --exp or the "exp" volflag to enable placeholder expansions
-            in README.md / .prologue.html / .epilogue.html
+            in README.md / PREADME.md / .prologue.html / .epilogue.html
 
             --exp-md (volflag exp_md) holds the list of placeholders which can be
             expanded in READMEs, and --exp-lg (volflag exp_lg) likewise for logues;
@@ -888,7 +888,7 @@ def get_sects():
             dedent(
                 """
             the mDNS protocol is multicast-based, which means there are thousands
-            of fun and intersesting ways for it to break unexpectedly
+            of fun and interesting ways for it to break unexpectedly
 
             things to check if it does not work at all:
 
@@ -997,6 +997,7 @@ def add_upload(ap):
     ap2.add_argument("--hardlink", action="store_true", help="enable hardlink-based dedup; will fallback on symlinks when that is impossible (across filesystems) (volflag=hardlink)")
     ap2.add_argument("--hardlink-only", action="store_true", help="do not fallback to symlinks when a hardlink cannot be made (volflag=hardlinkonly)")
     ap2.add_argument("--no-dupe", action="store_true", help="reject duplicate files during upload; only matches within the same volume (volflag=nodupe)")
+    ap2.add_argument("--no-clone", action="store_true", help="do not use existing data on disk to satisfy dupe uploads; reduces server HDD reads in exchange for much more network load (volflag=noclone)")
     ap2.add_argument("--no-snap", action="store_true", help="disable snapshots -- forget unfinished uploads on shutdown; don't create .hist/up2k.snap files -- abandoned/interrupted uploads must be cleaned up manually")
     ap2.add_argument("--snap-wri", metavar="SEC", type=int, default=300, help="write upload state to ./hist/up2k.snap every \033[33mSEC\033[0m seconds; allows resuming incomplete uploads after a server crash")
     ap2.add_argument("--snap-drop", metavar="MIN", type=float, default=1440.0, help="forget unfinished uploads after \033[33mMIN\033[0m minutes; impossible to resume them after that (360=6h, 1440=24h)")
@@ -1078,6 +1079,7 @@ def add_auth(ap):
     ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")
     ap2.add_argument("--ses-len", metavar="CHARS", type=int, default=20, help="session key length; default is 120 bits ((20//4)*4*6)")
     ap2.add_argument("--no-ses", action="store_true", help="disable sessions; use plaintext passwords in cookies")
+    ap2.add_argument("--ipu", metavar="CIDR=USR", type=u, action="append", help="users with IP matching \033[33mCIDR\033[0m are auto-authenticated as username \033[33mUSR\033[0m; example: [\033[32m172.16.24.0/24=dave]")
 
 
 def add_chpw(ap):
@@ -1246,7 +1248,7 @@ def add_safety(ap):
     ap2.add_argument("--no-dot-mv", action="store_true", help="disallow moving dotfiles; makes it impossible to move folders containing dotfiles")
     ap2.add_argument("--no-dot-ren", action="store_true", help="disallow renaming dotfiles; makes it impossible to turn something into a dotfile")
     ap2.add_argument("--no-logues", action="store_true", help="disable rendering .prologue/.epilogue.html into directory listings")
-    ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme.md into directory listings")
+    ap2.add_argument("--no-readme", action="store_true", help="disable rendering readme/preadme.md into directory listings")
     ap2.add_argument("--vague-403", action="store_true", help="send 404 instead of 403 (security through ambiguity, very enterprise)")
     ap2.add_argument("--force-js", action="store_true", help="don't send folder listings as HTML, force clients to use the embedded json instead -- slight protection against misbehaving search engines which ignore \033[33m--no-robots\033[0m")
     ap2.add_argument("--no-robots", action="store_true", help="adds http and html headers asking search engines to not index anything (volflag=norobots)")
@@ -1444,7 +1446,7 @@ def add_ui(ap, retry):
     ap2.add_argument("--k304", metavar="NUM", type=int, default=0, help="configure the option to enable/disable k304 on the controlpanel (workaround for buggy reverse-proxies); [\033[32m0\033[0m] = hidden and default-off, [\033[32m1\033[0m] = visible and default-off, [\033[32m2\033[0m] = visible and default-on")
     ap2.add_argument("--md-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for README.md docs (volflag=md_sbf); see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#attr-sandbox")
     ap2.add_argument("--lg-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for prologue/epilogue docs  (volflag=lg_sbf)")
-    ap2.add_argument("--no-sb-md", action="store_true", help="don't sandbox README.md documents (volflags: no_sb_md | sb_md)")
+    ap2.add_argument("--no-sb-md", action="store_true", help="don't sandbox README/PREADME.md documents (volflags: no_sb_md | sb_md)")
     ap2.add_argument("--no-sb-lg", action="store_true", help="don't sandbox prologue/epilogue docs (volflags: no_sb_lg | sb_lg); enables non-js support")
 
 
@@ -1468,6 +1470,7 @@ def add_debug(ap):
     ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to \033[33m--bf-nc\033[0m and \033[33m--bf-dir\033[0m")
     ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than \033[33mNUM\033[0m files at \033[33m--kf-dir\033[0m already; default: 6.3 GiB max (200*32M)")
     ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at \033[33mPATH\033[0m; default: folder named 'bf' wherever copyparty was started")
+    ap2.add_argument("--bf-log", metavar="PATH", type=u, default="", help="bak-flips: log corruption info to a textfile at \033[33mPATH\033[0m")
 
 
 # fmt: on

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 15, 5)
+VERSION = (1, 15, 7)
 CODENAME = "fill the drives"
-BUILD_DT = (2024, 10, 5)
+BUILD_DT = (2024, 10, 14)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -917,7 +917,7 @@ class AuthSrv(object):
 
         for un, gn in un_gn:
             # if ap/vp has a user/group placeholder, make sure to keep
-            # track so the same user/gruop is mapped when setting perms;
+            # track so the same user/group is mapped when setting perms;
             # otherwise clear un/gn to indicate it's a regular volume
 
             src1 = src0.replace("${u}", un or "\n")

copyparty/cfg.py

@@ -13,6 +13,7 @@ def vf_bmap()   :
         "dav_rt": "davrt",
         "ed": "dots",
         "hardlink_only": "hardlinkonly",
+        "no_clone": "noclone",
         "no_dirsz": "nodirsz",
         "no_dupe": "nodupe",
         "no_forget": "noforget",
@@ -135,7 +136,8 @@ flagcats = {
         "hardlink": "enable hardlink-based file deduplication,\nwith fallback on symlinks when that is impossible",
         "hardlinkonly": "dedup with hardlink only, never symlink;\nmake a full copy if hardlink is impossible",
         "safededup": "verify on-disk data before using it for dedup",
-        "nodupe": "rejects existing files (instead of symlinking them)",
+        "noclone": "take dupe data from clients, even if available on HDD",
+        "nodupe": "rejects existing files (instead of linking/cloning them)",
         "sparse": "force use of sparse files, mainly for s3-backed storage",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",

copyparty/ftpd.py

@@ -72,6 +72,7 @@ class FtpAuth(DummyAuthorizer):
             else:
                 raise AuthenticationFailed("banned")
 
+        args = self.hub.args
         asrv = self.hub.asrv
         uname = "*"
         if username != "anonymous":
@@ -82,6 +83,9 @@ class FtpAuth(DummyAuthorizer):
                     uname = zs
                     break
 
+        if args.ipu and uname == "*":
+            uname = args.ipu_iu[args.ipu_nm.map(ip)]
+
         if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
             g = self.hub.gpwd
             if g.lim:

copyparty/httpcli.py

@@ -123,6 +123,10 @@ _ = (argparse, threading)
 
 NO_CACHE = {"Cache-Control": "no-cache"}
 
+LOGUES = [[0, ".prologue.html"], [1, ".epilogue.html"]]
+
+READMES = [[0, ["preadme.md", "PREADME.md"]], [1, ["readme.md", "README.md"]]]
+
 
 class HttpCli(object):
     """
@@ -580,6 +584,9 @@ class HttpCli(object):
                 or "*"
             )
 
+        if self.args.ipu and self.uname == "*":
+            self.uname = self.conn.ipu_iu[self.conn.ipu_nm.map(self.ip)]
+
         self.rvol = self.asrv.vfs.aread[self.uname]
         self.wvol = self.asrv.vfs.awrite[self.uname]
         self.avol = self.asrv.vfs.aadmin[self.uname]
@@ -2019,13 +2026,32 @@ class HttpCli(object):
         return True
 
     def bakflip(
-        self, f , ofs , sz , sha , flags  
+        self,
+        f ,
+        ap ,
+        ofs ,
+        sz ,
+        good_sha ,
+        bad_sha ,
+        flags  ,
     )  :
+        now = time.time()
+        t = "bad-chunk:  %.3f  %s  %s  %d  %s  %s  %s"
+        t = t % (now, bad_sha, good_sha, ofs, self.ip, self.uname, ap)
+        self.log(t, 5)
+
+        if self.args.bf_log:
+            try:
+                with open(self.args.bf_log, "ab+") as f2:
+                    f2.write((t + "\n").encode("utf-8", "replace"))
+            except Exception as ex:
+                self.log("append %s failed: %r" % (self.args.bf_log, ex))
+
         if not self.args.bak_flips or self.args.nw:
             return
 
         sdir = self.args.bf_dir
-        fp = os.path.join(sdir, sha)
+        fp = os.path.join(sdir, bad_sha)
         if bos.path.exists(fp):
             return self.log("no bakflip; have it", 6)
 
@@ -2145,11 +2171,17 @@ class HttpCli(object):
         except UnrecvEOF:
             raise Pebkac(422, "client disconnected while posting JSON")
 
-        self.log("decoding {} bytes of {} json".format(len(json_buf), enc))
         try:
             body = json.loads(json_buf.decode(enc, "replace"))
+            try:
+                zds = {k: v for k, v in body.items()}
+                zds["hash"] = "%d chunks" % (len(body["hash"]))
+            except:
+                zds = body
+            t = "POST len=%d type=%s ip=%s user=%s req=%r json=%s"
+            self.log(t % (len(json_buf), enc, self.ip, self.uname, self.req, zds))
         except:
-            raise Pebkac(422, "you POSTed invalid json")
+            raise Pebkac(422, "you POSTed %d bytes of invalid json" % (len(json_buf),))
 
         # self.reply(b"cloudflare", 503)
         # return True
@@ -2347,7 +2379,9 @@ class HttpCli(object):
 
                     if sha_b64 != chash:
                         try:
-                            self.bakflip(f, cstart[0], post_sz, sha_b64, vfs.flags)
+                            self.bakflip(
+                                f, path, cstart[0], post_sz, chash, sha_b64, vfs.flags
+                            )
                         except:
                             self.log("bakflip failed: " + min_ex())
 
@@ -2404,13 +2438,13 @@ class HttpCli(object):
         finally:
             if locked:
                 # now block until all chunks released+confirmed
-                x = broker.ask("up2k.confirm_chunks", ptop, wark, locked)
+                x = broker.ask("up2k.confirm_chunks", ptop, wark, written, locked)
                 num_left, t = x.get()
                 if num_left < 0:
                     self.loud_reply(t, status=500)
                     return False
                 t = "got %d more chunks, %d left"
-                self.log(t % (len(locked), num_left), 6)
+                self.log(t % (len(written), num_left), 6)
 
         if num_left < 0:
             raise Pebkac(500, "unconfirmed; see serverlog")
@@ -3022,7 +3056,7 @@ class HttpCli(object):
             if ex.errno != errno.ENOENT:
                 raise
 
-        # if file exists, chekc that timestamp matches the client's
+        # if file exists, check that timestamp matches the client's
         if srv_lastmod >= 0:
             same_lastmod = cli_lastmod3 in [-1, srv_lastmod3]
             if not same_lastmod:
@@ -3233,7 +3267,7 @@ class HttpCli(object):
     )   :
         logues = ["", ""]
         if not self.args.no_logues:
-            for n, fn in enumerate([".prologue.html", ".epilogue.html"]):
+            for n, fn in LOGUES:
                 if lnames is not None and fn not in lnames:
                     continue
                 fn = "%s/%s" % (abspath, fn)
@@ -3245,25 +3279,31 @@ class HttpCli(object):
                             logues[n], vn.flags.get("exp_lg") or []
                         )
 
-        readme = ""
-        if not self.args.no_readme and not logues[1]:
-            if lnames is None:
-                fns = ["README.md", "readme.md"]
-            elif "readme.md" in lnames:
-                fns = [lnames["readme.md"]]
+        readmes = ["", ""]
+        for n, fns in [] if self.args.no_readme else READMES:
+            if logues[n]:
+                continue
+            elif lnames is None:
+                pass
+            elif fns[0] in lnames:
+                fns = [lnames[fns[0]]]
             else:
                 fns = []
 
+            txt = ""
             for fn in fns:
                 fn = "%s/%s" % (abspath, fn)
                 if bos.path.isfile(fn):
                     with open(fsenc(fn), "rb") as f:
-                        readme = f.read().decode("utf-8")
+                        txt = f.read().decode("utf-8")
                         break
-            if readme and "exp" in vn.flags:
-                readme = self._expand(readme, vn.flags.get("exp_md") or [])
 
-        return logues, readme
+            if txt and "exp" in vn.flags:
+                txt = self._expand(txt, vn.flags.get("exp_md") or [])
+
+            readmes[n] = txt
+
+        return logues, readmes
 
     def _expand(self, txt , phs )  :
         for ph in phs:
@@ -4763,7 +4803,7 @@ class HttpCli(object):
 
             fmt = fmt.format(len(nfmt.format(biggest)))
             retl = [
-                "# {}: {}".format(x, ls[x])
+                ("# %s: %s" % (x, ls[x])).replace(r"</span> // <span>", " // ")
                 for x in ["acct", "perms", "srvinf"]
                 if x in ls
             ]
@@ -5116,9 +5156,9 @@ class HttpCli(object):
             j2a["no_prism"] = True
 
         if not self.can_read and not is_dk:
-            logues, readme = self._add_logues(vn, abspath, None)
+            logues, readmes = self._add_logues(vn, abspath, None)
             ls_ret["logues"] = j2a["logues"] = logues
-            ls_ret["readme"] = cgv["readme"] = readme
+            ls_ret["readmes"] = cgv["readmes"] = readmes
 
             if is_ls:
                 return self.tx_ls(ls_ret)
@@ -5374,11 +5414,18 @@ class HttpCli(object):
         else:
             taglist = list(tagset)
 
-        logues, readme = self._add_logues(vn, abspath, lnames)
+        logues, readmes = self._add_logues(vn, abspath, lnames)
         ls_ret["logues"] = j2a["logues"] = logues
-        ls_ret["readme"] = cgv["readme"] = readme
+        ls_ret["readmes"] = cgv["readmes"] = readmes
 
-        if not files and not dirs and not readme and not logues[0] and not logues[1]:
+        if (
+            not files
+            and not dirs
+            and not readmes[0]
+            and not readmes[1]
+            and not logues[0]
+            and not logues[1]
+        ):
             logues[1] = "this folder is empty"
 
         if "descript.ion" in lnames and os.path.isfile(
@@ -5423,7 +5470,11 @@ class HttpCli(object):
         if doc:
             j2a["docname"] = doc
             doctxt = None
-            if next((x for x in files if x["name"] == doc), None):
+            dfn = lnames.get(doc.lower())
+            if dfn and dfn != doc:
+                # found Foo but want FOO
+                dfn = next((x for x in files if x["name"] == doc), None)
+            if dfn:
                 docpath = os.path.join(abspath, doc)
                 sz = bos.path.getsize(docpath)
                 if sz < 1024 * self.args.txt_max:

copyparty/httpconn.py

@@ -56,6 +56,8 @@ class HttpConn(object):
         self.asrv  = hsrv.asrv  # mypy404
         self.u2fh  = hsrv.u2fh  # mypy404
         self.pipes  = hsrv.pipes  # mypy404
+        self.ipu_iu   = hsrv.ipu_iu
+        self.ipu_nm  = hsrv.ipu_nm
         self.ipa_nm  = hsrv.ipa_nm
         self.xff_nm  = hsrv.xff_nm
         self.xff_lan  = hsrv.xff_lan  # type: ignore

copyparty/httpsrv.py

@@ -69,6 +69,7 @@ from .util import (
     build_netmap,
     has_resource,
     ipnorm,
+    load_ipu,
     load_resource,
     min_ex,
     shut_socket,
@@ -171,6 +172,11 @@ class HttpSrv(object):
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
         self.prism = has_resource(self.E, "web/deps/prism.js.gz")
 
+        if self.args.ipu:
+            self.ipu_iu, self.ipu_nm = load_ipu(self.log, self.args.ipu)
+        else:
+            self.ipu_iu = self.ipu_nm = None
+
         self.ipa_nm = build_netmap(self.args.ipa)
         self.xff_nm = build_netmap(self.args.xff_src)
         self.xff_lan = build_netmap("lan")

copyparty/mtag.py

@@ -467,7 +467,7 @@ class MTag(object):
                 sv = str(zv).split("/")[0].strip().lstrip("0")
                 ret[sk] = sv or 0
 
-        # normalize key notation to rkeobo
+        # normalize key notation to rekobo
         okey = ret.get("key")
         if okey:
             key = str(okey).replace(" ", "").replace("maj", "").replace("min", "m")

copyparty/ssdp.py

@@ -80,7 +80,7 @@ class SSDPr(object):
         name = self.args.doctitle
         zs = zs.strip().format(c(ubase), c(url), c(name), c(self.args.zsid))
         hc.reply(zs.encode("utf-8", "replace"))
-        return False  # close connectino
+        return False  # close connection
 
 
 class SSDPd(MCast):

copyparty/svchub.py

@@ -54,6 +54,7 @@ from .util import (
     alltrace,
     ansi_re,
     build_netmap,
+    load_ipu,
     min_ex,
     mp,
     odfusion,
@@ -215,6 +216,11 @@ class SvcHub(object):
             noch.update([x for x in zsl if x])
         args.chpw_no = noch
 
+        if args.ipu:
+            iu, nm = load_ipu(self.log, args.ipu)
+            setattr(args, "ipu_iu", iu)
+            setattr(args, "ipu_nm", nm)
+
         if not self.args.no_ses:
             self.setup_session_db()
 

copyparty/tcpsrv.py

@@ -92,7 +92,7 @@ class TcpSrv(object):
                                 continue
 
                             # binding 0.0.0.0 after :: fails on dualstack
-                            # but is necessary on non-dualstakc
+                            # but is necessary on non-dualstack
                             if successful_binds:
                                 continue