copyparty 1.16.7__py3-none-any.whl → 1.16.9__py3-none-any.whl

copyparty/__main__.py

@@ -54,6 +54,8 @@ from .util import (
     RAM_TOTAL,
     SQLITE_VER,
     UNPLICATIONS,
+    URL_BUG,
+    URL_PRJ,
     Daemon,
     align_tab,
     ansi_re,
@@ -326,17 +328,16 @@ def ensure_webdeps()  :
     if has_resource(E, "web/deps/mini-fa.woff"):
         return
 
-    warn(
-        """could not find webdeps;
+    t = """could not find webdeps;
   if you are running the sfx, or exe, or pypi package, or docker image,
   then this is a bug! Please let me know so I can fix it, thanks :-)
-  https://github.com/9001/copyparty/issues/new?labels=bug&template=bug_report.md
+  %s
 
   however, if you are a dev, or running copyparty from source, and you want
   full client functionality, you will need to build or obtain the webdeps:
-  https://github.com/9001/copyparty/blob/hovudstraum/docs/devnotes.md#building
+  %s/blob/hovudstraum/docs/devnotes.md#building
     """
-    )
+    warn(t % (URL_BUG, URL_PRJ))
 
 
 def configure_ssl_ver(al )  :
@@ -731,6 +732,10 @@ def get_sects():
               the \033[33m,,\033[35m stops copyparty from reading the rest as flags and
               the \033[33m--\033[35m stops notify-send from reading the message as args
               and the alert will be "hey" followed by the messagetext
+
+             \033[36m--xau zmq:pub:tcp://*:5556\033[35m announces uploads on zeromq;
+             \033[36m--xau t3,zmq:push:tcp://*:5557\033[35m also works, and you can
+             \033[36m--xau t3,j,zmq:req:tcp://localhost:5555\033[35m too for example
             \033[0m
             each hook is executed once for each event, except for \033[36mxiu\033[0m
             which builds up a backlog of uploads, running the hook just once
@@ -1468,12 +1473,14 @@ def add_ui(ap, retry):
     ap2.add_argument("--txt-max", metavar="KiB", type=int, default=64, help="max size of embedded textfiles on ?doc= (anything bigger will be lazy-loaded by JS)")
     ap2.add_argument("--doctitle", metavar="TXT", type=u, default="copyparty @ --name", help="title / service-name to show in html documents")
     ap2.add_argument("--bname", metavar="TXT", type=u, default="--name", help="server name (displayed in filebrowser document title)")
-    ap2.add_argument("--pb-url", metavar="URL", type=u, default="https://github.com/9001/copyparty", help="powered-by link; disable with \033[33m-np\033[0m")
+    ap2.add_argument("--pb-url", metavar="URL", type=u, default=URL_PRJ, help="powered-by link; disable with \033[33m-np\033[0m")
     ap2.add_argument("--ver", action="store_true", help="show version on the control panel (incompatible with \033[33m-nb\033[0m)")
     ap2.add_argument("--k304", metavar="NUM", type=int, default=0, help="configure the option to enable/disable k304 on the controlpanel (workaround for buggy reverse-proxies); [\033[32m0\033[0m] = hidden and default-off, [\033[32m1\033[0m] = visible and default-off, [\033[32m2\033[0m] = visible and default-on")
     ap2.add_argument("--no304", metavar="NUM", type=int, default=0, help="configure the option to enable/disable no304 on the controlpanel (workaround for buggy caching in browsers); [\033[32m0\033[0m] = hidden and default-off, [\033[32m1\033[0m] = visible and default-off, [\033[32m2\033[0m] = visible and default-on")
-    ap2.add_argument("--md-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for README.md docs (volflag=md_sbf); see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#attr-sandbox")
-    ap2.add_argument("--lg-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to ALLOW for prologue/epilogue docs  (volflag=lg_sbf)")
+    ap2.add_argument("--md-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to allow in the iframe 'sandbox' attribute for README.md docs (volflag=md_sbf); see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe#sandbox")
+    ap2.add_argument("--lg-sbf", metavar="FLAGS", type=u, default="downloads forms popups scripts top-navigation-by-user-activation", help="list of capabilities to allow in the iframe 'sandbox' attribute for prologue/epilogue docs (volflag=lg_sbf)")
+    ap2.add_argument("--md-sba", metavar="TXT", type=u, default="", help="the value of the iframe 'allow' attribute for README.md docs, for example [\033[32mfullscreen\033[0m] (volflag=md_sba)")
+    ap2.add_argument("--lg-sba", metavar="TXT", type=u, default="", help="the value of the iframe 'allow' attribute for prologue/epilogue docs (volflag=lg_sba); see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy#iframes")
     ap2.add_argument("--no-sb-md", action="store_true", help="don't sandbox README/PREADME.md documents (volflags: no_sb_md | sb_md)")
     ap2.add_argument("--no-sb-lg", action="store_true", help="don't sandbox prologue/epilogue docs (volflags: no_sb_lg | sb_lg); enables non-js support")
 

copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 16, 7)
+VERSION = (1, 16, 9)
 CODENAME = "COPYparty"
-BUILD_DT = (2024, 12, 23)
+BUILD_DT = (2025, 1, 22)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

copyparty/authsrv.py

@@ -1825,7 +1825,11 @@ class AuthSrv(object):
             if fka and not fk:
                 fk = fka
             if fk:
-                vol.flags["fk"] = int(fk) if fk is not True else 8
+                fk = 8 if fk is True else int(fk)
+                if fk > 72:
+                    t = "max filekey-length is 72; volume /%s specified %d (anything higher than 16 is pointless btw)"
+                    raise Exception(t % (vol.vpath, fk))
+                vol.flags["fk"] = fk
                 have_fk = True
 
             dk = vol.flags.get("dk")
@@ -2332,6 +2336,7 @@ class AuthSrv(object):
                 "frand": bool(vf.get("rand")),
                 "lifetime": vf.get("lifetime") or 0,
                 "unlist": vf.get("unlist") or "",
+                "sb_lg": "" if "no_sb_lg" in vf else (vf.get("lg_sbf") or "y"),
             }
             js_htm = {
                 "s_name": self.args.bname,
@@ -2344,6 +2349,8 @@ class AuthSrv(object):
                 "have_unpost": int(self.args.unpost),
                 "have_emp": self.args.emp,
                 "sb_md": "" if "no_sb_md" in vf else (vf.get("md_sbf") or "y"),
+                "sba_md": vf.get("md_sba") or "",
+                "sba_lg": vf.get("lg_sba") or "",
                 "txt_ext": self.args.textfiles.replace(",", " "),
                 "def_hcols": list(vf.get("mth") or []),
                 "unlist0": vf.get("unlist") or "",

copyparty/cfg.py

@@ -74,6 +74,8 @@ def vf_vmap()   :
         "html_head",
         "lg_sbf",
         "md_sbf",
+        "lg_sba",
+        "md_sba",
         "nrand",
         "og_desc",
         "og_site",
@@ -144,6 +146,7 @@ flagcats = {
         "noclone": "take dupe data from clients, even if available on HDD",
         "nodupe": "rejects existing files (instead of linking/cloning them)",
         "sparse": "force use of sparse files, mainly for s3-backed storage",
+        "nosparse": "deny use of sparse files, mainly for slow storage",
         "daw": "enable full WebDAV write support (dangerous);\nPUT-operations will now \033[1;31mOVERWRITE\033[0;35m existing files",
         "nosub": "forces all uploads into the top folder of the vfs",
         "magic": "enables filetype detection for nameless uploads",
@@ -240,6 +243,8 @@ flagcats = {
         "sb_lg": "enable js sandbox for prologue/epilogue (default)",
         "md_sbf": "list of markdown-sandbox safeguards to disable",
         "lg_sbf": "list of *logue-sandbox safeguards to disable",
+        "md_sba": "value of iframe allow-prop for markdown-sandbox",
+        "lg_sba": "value of iframe allow-prop for *logue-sandbox",
         "nohtml": "return html and markdown as text/html",
     },
     "others": {

copyparty/dxml.py

@@ -1,9 +1,16 @@
+# coding: utf-8
+from __future__ import print_function, unicode_literals
+
 import importlib
 import sys
 import xml.etree.ElementTree as ET
 
 from .__init__ import PY2
 
+class BadXML(Exception):
+    pass
+
+
 def get_ET()  :
     pn = "xml.etree.ElementTree"
     cn = "_elementtree"
@@ -30,7 +37,7 @@ def get_ET()  :
 XMLParser  = get_ET()
 
 
-class DXMLParser(XMLParser):  # type: ignore
+class _DXMLParser(XMLParser):  # type: ignore
     def __init__(self)  :
         tb = ET.TreeBuilder()
         super(DXMLParser, self).__init__(target=tb)
@@ -45,8 +52,12 @@ class DXMLParser(XMLParser):  # type: ignore
         raise BadXML("{}, {}".format(a, ka))
 
 
-class BadXML(Exception):
-    pass
+class _NG(XMLParser):  # type: ignore
+    def __int__(self)  :
+        raise BadXML("dxml selftest failed")
+
+
+DXMLParser = _DXMLParser
 
 
 def parse_xml(txt )  :
@@ -55,6 +66,40 @@ def parse_xml(txt )  :
     return parser.close()  # type: ignore
 
 
+def selftest()  :
+    qbe = r"""<!DOCTYPE d [
+<!ENTITY a "nice_bakuretsu">
+]>
+<root>&a;&a;&a;</root>"""
+
+    emb = r"""<!DOCTYPE d [
+<!ENTITY a SYSTEM "file:///etc/hostname">
+]>
+<root>&a;</root>"""
+
+    # future-proofing; there's never been any known vulns
+    # regarding DTDs and ET.XMLParser, but might as well
+    # block them since webdav-clients don't use them
+    dtd = r"""<!DOCTYPE d SYSTEM "a.dtd">
+<root>a</root>"""
+
+    for txt in (qbe, emb, dtd):
+        try:
+            parse_xml(txt)
+            t = "WARNING: dxml selftest failed:\n%s\n"
+            print(t % (txt,), file=sys.stderr)
+            return False
+        except BadXML:
+            pass
+
+    return True
+
+
+DXML_OK = selftest()
+if not DXML_OK:
+    DXMLParser = _NG
+
+
 def mktnod(name , text )  :
     el = ET.Element(name)
     el.text = text

copyparty/httpcli.py

@@ -128,6 +128,8 @@ NO_CACHE = {"Cache-Control": "no-cache"}
 
 ALL_COOKIES = "k304 no304 js idxh dots cppwd cppws".split()
 
+BADXFF = " due to dangerous misconfiguration (the http-header specified by --xff-hdr was received from an untrusted reverse-proxy)"
+
 H_CONN_KEEPALIVE = "Connection: Keep-Alive"
 H_CONN_CLOSE = "Connection: Close"
 
@@ -157,6 +159,8 @@ class HttpCli(object):
 
     def __init__(self, conn )  :
 
+        empty_stringlist  = []
+
         self.t0 = time.time()
         self.conn = conn
         self.u2mutex = conn.u2mutex  # mypy404
@@ -202,9 +206,7 @@ class HttpCli(object):
         self.trailing_slash = True
         self.uname = " "
         self.pw = " "
-        self.rvol = [" "]
-        self.wvol = [" "]
-        self.avol = [" "]
+        self.rvol = self.wvol = self.avol = empty_stringlist
         self.do_log = True
         self.can_read = False
         self.can_write = False
@@ -385,6 +387,7 @@ class HttpCli(object):
                     ) + "0.0/16"
                     zs2 = ' or "--xff-src=lan"' if self.conn.xff_lan.map(pip) else ""
                     self.log(t % (self.args.xff_hdr, pip, cli_ip, zso, zs, zs2), 3)
+                    self.bad_xff = True
                 else:
                     self.ip = cli_ip
                     self.is_vproxied = bool(self.args.R)
@@ -505,7 +508,7 @@ class HttpCli(object):
                 return False
 
             if "k" in uparam:
-                m = RE_K.search(uparam["k"])
+                m = re_k.search(uparam["k"])
                 if m:
                     zs = uparam["k"]
                     t = "malicious user; illegal filekey; req(%r) k(%r) => %r"
@@ -1889,7 +1892,7 @@ class HttpCli(object):
                 return self.handle_stash(False)
 
             if "save" in opt:
-                post_sz, _, _, _, path, _ = self.dump_to_file(False)
+                post_sz, _, _, _, _, path, _ = self.dump_to_file(False)
                 self.log("urlform: %d bytes, %r" % (post_sz, path))
             elif "print" in opt:
                 reader, _ = self.get_body_reader()
@@ -1970,11 +1973,11 @@ class HttpCli(object):
         else:
             return read_socket(self.sr, bufsz, remains), remains
 
-    def dump_to_file(self, is_put )       :
-        # post_sz, sha_hex, sha_b64, remains, path, url
+    def dump_to_file(self, is_put )        :
+        # post_sz, halg, sha_hex, sha_b64, remains, path, url
         reader, remains = self.get_body_reader()
         vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
-        rnd, _, lifetime, xbu, xau = self.upload_flags(vfs)
+        rnd, lifetime, xbu, xau = self.upload_flags(vfs)
         lim = vfs.get_dbv(rem)[0].lim
         fdir = vfs.canonical(rem)
         if lim:
@@ -2122,12 +2125,14 @@ class HttpCli(object):
                 # small toctou, but better than clobbering a hardlink
                 wunlink(self.log, path, vfs.flags)
 
+        halg = "sha512"
         hasher = None
         copier = hashcopy
         if "ck" in self.ouparam or "ck" in self.headers:
-            zs = self.ouparam.get("ck") or self.headers.get("ck") or ""
+            halg = zs = self.ouparam.get("ck") or self.headers.get("ck") or ""
             if not zs or zs == "no":
                 copier = justcopy
+                halg = ""
             elif zs == "md5":
                 hasher = hashlib.md5(**USED4SEC)
             elif zs == "sha1":
@@ -2161,7 +2166,7 @@ class HttpCli(object):
                 raise
 
         if self.args.nw:
-            return post_sz, sha_hex, sha_b64, remains, path, ""
+            return post_sz, halg, sha_hex, sha_b64, remains, path, ""
 
         at = mt = time.time() - lifetime
         cli_mt = self.headers.get("x-oc-mtime")
@@ -2272,19 +2277,30 @@ class HttpCli(object):
             self.args.RS + vpath + vsuf,
         )
 
-        return post_sz, sha_hex, sha_b64, remains, path, url
+        return post_sz, halg, sha_hex, sha_b64, remains, path, url
 
     def handle_stash(self, is_put )  :
-        post_sz, sha_hex, sha_b64, remains, path, url = self.dump_to_file(is_put)
+        post_sz, halg, sha_hex, sha_b64, remains, path, url = self.dump_to_file(is_put)
         spd = self._spd(post_sz)
         t = "%s wrote %d/%d bytes to %r  # %s"
         self.log(t % (spd, post_sz, remains, path, sha_b64[:28]))  # 21
 
-        ac = self.uparam.get(
-            "want", self.headers.get("accept", "").lower().split(";")[-1]
-        )
+        mime = "text/plain; charset=utf-8"
+        ac = self.uparam.get("want") or self.headers.get("accept") or ""
+        if ac:
+            ac = ac.split(";", 1)[0].lower()
+            if ac == "application/json":
+                ac = "json"
         if ac == "url":
             t = url
+        elif ac == "json" or "j" in self.uparam:
+            jmsg = {"fileurl": url, "filesz": post_sz}
+            if halg:
+                jmsg[halg] = sha_hex[:56]
+                jmsg["sha_b64"] = sha_b64
+
+            mime = "application/json"
+            t = json.dumps(jmsg, indent=2, sort_keys=True)
         else:
             t = "{}\n{}\n{}\n{}\n".format(post_sz, sha_b64, sha_hex[:56], url)
 
@@ -2294,7 +2310,7 @@ class HttpCli(object):
             h["X-OC-MTime"] = "accepted"
             t = ""  # some webdav clients expect/prefer this
 
-        self.reply(t.encode("utf-8"), 201, headers=h)
+        self.reply(t.encode("utf-8", "replace"), 201, mime=mime, headers=h)
         return True
 
     def bakflip(
@@ -2967,7 +2983,7 @@ class HttpCli(object):
         self.redirect(vpath, "?edit")
         return True
 
-    def upload_flags(self, vfs )      :
+    def upload_flags(self, vfs )     :
         if self.args.nw:
             rnd = 0
         else:
@@ -2975,10 +2991,6 @@ class HttpCli(object):
             if vfs.flags.get("rand"):  # force-enable
                 rnd = max(rnd, vfs.flags["nrand"])
 
-        ac = self.uparam.get(
-            "want", self.headers.get("accept", "").lower().split(";")[-1]
-        )
-        want_url = ac == "url"
         zs = self.uparam.get("life", self.headers.get("life", ""))
         if zs:
             vlife = vfs.flags.get("lifetime") or 0
@@ -2988,7 +3000,6 @@ class HttpCli(object):
 
         return (
             rnd,
-            want_url,
             lifetime,
             vfs.flags.get("xbu") or [],
             vfs.flags.get("xau") or [],
@@ -3041,7 +3052,14 @@ class HttpCli(object):
             if not nullwrite:
                 bos.makedirs(fdir_base)
 
-        rnd, want_url, lifetime, xbu, xau = self.upload_flags(vfs)
+        rnd, lifetime, xbu, xau = self.upload_flags(vfs)
+        zs = self.uparam.get("want") or self.headers.get("accept") or ""
+        if zs:
+            zs = zs.split(";", 1)[0].lower()
+            if zs == "application/json":
+                zs = "json"
+        want_url = zs == "url"
+        want_json = zs == "json" or "j" in self.uparam
 
         files       = []
         # sz, sha_hex, sha_b64, p_file, fname, abspath
@@ -3363,7 +3381,9 @@ class HttpCli(object):
                 msg += "\n" + errmsg
 
             self.reply(msg.encode("utf-8", "replace"), status=sc)
-        elif "j" in self.uparam:
+        elif want_json:
+            if len(jmsg["files"]) == 1:
+                jmsg["fileurl"] = jmsg["files"][0]["url"]
             jtxt = json.dumps(jmsg, indent=2, sort_keys=True).encode("utf-8", "replace")
             self.reply(jtxt, mime="application/json", status=sc)
         else:
@@ -4317,7 +4337,7 @@ class HttpCli(object):
             self.log,
             self.asrv,
             fgen,
-            utf8="utf" in uarg,
+            utf8="utf" in uarg or not uarg,
             pre_crc="crc" in uarg,
             cmp=uarg if cancmp or uarg == "pax" else "",
         )
@@ -4531,12 +4551,12 @@ class HttpCli(object):
             else self.conn.hsrv.nm.map(self.ip) or host
         )
         # safer than html_escape/quotep since this avoids both XSS and shell-stuff
-        pw = re.sub(r"[<>&$?`\"']", "_", self.pw or "pw")
+        pw = re.sub(r"[<>&$?`\"']", "_", self.pw or "hunter2")
         vp = re.sub(r"[<>&$?`\"']", "_", self.uparam["hc"] or "").lstrip("/")
         pw = pw.replace(" ", "%20")
         vp = vp.replace(" ", "%20")
         if pw in self.asrv.sesa:
-            pw = "pwd"
+            pw = "hunter2"
 
         html = self.j2s(
             "svcs",
@@ -4965,8 +4985,16 @@ class HttpCli(object):
             and (self.uname in vol.axs.uread or self.uname in vol.axs.upget)
         }
 
+        bad_xff = hasattr(self, "bad_xff")
+        if bad_xff:
+            allvols = []
+            t = "will not return list of recent uploads" + BADXFF
+            self.log(t, 1)
+            if self.avol:
+                raise Pebkac(500, t)
+
         x = self.conn.hsrv.broker.ask(
-            "up2k.get_unfinished_by_user", self.uname, self.ip
+            "up2k.get_unfinished_by_user", self.uname, "" if bad_xff else self.ip
         )
         uret = x.get()
 
@@ -5360,12 +5388,16 @@ class HttpCli(object):
         if self.args.no_del:
             raise Pebkac(403, "the delete feature is disabled in server config")
 
+        unpost = "unpost" in self.uparam
+        if unpost and hasattr(self, "bad_xff"):
+            self.log("unpost was denied" + BADXFF, 1)
+            raise Pebkac(403, "the delete feature is disabled in server config")
+
         if not req:
             req = [self.vpath]
         elif self.is_vproxied:
             req = [x[len(self.args.SR) :] for x in req]
 
-        unpost = "unpost" in self.uparam
         nlim = int(self.uparam.get("lim") or 0)
         lim = [nlim, nlim] if nlim else []
 
@@ -5765,7 +5797,7 @@ class HttpCli(object):
             "taglist": [],
             "have_tags_idx": int(e2t),
             "have_b_u": (self.can_write and self.uparam.get("b") == "u"),
-            "sb_lg": "" if "no_sb_lg" in vf else (vf.get("lg_sbf") or "y"),
+            "sb_lg": vn.js_ls["sb_lg"],
             "url_suf": url_suf,
             "title": html_escape("%s %s" % (self.args.bname, self.vpath), crlf=True),
             "srv_info": srv_infot,

copyparty/svchub.py

@@ -44,6 +44,8 @@ from .util import (
     FFMPEG_URL,
     HAVE_PSUTIL,
     HAVE_SQLITE3,
+    HAVE_ZMQ,
+    URL_BUG,
     UTC,
     VERSIONS,
     Daemon,
@@ -54,6 +56,7 @@ from .util import (
     alltrace,
     ansi_re,
     build_netmap,
+    expat_ver,
     load_ipu,
     min_ex,
     mp,
@@ -629,6 +632,7 @@ class SvcHub(object):
             (HAVE_FFPROBE, "ffprobe", t_ff + ", read audio/media tags"),
             (HAVE_MUTAGEN, "mutagen", "read audio tags (ffprobe is better but slower)"),
             (HAVE_ARGON2, "argon2", "secure password hashing (advanced users only)"),
+            (HAVE_ZMQ, "pyzmq", "send zeromq messages from event-hooks"),
             (HAVE_HEIF, "pillow-heif", "read .heif images with pillow (rarely useful)"),
             (HAVE_AVIF, "pillow-avif", "read .avif images with pillow (rarely useful)"),
         ]
@@ -685,6 +689,15 @@ class SvcHub(object):
             if self.args.bauth_last:
                 self.log("root", "WARNING: ignoring --bauth-last due to --no-bauth", 3)
 
+        if not self.args.no_dav:
+            from .dxml import DXML_OK
+
+            if not DXML_OK:
+                if not self.args.no_dav:
+                    self.args.no_dav = True
+                    t = "WARNING:\nDisabling WebDAV support because dxml selftest failed. Please report this bug;\n%s\n...and include the following information in the bug-report:\n%s | expat %s\n"
+                    self.log("root", t % (URL_BUG, VERSIONS, expat_ver()), 1)
+
     def _process_config(self)  :
         al = self.args
 

copyparty/up2k.py

@@ -790,7 +790,7 @@ class Up2k(object):
                 continue
 
             self.log("xiu: %d# %r" % (len(wrfs), cmd))
-            runihook(self.log, cmd, vol, ups)
+            runihook(self.log, self.args.hook_v, cmd, vol, ups)
 
     def _vis_job_progress(self, job  )  :
         perc = 100 - (len(job["need"]) * 100.0 / (len(job["hash"]) or 1))
@@ -851,7 +851,7 @@ class Up2k(object):
         self.iacct = self.asrv.iacct
         self.grps = self.asrv.grps
 
-        have_e2d = self.args.idp_h_usr
+        have_e2d = self.args.idp_h_usr or self.args.chpw or self.args.shr
         vols = list(all_vols.values())
         t0 = time.time()
 
@@ -1112,6 +1112,7 @@ class Up2k(object):
         reg = {}
         drp = None
         emptylist = []
+        dotpart = "." if self.args.dotpart else ""
         snap = os.path.join(histpath, "up2k.snap")
         if bos.path.exists(snap):
             with gzip.GzipFile(snap, "rb") as f:
@@ -1124,6 +1125,8 @@ class Up2k(object):
             except:
                 pass
 
+            reg = reg2  # diff-golf
+
             if reg2 and "dwrk" not in reg2[next(iter(reg2))]:
                 for job in reg2.values():
                     job["dwrk"] = job["wark"]
@@ -1131,7 +1134,8 @@ class Up2k(object):
             rm = []
             for k, job in reg2.items():
                 job["ptop"] = ptop
-                if "done" in job:
+                is_done = "done" in job
+                if is_done:
                     job["need"] = job["hash"] = emptylist
                 else:
                     if "need" not in job:
@@ -1139,10 +1143,13 @@ class Up2k(object):
                     if "hash" not in job:
                         job["hash"] = []
 
-                fp = djoin(ptop, job["prel"], job["name"])
+                if is_done:
+                    fp = djoin(ptop, job["prel"], job["name"])
+                else:
+                    fp = djoin(ptop, job["prel"], dotpart + job["name"] + ".PARTIAL")
+
                 if bos.path.exists(fp):
-                    reg[k] = job
-                    if "done" in job:
+                    if is_done:
                         continue
                     job["poke"] = time.time()
                     job["busy"] = {}
@@ -1150,11 +1157,18 @@ class Up2k(object):
                     self.log("ign deleted file in snap: %r" % (fp,))
                     if not n4g:
                         rm.append(k)
-                        continue
 
             for x in rm:
                 del reg2[x]
 
+            # optimize pre-1.15.4 entries
+            if next((x for x in reg.values() if "done" in x and "poke" in x), None):
+                zsl = "host tnam busy sprs poke t0c".split()
+                for job in reg.values():
+                    if "done" in job:
+                        for k in zsl:
+                            job.pop(k, None)
+
             if drp is None:
                 drp = [k for k, v in reg.items() if not v["need"]]
             else:
@@ -2989,7 +3003,7 @@ class Up2k(object):
                 if wark in reg:
                     del reg[wark]
                 job["hash"] = job["need"] = []
-                job["done"] = True
+                job["done"] = 1
                 job["busy"] = {}
 
             if lost:
@@ -4867,7 +4881,8 @@ class Up2k(object):
             except:
                 pass
 
-        xbu = self.flags[job["ptop"]].get("xbu")
+        vf = self.flags[job["ptop"]]
+        xbu = vf.get("xbu")
         ap_chk = djoin(pdir, job["name"])
         vp_chk = djoin(job["vtop"], job["prel"], job["name"])
         if xbu:
@@ -4897,7 +4912,7 @@ class Up2k(object):
                 if x:
                     zvfs = vfs
                     pdir, _, job["name"], (vfs, rem) = x
-                    job["vcfg"] = vfs.flags
+                    job["vcfg"] = vf = vfs.flags
                     job["ptop"] = vfs.realpath
                     job["vtop"] = vfs.vpath
                     job["prel"] = rem
@@ -4947,8 +4962,13 @@ class Up2k(object):
                 fs = self.fstab.get(pdir)
                 if fs == "ok":
                     pass
-                elif "sparse" in self.flags[job["ptop"]]:
-                    t = "volflag 'sparse' is forcing use of sparse files for uploads to [%s]"
+                elif "nosparse" in vf:
+                    t = "volflag 'nosparse' is preventing creation of sparse files for uploads to [%s]"
+                    self.log(t % (job["ptop"],))
+                    relabel = True
+                    sprs = False
+                elif "sparse" in vf:
+                    t = "volflag 'sparse' is forcing creation of sparse files for uploads to [%s]"
                     self.log(t % (job["ptop"],))
                     relabel = True
                 else: