copyparty 1.10.2__py3-none-any.whl → 1.11.1__py3-none-any.whl

copyparty/up2k.py

@@ -196,11 +196,16 @@ class Up2k(object):
 
         Daemon(self.deferred_init, "up2k-deferred-init")
 
-    def reload(self)  :
-        self.gid += 1
-        self.log("reload #{} initiated".format(self.gid))
+    def reload(self, rescan_all_vols )  :
+        """mutex me"""
+        self.log("reload #{} scheduled".format(self.gid + 1))
         all_vols = self.asrv.vfs.all_vols
-        self.rescan(all_vols, list(all_vols.keys()), True, False)
+
+        scan_vols = [k for k, v in all_vols.items() if v.realpath not in self.registry]
+        if rescan_all_vols:
+            scan_vols = list(all_vols.keys())
+
+        self._rescan(all_vols, scan_vols, True, False)
 
     def deferred_init(self)  :
         all_vols = self.asrv.vfs.all_vols
@@ -229,8 +234,6 @@ class Up2k(object):
                 for n in range(max(1, self.args.mtag_mt)):
                     Daemon(self._tagger, "tagger-{}".format(n))
 
-                Daemon(self._run_all_mtp, "up2k-mtp-init")
-
     def log(self, msg , c   = 0)  :
         if self.pp:
             msg += "\033[K"
@@ -279,9 +282,48 @@ class Up2k(object):
         }
         return json.dumps(ret, indent=4)
 
+    def get_unfinished_by_user(self, uname, ip)  :
+        if PY2 or not self.mutex.acquire(timeout=2):
+            return '[{"timeout":1}]'
+
+        ret      = []
+        try:
+            for ptop, tab2 in self.registry.items():
+                cfg = self.flags.get(ptop, {}).get("u2abort", 1)
+                if not cfg:
+                    continue
+                addr = (ip or "\n") if cfg in (1, 2) else ""
+                user = (uname or "\n") if cfg in (1, 3) else ""
+                drp = self.droppable.get(ptop, {})
+                for wark, job in tab2.items():
+                    if (
+                        wark in drp
+                        or (user and user != job["user"])
+                        or (addr and addr != job["addr"])
+                    ):
+                        continue
+
+                    zt5 = (
+                        int(job["t0"]),
+                        djoin(job["vtop"], job["prel"], job["name"]),
+                        job["size"],
+                        len(job["need"]),
+                        len(job["hash"]),
+                    )
+                    ret.append(zt5)
+        finally:
+            self.mutex.release()
+
+        ret.sort(reverse=True)
+        ret2 = [
+            {"at": at, "vp": "/" + vp, "pd": 100 - ((nn * 100) // (nh or 1)), "sz": sz}
+            for (at, vp, sz, nn, nh) in ret
+        ]
+        return json.dumps(ret2, indent=0)
+
     def get_unfinished(self)  :
         if PY2 or not self.mutex.acquire(timeout=0.5):
-            return "{}"
+            return ""
 
         ret    = {}
         try:
@@ -334,14 +376,21 @@ class Up2k(object):
     def rescan(
         self, all_vols  , scan_vols , wait , fscan 
     )  :
+        with self.mutex:
+            return self._rescan(all_vols, scan_vols, wait, fscan)
+
+    def _rescan(
+        self, all_vols  , scan_vols , wait , fscan 
+    )  :
+        """mutex me"""
         if not wait and self.pp:
             return "cannot initiate; scan is already in progress"
 
-        args = (all_vols, scan_vols, fscan)
+        self.gid += 1
         Daemon(
             self.init_indexes,
             "up2k-rescan-{}".format(scan_vols[0] if scan_vols else "all"),
-            args,
+            (all_vols, scan_vols, fscan, self.gid),
         )
         return ""
 
@@ -453,7 +502,7 @@ class Up2k(object):
                     if vp:
                         fvp = "%s/%s" % (vp, fvp)
 
-                    self._handle_rm(LEELOO_DALLAS, "", fvp, [], True)
+                    self._handle_rm(LEELOO_DALLAS, "", fvp, [], True, False)
                     nrm += 1
 
             if nrm:
@@ -572,19 +621,32 @@ class Up2k(object):
         return True, ret
 
     def init_indexes(
-        self, all_vols  , scan_vols , fscan 
+        self, all_vols  , scan_vols , fscan , gid  = 0
     )  :
-        gid = self.gid
-        while self.pp and gid == self.gid:
-            time.sleep(0.1)
+        if not gid:
+            with self.mutex:
+                gid = self.gid
 
-        if gid != self.gid:
-            return False
+        nspin = 0
+        while True:
+            nspin += 1
+            if nspin > 1:
+                time.sleep(0.1)
+
+            with self.mutex:
+                if gid != self.gid:
+                    return False
+
+                if self.pp:
+                    continue
+
+                self.pp = ProgressPrinter(self.log, self.args)
+
+            break
 
         if gid:
-            self.log("reload #{} running".format(self.gid))
+            self.log("reload #%d running" % (gid,))
 
-        self.pp = ProgressPrinter(self.log, self.args)
         vols = list(all_vols.values())
         t0 = time.time()
         have_e2d = False
@@ -768,20 +830,14 @@ class Up2k(object):
             msg = "could not read tags because no backends are available (Mutagen or FFprobe)"
             self.log(msg, c=1)
 
-        thr = None
-        if self.mtag:
-            t = "online (running mtp)"
-            if scan_vols:
-                thr = Daemon(self._run_all_mtp, "up2k-mtp-scan", r=False)
-        else:
-            self.pp = None
-            t = "online, idle"
-
+        t = "online (running mtp)" if self.mtag else "online, idle"
         for vol in vols:
             self.volstate[vol.vpath] = t
 
-        if thr:
-            thr.start()
+        if self.mtag:
+            Daemon(self._run_all_mtp, "up2k-mtp-scan", (gid,))
+        else:
+            self.pp = None
 
         return have_e2d
 
@@ -1806,26 +1862,28 @@ class Up2k(object):
         self.pending_tags = []
         return ret
 
-    def _run_all_mtp(self)  :
-        gid = self.gid
+    def _run_all_mtp(self, gid )  :
         t0 = time.time()
         for ptop, flags in self.flags.items():
             if "mtp" in flags:
                 if ptop not in self.entags:
                     t = "skipping mtp for unavailable volume {}"
                     self.log(t.format(ptop), 1)
-                    continue
-                self._run_one_mtp(ptop, gid)
+                else:
+                    self._run_one_mtp(ptop, gid)
+
+            vtop = "\n"
+            for vol in self.asrv.vfs.all_vols.values():
+                if vol.realpath == ptop:
+                    vtop = vol.vpath
+            if "running mtp" in self.volstate.get(vtop, ""):
+                self.volstate[vtop] = "online, idle"
 
         td = time.time() - t0
         msg = "mtp finished in {:.2f} sec ({})"
         self.log(msg.format(td, s2hms(td, True)))
 
         self.pp = None
-        for k in list(self.volstate.keys()):
-            if "OFFLINE" not in self.volstate[k]:
-                self.volstate[k] = "online, idle"
-
         if self.args.exit == "idx":
             self.hub.sigterm()
 
@@ -2668,6 +2726,9 @@ class Up2k(object):
                             a = [job[x] for x in zs.split()]
                             self.db_add(cur, vfs.flags, *a)
                             cur.connection.commit()
+                elif wark in reg:
+                    # checks out, but client may have hopped IPs
+                    job["addr"] = cj["addr"]
 
             if not job:
                 ap1 = djoin(cj["ptop"], cj["prel"])
@@ -3204,7 +3265,13 @@ class Up2k(object):
                 pass
 
     def handle_rm(
-        self, uname , ip , vpaths , lim , rm_up 
+        self,
+        uname ,
+        ip ,
+        vpaths ,
+        lim ,
+        rm_up ,
+        unpost ,
     )  :
         n_files = 0
         ok = {}
@@ -3214,7 +3281,7 @@ class Up2k(object):
                 self.log("hit delete limit of {} files".format(lim[1]), 3)
                 break
 
-            a, b, c = self._handle_rm(uname, ip, vp, lim, rm_up)
+            a, b, c = self._handle_rm(uname, ip, vp, lim, rm_up, unpost)
             n_files += a
             for k in b:
                 ok[k] = 1
@@ -3228,25 +3295,43 @@ class Up2k(object):
         return "deleted {} files (and {}/{} folders)".format(n_files, iok, iok + ing)
 
     def _handle_rm(
-        self, uname , ip , vpath , lim , rm_up 
+        self, uname , ip , vpath , lim , rm_up , unpost 
     )    :
         self.db_act = time.time()
-        try:
+        partial = ""
+        if not unpost:
             permsets = [[True, False, False, True]]
             vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
             vn, rem = vn.get_dbv(rem)
-            unpost = False
-        except:
+        else:
             # unpost with missing permissions? verify with db
-            if not self.args.unpost:
-                raise Pebkac(400, "the unpost feature is disabled in server config")
-
-            unpost = True
             permsets = [[False, True]]
             vn, rem = self.asrv.vfs.get(vpath, uname, *permsets[0])
             vn, rem = vn.get_dbv(rem)
+            ptop = vn.realpath
             with self.mutex:
-                _, _, _, _, dip, dat = self._find_from_vpath(vn.realpath, rem)
+                abrt_cfg = self.flags.get(ptop, {}).get("u2abort", 1)
+                addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
+                user = (uname or "\n") if abrt_cfg in (1, 3) else ""
+                reg = self.registry.get(ptop, {}) if abrt_cfg else {}
+                for wark, job in reg.items():
+                    if (user and user != job["user"]) or (addr and addr != job["addr"]):
+                        continue
+                    if djoin(job["prel"], job["name"]) == rem:
+                        if job["ptop"] != ptop:
+                            t = "job.ptop [%s] != vol.ptop [%s] ??"
+                            raise Exception(t % (job["ptop"] != ptop))
+                        partial = vn.canonical(vjoin(job["prel"], job["tnam"]))
+                        break
+                if partial:
+                    dip = ip
+                    dat = time.time()
+                else:
+                    if not self.args.unpost:
+                        t = "the unpost feature is disabled in server config"
+                        raise Pebkac(400, t)
+
+                    _, _, _, _, dip, dat = self._find_from_vpath(ptop, rem)
 
             t = "you cannot delete this: "
             if not dip:
@@ -3339,6 +3424,9 @@ class Up2k(object):
                             cur.connection.commit()
 
                 wunlink(self.log, abspath, dbv.flags)
+                if partial:
+                    wunlink(self.log, partial, dbv.flags)
+                    partial = ""
                 if xad:
                     runhook(
                         self.log,
@@ -3934,7 +4022,13 @@ class Up2k(object):
 
             if not ANYWIN and sprs and sz > 1024 * 1024:
                 fs = self.fstab.get(pdir)
-                if fs != "ok":
+                if fs == "ok":
+                    pass
+                elif "sparse" in self.flags[job["ptop"]]:
+                    t = "volflag 'sparse' is forcing use of sparse files for uploads to [%s]"
+                    self.log(t % (job["ptop"],))
+                    relabel = True
+                else:
                     relabel = True
                     f.seek(1024 * 1024 - 1)
                     f.write(b"e")

copyparty/util.py

@@ -165,7 +165,7 @@ else:
 
 SYMTIME = sys.version_info > (3, 6) and os.utime in os.supports_follow_symlinks
 
-META_NOBOTS = '<meta name="robots" content="noindex, nofollow">'
+META_NOBOTS = '<meta name="robots" content="noindex, nofollow">\n'
 
 FFMPEG_URL = "https://www.gyan.dev/ffmpeg/builds/ffmpeg-git-full.7z"
 
@@ -538,20 +538,26 @@ class HLog(logging.Handler):
 
 
 class NetMap(object):
-    def __init__(self, ips , netdevs  )  :
+    def __init__(self, ips , cidrs , keep_lo=False)  :
+        """
+        ips: list of plain ipv4/ipv6 IPs, not cidr
+        cidrs: list of cidr-notation IPs (ip/prefix)
+        """
         if "::" in ips:
             ips = [x for x in ips if x != "::"] + list(
-                [x.split("/")[0] for x in netdevs if ":" in x]
+                [x.split("/")[0] for x in cidrs if ":" in x]
             )
             ips.append("0.0.0.0")
 
         if "0.0.0.0" in ips:
             ips = [x for x in ips if x != "0.0.0.0"] + list(
-                [x.split("/")[0] for x in netdevs if ":" not in x]
+                [x.split("/")[0] for x in cidrs if ":" not in x]
             )
 
-        ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
-        ips = find_prefix(ips, netdevs)
+        if not keep_lo:
+            ips = [x for x in ips if x not in ("::1", "127.0.0.1")]
+
+        ips = find_prefix(ips, cidrs)
 
         self.cache   = {}
         self.b2sip   = {}
@@ -568,6 +574,9 @@ class NetMap(object):
         self.bip.sort(reverse=True)
 
     def map(self, ip )  :
+        if ip.startswith("::ffff:"):
+            ip = ip[7:]
+
         try:
             return self.cache[ip]
         except:
@@ -1899,10 +1908,10 @@ def ipnorm(ip )  :
     return ip
 
 
-def find_prefix(ips , netdevs  )  :
+def find_prefix(ips , cidrs )  :
     ret = []
     for ip in ips:
-        hit = next((x for x in netdevs if x.startswith(ip + "/")), None)
+        hit = next((x for x in cidrs if x.startswith(ip + "/") or ip == x), None)
         if hit:
             ret.append(hit)
     return ret
@@ -2296,6 +2305,41 @@ def list_ips()  :
     return list(ret)
 
 
+def build_netmap(csv ):
+    csv = csv.lower().strip()
+
+    if csv in ("any", "all", "no", ",", ""):
+        return None
+
+    if csv in ("lan", "local", "private", "prvt"):
+        csv = "10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, fd00::/8"  # lan
+        csv += ", 169.254.0.0/16, fe80::/10"  # link-local
+        csv += ", 127.0.0.0/8, ::1/128"  # loopback
+
+    srcs = [x.strip() for x in csv.split(",") if x.strip()]
+    cidrs = []
+    for zs in srcs:
+        if not zs.endswith("."):
+            cidrs.append(zs)
+            continue
+
+        # translate old syntax "172.19." => "172.19.0.0/16"
+        words = len(zs.rstrip(".").split("."))
+        if words == 1:
+            zs += "0.0.0/8"
+        elif words == 2:
+            zs += "0.0/16"
+        elif words == 3:
+            zs += "0/24"
+        else:
+            raise Exception("invalid config value [%s]" % (zs,))
+
+        cidrs.append(zs)
+
+    ips = [x.split("/")[0] for x in cidrs]
+    return NetMap(ips, cidrs, True)
+
+
 def yieldfile(fn )    :
     with open(fsenc(fn), "rb", 512 * 1024) as f:
         while True:

copyparty/web/browser.html

@@ -7,9 +7,9 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8, minimum-scale=0.6">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/browser.css?_={{ ts }}">
+{{ html_head }}
 	{%- if css %}
 	<link rel="stylesheet" media="screen" href="{{ css }}_={{ ts }}">
 	{%- endif %}

copyparty/web/browser2.html

@@ -6,12 +6,12 @@
 	<title>{{ title }}</title>
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
-{{ html_head }}
 	<style>
 		html{font-family:sans-serif}
 		td{border:1px solid #999;border-width:1px 1px 0 0;padding:0 5px}
 		a{display:block}
 	</style>
+{{ html_head }}
 </head>
 
 <body>

copyparty/web/md.html

@@ -4,12 +4,12 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/md.css?_={{ ts }}">
 	{%- if edit %}
 	<link rel="stylesheet" href="{{ r }}/.cpr/md2.css?_={{ ts }}">
 	{%- endif %}
+{{ html_head }}
 </head>
 <body>
 	<div id="mn"></div>

copyparty/web/mde.html

@@ -4,11 +4,11 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.7">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/mde.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/deps/mini-fa.css?_={{ ts }}">
 	<link rel="stylesheet" href="{{ r }}/.cpr/deps/easymde.css?_={{ ts }}">
+{{ html_head }}
 </head>
 <body>
 	<div id="mw">

copyparty/web/msg.html

@@ -7,8 +7,8 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/msg.css?_={{ ts }}">
+{{ html_head }}
 </head>
 
 <body>

copyparty/web/splash.html

@@ -7,9 +7,9 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/splash.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
+{{ html_head }}
 </head>
 
 <body>
@@ -78,13 +78,15 @@
 
 		<h1 id="cc">client config:</h1>
 		<ul>
+			{% if k304 or k304vis %}
 			{% if k304 %}
 			<li><a id="h" href="{{ r }}/?k304=n">disable k304</a> (currently enabled)
 			{%- else %}
 			<li><a id="i" href="{{ r }}/?k304=y" class="r">enable k304</a> (currently disabled)
 			{% endif %}
 			<blockquote id="j">enabling this will disconnect your client on every HTTP 304, which can prevent some buggy proxies from getting stuck (suddenly not loading pages), <em>but</em> it will also make things slower in general</blockquote></li>
-			
+			{% endif %}
+
 			<li><a id="k" href="{{ r }}/?reset" class="r" onclick="localStorage.clear();return true">reset client settings</a></li>
 		</ul>
 

copyparty/web/svcs.html

@@ -7,10 +7,10 @@
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
 	<meta name="viewport" content="width=device-width, initial-scale=0.8">
 	<meta name="theme-color" content="#333">
-{{ html_head }}
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/splash.css?_={{ ts }}">
 	<link rel="stylesheet" media="screen" href="{{ r }}/.cpr/ui.css?_={{ ts }}">
 	<style>ul{padding-left:1.3em}li{margin:.4em 0}</style>
+{{ html_head }}
 </head>
 
 <body>