copyparty 1.15.6__tar.gz → 1.15.8__tar.gz

{copyparty-1.15.6 → copyparty-1.15.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: copyparty
-Version: 1.15.6
+Version: 1.15.8
 Summary:   Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
 Author-email: ed <copyparty@ocv.me>
 License: MIT
@@ -134,6 +134,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [event hooks](#event-hooks) - trigger a program on uploads, renames etc ([examples](./bin/hooks/))
         * [upload events](#upload-events) - the older, more powerful approach ([examples](./bin/mtag/))
     * [handlers](#handlers) - redefine behavior with plugins ([examples](./bin/handlers/))
+    * [ip auth](#ip-auth) - autologin based on IP range (CIDR)
     * [identity providers](#identity-providers) - replace copyparty passwords with oauth and such
     * [user-changeable passwords](#user-changeable-passwords) - if permitted, users can change their own passwords
     * [using the cloud as storage](#using-the-cloud-as-storage) - connecting to an aws s3 bucket and similar
@@ -272,7 +273,7 @@ also see [comparison to similar software](./docs/versus.md)
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
-    * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
+    * **no filesize limit!** even on Cloudflare
   * ☑ stash: simple PUT filedropper
   * ☑ filename randomizer
   * ☑ write-only folders
@@ -707,7 +708,7 @@ up2k has several advantages:
   * uploads resume if you reboot your browser or pc, just upload the same files again
   * server detects any corruption; the client reuploads affected chunks
   * the client doesn't upload anything that already exists on the server
-  * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
+  * no filesize limit, even when a proxy limits the request size (for example Cloudflare)
 * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
 * the last-modified timestamp of the file is preserved
 
@@ -743,6 +744,8 @@ note that since up2k has to read each file twice, `[🎈] bup` can *theoreticall
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
+if the server is behind a proxy which imposes a request-size limit, you can configure up2k to sneak below the limit with server-option `--u2sz` (the default is 96 MiB to support Cloudflare)
+
 
 ### file-search
 
@@ -1486,6 +1489,22 @@ redefine behavior with plugins ([examples](./bin/handlers/))
 replace 404 and 403 errors with something completely different (that's it for now)
 
 
+## ip auth
+
+autologin based on IP range (CIDR)  , using the global-option `--ipu`
+
+for example, if everyone with an IP that starts with `192.168.123` should automatically log in as the user `spartacus`, then you can either specify `--ipu=192.168.123.0/24=spartacus` as a commandline option, or put this in a config file:
+
+```yaml
+[global]
+  ipu: 192.168.123.0/24=spartacus
+```
+
+repeat the option to map additional subnets
+
+**be careful with this one!** if you have a reverseproxy, then you definitely want to make sure you have [real-ip](#real-ip) configured correctly, and it's probably a good idea to nullmap the reverseproxy's IP just in case; so if your reverseproxy is sending requests from `172.24.27.9` then that would be `--ipu=172.24.27.9/32=`
+
+
 ## identity providers
 
 replace copyparty passwords with oauth and such

{copyparty-1.15.6 → copyparty-1.15.8}/README.md

@@ -80,6 +80,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [event hooks](#event-hooks) - trigger a program on uploads, renames etc ([examples](./bin/hooks/))
         * [upload events](#upload-events) - the older, more powerful approach ([examples](./bin/mtag/))
     * [handlers](#handlers) - redefine behavior with plugins ([examples](./bin/handlers/))
+    * [ip auth](#ip-auth) - autologin based on IP range (CIDR)
     * [identity providers](#identity-providers) - replace copyparty passwords with oauth and such
     * [user-changeable passwords](#user-changeable-passwords) - if permitted, users can change their own passwords
     * [using the cloud as storage](#using-the-cloud-as-storage) - connecting to an aws s3 bucket and similar
@@ -218,7 +219,7 @@ also see [comparison to similar software](./docs/versus.md)
 * upload
   * ☑ basic: plain multipart, ie6 support
   * ☑ [up2k](#uploading): js, resumable, multithreaded
-    * **no filesize limit!** ...unless you use Cloudflare, then it's 383.9 GiB
+    * **no filesize limit!** even on Cloudflare
   * ☑ stash: simple PUT filedropper
   * ☑ filename randomizer
   * ☑ write-only folders
@@ -653,7 +654,7 @@ up2k has several advantages:
   * uploads resume if you reboot your browser or pc, just upload the same files again
   * server detects any corruption; the client reuploads affected chunks
   * the client doesn't upload anything that already exists on the server
-  * no filesize limit unless imposed by a proxy, for example Cloudflare, which blocks uploads over 383.9 GiB
+  * no filesize limit, even when a proxy limits the request size (for example Cloudflare)
 * much higher speeds than ftp/scp/tarpipe on some internet connections (mainly american ones) thanks to parallel connections
 * the last-modified timestamp of the file is preserved
 
@@ -689,6 +690,8 @@ note that since up2k has to read each file twice, `[🎈] bup` can *theoreticall
 
 if you are resuming a massive upload and want to skip hashing the files which already finished, you can enable `turbo` in the `[⚙️] config` tab, but please read the tooltip on that button
 
+if the server is behind a proxy which imposes a request-size limit, you can configure up2k to sneak below the limit with server-option `--u2sz` (the default is 96 MiB to support Cloudflare)
+
 
 ### file-search
 
@@ -1432,6 +1435,22 @@ redefine behavior with plugins ([examples](./bin/handlers/))
 replace 404 and 403 errors with something completely different (that's it for now)
 
 
+## ip auth
+
+autologin based on IP range (CIDR)  , using the global-option `--ipu`
+
+for example, if everyone with an IP that starts with `192.168.123` should automatically log in as the user `spartacus`, then you can either specify `--ipu=192.168.123.0/24=spartacus` as a commandline option, or put this in a config file:
+
+```yaml
+[global]
+  ipu: 192.168.123.0/24=spartacus
+```
+
+repeat the option to map additional subnets
+
+**be careful with this one!** if you have a reverseproxy, then you definitely want to make sure you have [real-ip](#real-ip) configured correctly, and it's probably a good idea to nullmap the reverseproxy's IP just in case; so if your reverseproxy is sending requests from `172.24.27.9` then that would be `--ipu=172.24.27.9/32=`
+
+
 ## identity providers
 
 replace copyparty passwords with oauth and such

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/__main__.py

@@ -1009,7 +1009,7 @@ def add_upload(ap):
     ap2.add_argument("--sparse", metavar="MiB", type=int, default=4, help="windows-only: minimum size of incoming uploads through up2k before they are made into sparse files")
     ap2.add_argument("--turbo", metavar="LVL", type=int, default=0, help="configure turbo-mode in up2k client; [\033[32m-1\033[0m] = forbidden/always-off, [\033[32m0\033[0m] = default-off and warn if enabled, [\033[32m1\033[0m] = default-off, [\033[32m2\033[0m] = on, [\033[32m3\033[0m] = on and disable datecheck")
     ap2.add_argument("--u2j", metavar="JOBS", type=int, default=2, help="web-client: number of file chunks to upload in parallel; 1 or 2 is good for low-latency (same-country) connections, 4-8 for android clients, 16 for cross-atlantic (max=64)")
-    ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for this size. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
+    ap2.add_argument("--u2sz", metavar="N,N,N", type=u, default="1,64,96", help="web-client: default upload chunksize (MiB); sets \033[33mmin,default,max\033[0m in the settings gui. Each HTTP POST will aim for \033[33mdefault\033[0m, and never exceed \033[33mmax\033[0m. Cloudflare max is 96. Big values are good for cross-atlantic but may increase HDD fragmentation on some FS. Disable this optimization with [\033[32m1,1,1\033[0m]")
     ap2.add_argument("--u2sort", metavar="TXT", type=u, default="s", help="upload order; [\033[32ms\033[0m]=smallest-first, [\033[32mn\033[0m]=alphabetical, [\033[32mfs\033[0m]=force-s, [\033[32mfn\033[0m]=force-n -- alphabetical is a bit slower on fiber/LAN but makes it easier to eyeball if everything went fine")
     ap2.add_argument("--write-uplog", action="store_true", help="write POST reports to textfiles in working-directory")
 
@@ -1079,6 +1079,7 @@ def add_auth(ap):
     ap2.add_argument("--ses-db", metavar="PATH", type=u, default=ses_db, help="where to store the sessions database (if you run multiple copyparty instances, make sure they use different DBs)")
     ap2.add_argument("--ses-len", metavar="CHARS", type=int, default=20, help="session key length; default is 120 bits ((20//4)*4*6)")
     ap2.add_argument("--no-ses", action="store_true", help="disable sessions; use plaintext passwords in cookies")
+    ap2.add_argument("--ipu", metavar="CIDR=USR", type=u, action="append", help="users with IP matching \033[33mCIDR\033[0m are auto-authenticated as username \033[33mUSR\033[0m; example: [\033[32m172.16.24.0/24=dave]")
 
 
 def add_chpw(ap):
@@ -1469,6 +1470,7 @@ def add_debug(ap):
     ap2.add_argument("--bak-flips", action="store_true", help="[up2k] if a client uploads a bitflipped/corrupted chunk, store a copy according to \033[33m--bf-nc\033[0m and \033[33m--bf-dir\033[0m")
     ap2.add_argument("--bf-nc", metavar="NUM", type=int, default=200, help="bak-flips: stop if there's more than \033[33mNUM\033[0m files at \033[33m--kf-dir\033[0m already; default: 6.3 GiB max (200*32M)")
     ap2.add_argument("--bf-dir", metavar="PATH", type=u, default="bf", help="bak-flips: store corrupted chunks at \033[33mPATH\033[0m; default: folder named 'bf' wherever copyparty was started")
+    ap2.add_argument("--bf-log", metavar="PATH", type=u, default="", help="bak-flips: log corruption info to a textfile at \033[33mPATH\033[0m")
 
 
 # fmt: on

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 15, 6)
+VERSION = (1, 15, 8)
 CODENAME = "fill the drives"
-BUILD_DT = (2024, 10, 12)
+BUILD_DT = (2024, 10, 16)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/authsrv.py

@@ -59,6 +59,7 @@ if PY2:
 LEELOO_DALLAS = "leeloo_dallas"
 
 SEE_LOG = "see log for details"
+SEESLOG = " (see serverlog for details)"
 SSEELOG = " ({})".format(SEE_LOG)
 BAD_CFG = "invalid config; {}".format(SEE_LOG)
 SBADCFG = " ({})".format(BAD_CFG)

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/ftpd.py

@@ -72,6 +72,7 @@ class FtpAuth(DummyAuthorizer):
             else:
                 raise AuthenticationFailed("banned")
 
+        args = self.hub.args
         asrv = self.hub.asrv
         uname = "*"
         if username != "anonymous":
@@ -82,6 +83,9 @@ class FtpAuth(DummyAuthorizer):
                     uname = zs
                     break
 
+        if args.ipu and uname == "*":
+            uname = args.ipu_iu[args.ipu_nm.map(ip)]
+
         if not uname or not (asrv.vfs.aread.get(uname) or asrv.vfs.awrite.get(uname)):
             g = self.hub.gpwd
             if g.lim:

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/httpcli.py

@@ -584,6 +584,9 @@ class HttpCli(object):
                 or "*"
             )
 
+        if self.args.ipu and self.uname == "*":
+            self.uname = self.conn.ipu_iu[self.conn.ipu_nm.map(self.ip)]
+
         self.rvol = self.asrv.vfs.aread[self.uname]
         self.wvol = self.asrv.vfs.awrite[self.uname]
         self.avol = self.asrv.vfs.aadmin[self.uname]
@@ -1871,7 +1874,7 @@ class HttpCli(object):
         f, fn = ren_open(fn, *open_a, **params)
         try:
             path = os.path.join(fdir, fn)
-            post_sz, sha_hex, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
+            post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
         finally:
             f.close()
 
@@ -2023,13 +2026,32 @@ class HttpCli(object):
         return True
 
     def bakflip(
-        self, f , ofs , sz , sha , flags  
+        self,
+        f ,
+        ap ,
+        ofs ,
+        sz ,
+        good_sha ,
+        bad_sha ,
+        flags  ,
     )  :
+        now = time.time()
+        t = "bad-chunk:  %.3f  %s  %s  %d  %s  %s  %s"
+        t = t % (now, bad_sha, good_sha, ofs, self.ip, self.uname, ap)
+        self.log(t, 5)
+
+        if self.args.bf_log:
+            try:
+                with open(self.args.bf_log, "ab+") as f2:
+                    f2.write((t + "\n").encode("utf-8", "replace"))
+            except Exception as ex:
+                self.log("append %s failed: %r" % (self.args.bf_log, ex))
+
         if not self.args.bak_flips or self.args.nw:
             return
 
         sdir = self.args.bf_dir
-        fp = os.path.join(sdir, sha)
+        fp = os.path.join(sdir, bad_sha)
         if bos.path.exists(fp):
             return self.log("no bakflip; have it", 6)
 
@@ -2315,7 +2337,7 @@ class HttpCli(object):
         broker = self.conn.hsrv.broker
         x = broker.ask("up2k.handle_chunks", ptop, wark, chashes)
         response = x.get()
-        chashes, chunksize, cstarts, path, lastmod, sprs = response
+        chashes, chunksize, cstarts, path, lastmod, fsize, sprs = response
         maxsize = chunksize * len(chashes)
         cstart0 = cstarts[0]
         locked = chashes  # remaining chunks to be received in this request
@@ -2323,6 +2345,50 @@ class HttpCli(object):
         num_left = -1  # num chunks left according to most recent up2k release
         treport = time.time()  # ratelimit up2k reporting to reduce overhead
 
+        if "x-up2k-subc" in self.headers:
+            sc_ofs = int(self.headers["x-up2k-subc"])
+            chash = chashes[0]
+
+            u2sc = self.conn.hsrv.u2sc
+            try:
+                sc_pofs, hasher = u2sc[chash]
+                if not sc_ofs:
+                    t = "client restarted the chunk; forgetting subchunk offset %d"
+                    self.log(t % (sc_pofs,))
+                    raise Exception()
+            except:
+                sc_pofs = 0
+                hasher = hashlib.sha512()
+
+            et = "subchunk protocol error; resetting chunk "
+            if sc_pofs != sc_ofs:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: the expected resume-point was %d, not %d"
+                raise Pebkac(400, t % (et, chash, sc_pofs, sc_ofs))
+            if len(cstarts) > 1:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: only a single subchunk can be uploaded in one request; you are sending %d chunks"
+                raise Pebkac(400, t % (et, chash, len(cstarts)))
+            csize = min(chunksize, fsize - cstart0[0])
+            cstart0[0] += sc_ofs  # also sets cstarts[0][0]
+            sc_next_ofs = sc_ofs + postsize
+            if sc_next_ofs > csize:
+                u2sc.pop(chash, None)
+                t = "%s[%s]: subchunk offset (%d) plus postsize (%d) exceeds chunksize (%d)"
+                raise Pebkac(400, t % (et, chash, sc_ofs, postsize, csize))
+            else:
+                final_subchunk = sc_next_ofs == csize
+                t = "subchunk %s %d:%d/%d %s"
+                zs = "END" if final_subchunk else ""
+                self.log(t % (chash[:15], sc_ofs, sc_next_ofs, csize, zs), 6)
+                if final_subchunk:
+                    u2sc.pop(chash, None)
+                else:
+                    u2sc[chash] = (sc_next_ofs, hasher)
+        else:
+            hasher = None
+            final_subchunk = True
+
         try:
             if self.args.nw:
                 path = os.devnull
@@ -2353,11 +2419,15 @@ class HttpCli(object):
                     reader = read_socket(
                         self.sr, self.args.s_rd_sz, min(remains, chunksize)
                     )
-                    post_sz, _, sha_b64 = hashcopy(reader, f, self.args.s_wr_slp)
+                    post_sz, _, sha_b64 = hashcopy(
+                        reader, f, hasher, 0, self.args.s_wr_slp
+                    )
 
-                    if sha_b64 != chash:
+                    if sha_b64 != chash and final_subchunk:
                         try:
-                            self.bakflip(f, cstart[0], post_sz, sha_b64, vfs.flags)
+                            self.bakflip(
+                                f, path, cstart[0], post_sz, chash, sha_b64, vfs.flags
+                            )
                         except:
                             self.log("bakflip failed: " + min_ex())
 
@@ -2385,7 +2455,8 @@ class HttpCli(object):
 
                     # be quick to keep the tcp winsize scale;
                     # if we can't confirm rn then that's fine
-                    written.append(chash)
+                    if final_subchunk:
+                        written.append(chash)
                     now = time.time()
                     if now - treport < 1:
                         continue
@@ -2773,7 +2844,7 @@ class HttpCli(object):
                         tabspath = os.path.join(fdir, tnam)
                         self.log("writing to {}".format(tabspath))
                         sz, sha_hex, sha_b64 = hashcopy(
-                            p_data, f, self.args.s_wr_slp, max_sz
+                            p_data, f, None, max_sz, self.args.s_wr_slp
                         )
                         if sz == 0:
                             raise Pebkac(400, "empty files in post")
@@ -3103,7 +3174,7 @@ class HttpCli(object):
             wunlink(self.log, fp, vfs.flags)
 
         with open(fsenc(fp), "wb", self.args.iobuf) as f:
-            sz, sha512, _ = hashcopy(p_data, f, self.args.s_wr_slp)
+            sz, sha512, _ = hashcopy(p_data, f, None, 0, self.args.s_wr_slp)
 
         if lim:
             lim.nup(self.ip)

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/httpconn.py

@@ -56,6 +56,8 @@ class HttpConn(object):
         self.asrv  = hsrv.asrv  # mypy404
         self.u2fh  = hsrv.u2fh  # mypy404
         self.pipes  = hsrv.pipes  # mypy404
+        self.ipu_iu   = hsrv.ipu_iu
+        self.ipu_nm  = hsrv.ipu_nm
         self.ipa_nm  = hsrv.ipa_nm
         self.xff_nm  = hsrv.xff_nm
         self.xff_lan  = hsrv.xff_lan  # type: ignore

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/httpsrv.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
+import hashlib
 import math
 import os
 import re
@@ -69,6 +70,7 @@ from .util import (
     build_netmap,
     has_resource,
     ipnorm,
+    load_ipu,
     load_resource,
     min_ex,
     shut_socket,
@@ -140,6 +142,7 @@ class HttpSrv(object):
         self.t_periodic  = None
 
         self.u2fh = FHC()
+        self.u2sc    = {}
         self.pipes = CachedDict(0.2)
         self.metrics = Metrics(self)
         self.nreq = 0
@@ -171,6 +174,11 @@ class HttpSrv(object):
         self.j2 = {x: env.get_template(x + ".html") for x in jn}
         self.prism = has_resource(self.E, "web/deps/prism.js.gz")
 
+        if self.args.ipu:
+            self.ipu_iu, self.ipu_nm = load_ipu(self.log, self.args.ipu)
+        else:
+            self.ipu_iu = self.ipu_nm = None
+
         self.ipa_nm = build_netmap(self.args.ipa)
         self.xff_nm = build_netmap(self.args.xff_src)
         self.xff_lan = build_netmap("lan")

{copyparty-1.15.6 → copyparty-1.15.8}/copyparty/svchub.py

@@ -54,6 +54,7 @@ from .util import (
     alltrace,
     ansi_re,
     build_netmap,
+    load_ipu,
     min_ex,
     mp,
     odfusion,
@@ -215,6 +216,11 @@ class SvcHub(object):
             noch.update([x for x in zsl if x])
         args.chpw_no = noch
 
+        if args.ipu:
+            iu, nm = load_ipu(self.log, args.ipu)
+            setattr(args, "ipu_iu", iu)
+            setattr(args, "ipu_nm", nm)
+
         if not self.args.no_ses:
             self.setup_session_db()