copyparty 1.12.2__tar.gz → 1.13.0__tar.gz

{copyparty-1.12.2 → copyparty-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: copyparty
-Version: 1.12.2
+Version: 1.13.0
 Summary:   Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
 Author-email: ed <copyparty@ocv.me>
 License: MIT
@@ -64,6 +64,8 @@ turn almost any device into a file server with resumable uploads/downloads using
 
 📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer)
 
+🎬 **videos:** [upload](https://a.ocv.me/pub/demo/pics-vids/up2k.webm) // [cli-upload](https://a.ocv.me/pub/demo/pics-vids/u2cli.webm) // [race-the-beam](https://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm)
+
 
 ## readme toc
 
@@ -71,7 +73,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [quickstart](#quickstart) - just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
         * [at home](#at-home) - make it accessible over the internet
         * [on servers](#on-servers) - you may also want these, especially on servers
-    * [features](#features)
+    * [features](#features) - also see [comparison to similar software](./docs/versus.md)
     * [testimonials](#testimonials) - small collection of user feedback
 * [motivations](#motivations) - project goals / philosophy
     * [notes](#notes) - general notes
@@ -92,6 +94,7 @@ turn almost any device into a file server with resumable uploads/downloads using
         * [file-search](#file-search) - dropping files into the browser also lets you see if they exist on the server
         * [unpost](#unpost) - undo/delete accidental uploads
         * [self-destruct](#self-destruct) - uploads can be given a lifetime
+        * [race the beam](#race-the-beam) - download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm))
     * [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
     * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
     * [media player](#media-player) - plays almost every audio format there is
@@ -181,7 +184,7 @@ enable thumbnails (images/audio/video), media indexing, and audio transcoding by
 
 * **Alpine:** `apk add py3-pillow ffmpeg`
 * **Debian:** `apt install --no-install-recommends python3-pil ffmpeg`
-* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg`
+* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg --allowerasing`
 * **FreeBSD:** `pkg install py39-sqlite3 py39-pillow ffmpeg`
 * **MacOS:** `port install py-Pillow ffmpeg`
 * **MacOS** (alternative): `brew install pillow ffmpeg`
@@ -236,6 +239,8 @@ firewall-cmd --reload
 
 ## features
 
+also see [comparison to similar software](./docs/versus.md)
+
 * backend stuff
   * ☑ IPv6
   * ☑ [multiprocessing](#performance) (actual multithreading)
@@ -258,6 +263,7 @@ firewall-cmd --reload
   * ☑ write-only folders
   * ☑ [unpost](#unpost): undo/delete accidental uploads
   * ☑ [self-destruct](#self-destruct) (specified server-side or client-side)
+  * ☑ [race the beam](#race-the-beam) (almost like peer-to-peer)
   * ☑ symlink/discard duplicates (content-matching)
 * download
   * ☑ single files in browser
@@ -683,7 +689,7 @@ up2k has several advantages:
 > it is perfectly safe to restart / upgrade copyparty while someone is uploading to it!  
 > all known up2k clients will resume just fine 💪
 
-see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
+see [up2k](./docs/devnotes.md#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
 
 ![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/129635371-48fc54ca-fa91-48e3-9b1d-ba413e4b68cb.png)
 
@@ -749,6 +755,13 @@ clients can specify a shorter expiration time using the [up2k ui](#uploading) --
 specifying a custom expiration time client-side will affect the timespan in which unposts are permitted, so keep an eye on the estimates in the up2k ui
 
 
+### race the beam
+
+download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm))  -- it's almost like peer-to-peer
+
+requires the file to be uploaded using up2k (which is the default drag-and-drop uploader), alternatively the command-line program
+
+
 ## file manager
 
 cut/paste, rename, and delete files/folders (if you have permission)

{copyparty-1.12.2 → copyparty-1.13.0}/README.md

@@ -10,6 +10,8 @@ turn almost any device into a file server with resumable uploads/downloads using
 
 📷 **screenshots:** [browser](#the-browser) // [upload](#uploading) // [unpost](#unpost) // [thumbnails](#thumbnails) // [search](#searching) // [fsearch](#file-search) // [zip-DL](#zip-downloads) // [md-viewer](#markdown-viewer)
 
+🎬 **videos:** [upload](https://a.ocv.me/pub/demo/pics-vids/up2k.webm) // [cli-upload](https://a.ocv.me/pub/demo/pics-vids/u2cli.webm) // [race-the-beam](https://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm)
+
 
 ## readme toc
 
@@ -17,7 +19,7 @@ turn almost any device into a file server with resumable uploads/downloads using
     * [quickstart](#quickstart) - just run **[copyparty-sfx.py](https://github.com/9001/copyparty/releases/latest/download/copyparty-sfx.py)** -- that's it! 🎉
         * [at home](#at-home) - make it accessible over the internet
         * [on servers](#on-servers) - you may also want these, especially on servers
-    * [features](#features)
+    * [features](#features) - also see [comparison to similar software](./docs/versus.md)
     * [testimonials](#testimonials) - small collection of user feedback
 * [motivations](#motivations) - project goals / philosophy
     * [notes](#notes) - general notes
@@ -38,6 +40,7 @@ turn almost any device into a file server with resumable uploads/downloads using
         * [file-search](#file-search) - dropping files into the browser also lets you see if they exist on the server
         * [unpost](#unpost) - undo/delete accidental uploads
         * [self-destruct](#self-destruct) - uploads can be given a lifetime
+        * [race the beam](#race-the-beam) - download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm))
     * [file manager](#file-manager) - cut/paste, rename, and delete files/folders (if you have permission)
     * [batch rename](#batch-rename) - select some files and press `F2` to bring up the rename UI
     * [media player](#media-player) - plays almost every audio format there is
@@ -127,7 +130,7 @@ enable thumbnails (images/audio/video), media indexing, and audio transcoding by
 
 * **Alpine:** `apk add py3-pillow ffmpeg`
 * **Debian:** `apt install --no-install-recommends python3-pil ffmpeg`
-* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg`
+* **Fedora:** rpmfusion + `dnf install python3-pillow ffmpeg --allowerasing`
 * **FreeBSD:** `pkg install py39-sqlite3 py39-pillow ffmpeg`
 * **MacOS:** `port install py-Pillow ffmpeg`
 * **MacOS** (alternative): `brew install pillow ffmpeg`
@@ -182,6 +185,8 @@ firewall-cmd --reload
 
 ## features
 
+also see [comparison to similar software](./docs/versus.md)
+
 * backend stuff
   * ☑ IPv6
   * ☑ [multiprocessing](#performance) (actual multithreading)
@@ -204,6 +209,7 @@ firewall-cmd --reload
   * ☑ write-only folders
   * ☑ [unpost](#unpost): undo/delete accidental uploads
   * ☑ [self-destruct](#self-destruct) (specified server-side or client-side)
+  * ☑ [race the beam](#race-the-beam) (almost like peer-to-peer)
   * ☑ symlink/discard duplicates (content-matching)
 * download
   * ☑ single files in browser
@@ -629,7 +635,7 @@ up2k has several advantages:
 > it is perfectly safe to restart / upgrade copyparty while someone is uploading to it!  
 > all known up2k clients will resume just fine 💪
 
-see [up2k](#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
+see [up2k](./docs/devnotes.md#up2k) for details on how it works, or watch a [demo video](https://a.ocv.me/pub/demo/pics-vids/#gf-0f6f5c0d)
 
 ![copyparty-upload-fs8](https://user-images.githubusercontent.com/241032/129635371-48fc54ca-fa91-48e3-9b1d-ba413e4b68cb.png)
 
@@ -695,6 +701,13 @@ clients can specify a shorter expiration time using the [up2k ui](#uploading) --
 specifying a custom expiration time client-side will affect the timespan in which unposts are permitted, so keep an eye on the estimates in the up2k ui
 
 
+### race the beam
+
+download files while they're still uploading ([demo video](http://a.ocv.me/pub/g/nerd-stuff/cpp/2024-0418-race-the-beam.webm))  -- it's almost like peer-to-peer
+
+requires the file to be uploaded using up2k (which is the default drag-and-drop uploader), alternatively the command-line program
+
+
 ## file manager
 
 cut/paste, rename, and delete files/folders (if you have permission)

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/__main__.py

@@ -850,7 +850,7 @@ def add_qr(ap, tty):
 
 def add_fs(ap):
     ap2 = ap.add_argument_group("filesystem options")
-    rm_re_def = "5/0.1" if ANYWIN else "0/0"
+    rm_re_def = "15/0.1" if ANYWIN else "0/0"
     ap2.add_argument("--rm-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be deleted because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=rm_retry)")
     ap2.add_argument("--mv-retry", metavar="T/R", type=u, default=rm_re_def, help="if a file cannot be renamed because it is busy, continue trying for \033[33mT\033[0m seconds, retry every \033[33mR\033[0m seconds; disable with 0/0 (volflag=mv_retry)")
     ap2.add_argument("--iobuf", metavar="BYTES", type=int, default=256*1024, help="file I/O buffer-size; if your volumes are on a network drive, try increasing to \033[32m524288\033[0m or even \033[32m4194304\033[0m (and let me know if that improves your performance)")
@@ -1085,6 +1085,8 @@ def add_optouts(ap):
     ap2.add_argument("--no-zip", action="store_true", help="disable download as zip/tar")
     ap2.add_argument("--no-tarcmp", action="store_true", help="disable download as compressed tar (?tar=gz, ?tar=bz2, ?tar=xz, ?tar=gz:9, ...)")
     ap2.add_argument("--no-lifetime", action="store_true", help="do not allow clients (or server config) to schedule an upload to be deleted after a given time")
+    ap2.add_argument("--no-pipe", action="store_true", help="disable race-the-beam (lockstep download of files which are currently being uploaded) (volflag=nopipe)")
+    ap2.add_argument("--no-db-ip", action="store_true", help="do not write uploader IPs into the database")
 
 
 def add_safety(ap):
@@ -1210,7 +1212,7 @@ def add_db_general(ap, hcores):
     ap2.add_argument("--no-hash", metavar="PTN", type=u, help="regex: disable hashing of matching absolute-filesystem-paths during e2ds folder scans (volflag=nohash)")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, default=noidx, help="regex: disable indexing of matching absolute-filesystem-paths during e2ds folder scans (volflag=noidx)")
     ap2.add_argument("--no-dhash", action="store_true", help="disable rescan acceleration; do full database integrity check -- makes the db ~5%% smaller and bootup/rescans 3~10x slower")
-    ap2.add_argument("--re-dhash", action="store_true", help="rebuild the cache if it gets out of sync (for example crash on startup during metadata scanning)")
+    ap2.add_argument("--re-dhash", action="store_true", help="force a cache rebuild on startup; enable this once if it gets out of sync (should never be necessary)")
     ap2.add_argument("--no-forget", action="store_true", help="never forget indexed files, even when deleted from disk -- makes it impossible to ever upload the same file twice -- only useful for offloading uploads to a cloud service or something (volflag=noforget)")
     ap2.add_argument("--dbd", metavar="PROFILE", default="wal", help="database durability profile; sets the tradeoff between robustness and speed, see \033[33m--help-dbd\033[0m (volflag=dbd)")
     ap2.add_argument("--xlink", action="store_true", help="on upload: check all volumes for dupes, not just the target volume (volflag=xlink)")

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/__version__.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
-VERSION = (1, 12, 2)
-CODENAME = "locksmith"
-BUILD_DT = (2024, 4, 12)
+VERSION = (1, 13, 0)
+CODENAME = "race the beam"
+BUILD_DT = (2024, 4, 20)
 
 S_VERSION = ".".join(map(str, VERSION))
 S_BUILD_DT = "{0:04d}-{1:02d}-{2:02d}".format(*BUILD_DT)

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/cfg.py

@@ -16,6 +16,7 @@ def vf_bmap()   :
         "no_dedup": "copydupes",
         "no_dupe": "nodupe",
         "no_forget": "noforget",
+        "no_pipe": "nopipe",
         "no_robots": "norobots",
         "no_thumb": "dthumb",
         "no_vthumb": "dvthumb",

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/httpcli.py

@@ -36,6 +36,7 @@ from .bos import bos
 from .star import StreamTar
 from .sutil import StreamArc, gfilter
 from .szip import StreamZip
+from .up2k import up2k_chunksize
 from .util import unquote  # type: ignore
 from .util import (
     APPLESAN_RE,
@@ -123,6 +124,7 @@ class HttpCli(object):
         self.ico = conn.ico  # mypy404
         self.thumbcli = conn.thumbcli  # mypy404
         self.u2fh = conn.u2fh  # mypy404
+        self.pipes = conn.pipes  # mypy404
         self.log_func = conn.log_func  # mypy404
         self.log_src = conn.log_src  # mypy404
         self.gen_fk = self._gen_fk if self.args.log_fk else gen_filekey
@@ -2925,17 +2927,42 @@ class HttpCli(object):
 
         return txt
 
-    def tx_file(self, req_path )  :
+    def tx_file(self, req_path , ptop  = None)  :
         status = 200
         logmsg = "{:4} {} ".format("", self.req)
         logtail = ""
 
+        if ptop is not None:
+            try:
+                dp, fn = os.path.split(req_path)
+                tnam = fn + ".PARTIAL"
+                if self.args.dotpart:
+                    tnam = "." + tnam
+                ap_data = os.path.join(dp, tnam)
+                st_data = bos.stat(ap_data)
+                if not st_data.st_size:
+                    raise Exception("partial is empty")
+                x = self.conn.hsrv.broker.ask("up2k.find_job_by_ap", ptop, req_path)
+                job = json.loads(x.get())
+                if not job:
+                    raise Exception("not found in registry")
+                self.pipes.set(req_path, job)
+            except Exception as ex:
+                self.log("will not pipe [%s]; %s" % (ap_data, ex), 6)
+                ptop = None
+
         #
         # if request is for foo.js, check if we have foo.js.gz
 
         file_ts = 0.0
         editions    = {}
         for ext in ("", ".gz"):
+            if ptop is not None:
+                sz = job["size"]
+                file_ts = job["lmod"]
+                editions["plain"] = (ap_data, sz)
+                break
+
             try:
                 fs_path = req_path + ext
                 st = bos.stat(fs_path)
@@ -3092,6 +3119,11 @@ class HttpCli(object):
             self.send_headers(length=upper - lower, status=status, mime=mime)
             return True
 
+        if ptop is not None:
+            return self.tx_pipe(
+                ptop, req_path, ap_data, job, lower, upper, status, mime, logmsg
+            )
+
         ret = True
         with open_func(*open_args) as f:
             self.send_headers(length=upper - lower, status=status, mime=mime)
@@ -3111,6 +3143,143 @@ class HttpCli(object):
 
         return ret
 
+    def tx_pipe(
+        self,
+        ptop ,
+        req_path ,
+        ap_data ,
+        job  ,
+        lower ,
+        upper ,
+        status ,
+        mime ,
+        logmsg ,
+    )  :
+        M = 1048576
+        self.send_headers(length=upper - lower, status=status, mime=mime)
+        wr_slp = self.args.s_wr_slp
+        wr_sz = self.args.s_wr_sz
+        file_size = job["size"]
+        chunk_size = up2k_chunksize(file_size)
+        num_need = -1
+        data_end = 0
+        remains = upper - lower
+        broken = False
+        spins = 0
+        tier = 0
+        tiers = ["uncapped", "reduced speed", "one byte per sec"]
+
+        while lower < upper and not broken:
+            with self.u2mutex:
+                job = self.pipes.get(req_path)
+                if not job:
+                    x = self.conn.hsrv.broker.ask("up2k.find_job_by_ap", ptop, req_path)
+                    job = json.loads(x.get())
+                    if job:
+                        self.pipes.set(req_path, job)
+
+            if not job:
+                t = "pipe: OK, upload has finished; yeeting remainder"
+                self.log(t, 2)
+                data_end = file_size
+                break
+
+            if num_need != len(job["need"]):
+                num_need = len(job["need"])
+                data_end = 0
+                for cid in job["hash"]:
+                    if cid in job["need"]:
+                        break
+                    data_end += chunk_size
+                t = "pipe: can stream %.2f MiB; requested range is %.2f to %.2f"
+                self.log(t % (data_end / M, lower / M, upper / M), 6)
+                with self.u2mutex:
+                    if data_end > self.u2fh.aps.get(ap_data, data_end):
+                        try:
+                            fhs = self.u2fh.cache[ap_data].all_fhs
+                            for fh in fhs:
+                                fh.flush()
+                            self.u2fh.aps[ap_data] = data_end
+                            self.log("pipe: flushed %d up2k-FDs" % (len(fhs),))
+                        except Exception as ex:
+                            self.log("pipe: u2fh flush failed: %r" % (ex,))
+
+            if lower >= data_end:
+                if data_end:
+                    t = "pipe: uploader is too slow; aborting download at %.2f MiB"
+                    self.log(t % (data_end / M))
+                    raise Pebkac(416, "uploader is too slow")
+
+                raise Pebkac(416, "no data available yet; please retry in a bit")
+
+            slack = data_end - lower
+            if slack >= 8 * M:
+                ntier = 0
+                winsz = M
+                bufsz = wr_sz
+                slp = wr_slp
+            else:
+                winsz = max(40, int(M * (slack / (12 * M))))
+                base_rate = M if not wr_slp else wr_sz / wr_slp
+                if winsz > base_rate:
+                    ntier = 0
+                    bufsz = wr_sz
+                    slp = wr_slp
+                elif winsz > 300:
+                    ntier = 1
+                    bufsz = winsz // 5
+                    slp = 0.2
+                else:
+                    ntier = 2
+                    bufsz = winsz = slp = 1
+
+            if tier != ntier:
+                tier = ntier
+                self.log("moved to tier %d (%s)" % (tier, tiers[tier]))
+
+            try:
+                with open(ap_data, "rb", self.args.iobuf) as f:
+                    f.seek(lower)
+                    page = f.read(min(winsz, data_end - lower, upper - lower))
+                if not page:
+                    raise Exception("got 0 bytes (EOF?)")
+            except Exception as ex:
+                self.log("pipe: read failed at %.2f MiB: %s" % (lower / M, ex), 3)
+                with self.u2mutex:
+                    self.pipes.c.pop(req_path, None)
+                spins += 1
+                if spins > 3:
+                    raise Pebkac(500, "file became unreadable")
+                time.sleep(2)
+                continue
+
+            spins = 0
+            pofs = 0
+            while pofs < len(page):
+                if slp:
+                    time.sleep(slp)
+
+                try:
+                    buf = page[pofs : pofs + bufsz]
+                    self.s.sendall(buf)
+                    zi = len(buf)
+                    remains -= zi
+                    lower += zi
+                    pofs += zi
+                except:
+                    broken = True
+                    break
+
+        if lower < upper and not broken:
+            with open(req_path, "rb") as f:
+                remains = sendfile_py(self.log, lower, upper, f, self.s, wr_sz, wr_slp)
+
+        spd = self._spd((upper - lower) - remains)
+        if self.do_log:
+            self.log("{},  {}".format(logmsg, spd))
+
+        return not broken
+
     def tx_zip(
         self,
         fmt ,
@@ -3748,7 +3917,7 @@ class HttpCli(object):
         if not allvols:
             ret = [{"kinshi": 1}]
 
-        jtxt = '{"u":%s,"c":%s}' % (uret, json.dumps(ret, indent=0))
+        jtxt = '{"u":%s,"c":%s}' % (uret, json.dumps(ret, separators=(",\n", ": ")))
         zi = len(uret.split('\n"pd":')) - 1
         self.log("%s #%d+%d %.2fsec" % (lm, zi, len(ret), time.time() - t0))
         self.reply(jtxt.encode("utf-8", "replace"), mime="application/json")
@@ -4027,7 +4196,9 @@ class HttpCli(object):
             ):
                 return self.tx_md(vn, abspath)
 
-            return self.tx_file(abspath)
+            return self.tx_file(
+                abspath, None if st.st_size or "nopipe" in vn.flags else vn.realpath
+            )
 
         elif is_dir and not self.can_read:
             if self._use_dirkey(abspath):

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/httpconn.py

@@ -52,6 +52,7 @@ class HttpConn(object):
         self.E  = self.args.E
         self.asrv  = hsrv.asrv  # mypy404
         self.u2fh  = hsrv.u2fh  # mypy404
+        self.pipes  = hsrv.pipes  # mypy404
         self.ipa_nm  = hsrv.ipa_nm
         self.xff_nm  = hsrv.xff_nm
         self.xff_lan  = hsrv.xff_lan  # type: ignore

{copyparty-1.12.2 → copyparty-1.13.0}/copyparty/httpsrv.py

@@ -61,6 +61,7 @@ from .u2idx import U2idx
 from .util import (
     E_SCK,
     FHC,
+    CachedDict,
     Daemon,
     Garda,
     Magician,
@@ -126,6 +127,7 @@ class HttpSrv(object):
         self.t_periodic  = None
 
         self.u2fh = FHC()
+        self.pipes = CachedDict(0.2)
         self.metrics = Metrics(self)
         self.nreq = 0
         self.nsus = 0