copyparty 1.13.1__py3-none-any.whl → 1.13.3__py3-none-any.whl

copyparty/szip.py

@@ -1,12 +1,12 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
 
-import argparse
 import calendar
 import stat
 import time
 import zlib
 
+from .authsrv import AuthSrv
 from .bos import bos
 from .sutil import StreamArc, errdesc
 from .util import min_ex, sanitize_fn, spack, sunpack, yieldfile
@@ -213,13 +213,13 @@ class StreamZip(StreamArc):
     def __init__(
         self,
         log ,
-        args ,
+        asrv ,
         fgen    ,
         utf8  = False,
         pre_crc  = False,
         **kwargs 
     )  :
-        super(StreamZip, self).__init__(log, args, fgen)
+        super(StreamZip, self).__init__(log, asrv, fgen)
 
         self.utf8 = utf8
         self.pre_crc = pre_crc
@@ -296,7 +296,7 @@ class StreamZip(StreamArc):
                 mbuf = b""
 
             if errors:
-                errf, txt = errdesc(errors)
+                errf, txt = errdesc(self.asrv.vfs, errors)
                 self.log("\n".join(([repr(errf)] + txt[1:])))
                 for x in self.ser(errf):
                     yield x

copyparty/th_cli.py

@@ -104,6 +104,11 @@ class ThumbCli(object):
 
             fmt = sfmt
 
+        elif fmt[:1] == "p" and not is_au:
+            t = "cannot thumbnail [%s]: png only allowed for waveforms"
+            self.log(t % (rem), 6)
+            return None
+
         histpath = self.asrv.vfs.histtab.get(ptop)
         if not histpath:
             self.log("no histpath for [{}]".format(ptop))

copyparty/th_srv.py

@@ -15,7 +15,7 @@ from queue import Queue
 from .__init__ import ANYWIN, TYPE_CHECKING
 from .authsrv import VFS
 from .bos import bos
-from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, ffprobe
+from .mtag import HAVE_FFMPEG, HAVE_FFPROBE, au_unpk, ffprobe
 from .util import BytesIO  # type: ignore
 from .util import (
     FFMPEG_URL,
@@ -294,6 +294,12 @@ class ThumbSrv(object):
             ext = abspath.split(".")[-1].lower()
             png_ok = False
             funs = []
+
+            if ext in self.args.au_unpk:
+                ap_unpk = au_unpk(self.log, self.args.au_unpk, abspath, vn)
+            else:
+                ap_unpk = abspath
+
             if not bos.path.exists(tpath):
                 for lib in self.args.th_dec:
                     if lib == "pil" and ext in self.fmt_pil:
@@ -313,9 +319,6 @@ class ThumbSrv(object):
                         else:
                             funs.append(self.conv_spec)
 
-            if not png_ok and tpath.endswith(".png"):
-                raise Pebkac(400, "png only allowed for waveforms")
-
             tdir, tfn = os.path.split(tpath)
             ttpath = os.path.join(tdir, "w", tfn)
             try:
@@ -325,7 +328,10 @@ class ThumbSrv(object):
 
             for fun in funs:
                 try:
-                    fun(abspath, ttpath, fmt, vn)
+                    if not png_ok and tpath.endswith(".png"):
+                        raise Exception("png only allowed for waveforms")
+
+                    fun(ap_unpk, ttpath, fmt, vn)
                     break
                 except Exception as ex:
                     msg = "{} could not create thumbnail of {}\n{}"
@@ -343,6 +349,9 @@ class ThumbSrv(object):
                         except:
                             pass
 
+            if abspath != ap_unpk:
+                wunlink(self.log, ap_unpk, vn.flags)
+
             try:
                 wrename(self.log, ttpath, tpath, vn.flags)
             except:
@@ -581,6 +590,25 @@ class ThumbSrv(object):
         cmd += [fsenc(tpath)]
         self._run_ff(cmd, vn)
 
+        if "pngquant" in vn.flags:
+            wtpath = tpath + ".png"
+            cmd = [
+                b"pngquant",
+                b"--strip",
+                b"--nofs",
+                b"--output",
+                fsenc(wtpath),
+                fsenc(tpath),
+            ]
+            ret = runcmd(cmd, timeout=vn.flags["convt"], nice=True, oom=400)[0]
+            if ret:
+                try:
+                    wunlink(self.log, wtpath, vn.flags)
+                except:
+                    pass
+            else:
+                wrename(self.log, wtpath, tpath, vn.flags)
+
     def conv_spec(self, abspath , tpath , fmt , vn )  :
         ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
@@ -643,8 +671,8 @@ class ThumbSrv(object):
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
-        if "ac" not in ret:
+        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
             raise Exception("not audio")
 
         if quality.endswith("k"):
@@ -665,7 +693,7 @@ class ThumbSrv(object):
             b"-v", b"error",
             b"-hide_banner",
             b"-i", fsenc(abspath),
-            b"-map_metadata", b"-1",
+        ] + self.big_tags(rawtags) + [
             b"-map", b"0:a:0",
             b"-ar", b"44100",
             b"-ac", b"2",
@@ -681,16 +709,16 @@ class ThumbSrv(object):
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
-        if "ac" not in ret:
+        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        if "ac" not in tags:
             raise Exception("not audio")
 
         try:
-            dur = ret[".dur"][1]
+            dur = tags[".dur"][1]
         except:
             dur = 0
 
-        src_opus = abspath.lower().endswith(".opus") or ret["ac"][1] == "opus"
+        src_opus = abspath.lower().endswith(".opus") or tags["ac"][1] == "opus"
         want_caf = tpath.endswith(".caf")
         tmp_opus = tpath
         if want_caf:
@@ -711,7 +739,7 @@ class ThumbSrv(object):
                 b"-v", b"error",
                 b"-hide_banner",
                 b"-i", fsenc(abspath),
-                b"-map_metadata", b"-1",
+            ] + self.big_tags(rawtags) + [
                 b"-map", b"0:a:0",
                 b"-c:a", b"libopus",
                 b"-b:a", bq,
@@ -768,6 +796,16 @@ class ThumbSrv(object):
             except:
                 pass
 
+    def big_tags(self, raw_tags  )  :
+        ret = []
+        for k, vs in raw_tags.items():
+            for v in vs:
+                if len(str(v)) >= 1024:
+                    bv = k.encode("utf-8", "replace")
+                    ret += [b"-metadata", bv + b"="]
+                    break
+        return ret
+
     def poke(self, tdir )  :
         if not self.poke_cd.poke(tdir):
             return

copyparty/up2k.py

@@ -10,7 +10,6 @@ import math
 import os
 import re
 import shutil
-import signal
 import stat
 import subprocess as sp
 import tempfile
@@ -29,6 +28,7 @@ from .fsutil import Fstab
 from .mtag import MParser, MTag
 from .util import (
     HAVE_SQLITE3,
+    VF_CAREFUL,
     SYMTIME,
     Daemon,
     MTHash,
@@ -88,9 +88,6 @@ CV_EXTS = set(zsg.split(","))
 HINT_HISTPATH = "you could try moving the database to another location (preferably an SSD or NVME drive) using either the --hist argument (global option for all volumes), or the hist volflag (just for this volume)"
 
 
-VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1}
-
-
 class Dbw(object):
     def __init__(self, c , n , t )  :
         self.c = c
@@ -458,7 +455,13 @@ class Up2k(object):
                 # important; not deferred by db_act
                 timeout = self._check_lifetimes()
 
-            timeout = min(timeout, now + self._check_xiu())
+            try:
+                timeout = min(timeout, now + self._check_xiu())
+            except Exception as ex:
+                if "closed cursor" in str(ex):
+                    self.log("sched_rescan: lost db")
+                    return
+                raise
 
             with self.mutex:
                 for vp, vol in sorted(self.asrv.vfs.all_vols.items()):
@@ -703,9 +706,9 @@ class Up2k(object):
                 try:
                     bos.makedirs(vol.realpath)  # gonna happen at snap anyways
                     dir_is_empty(self.log_func, not self.args.no_scandir, vol.realpath)
-                except:
+                except Exception as ex:
                     self.volstate[vol.vpath] = "OFFLINE (cannot access folder)"
-                    self.log("cannot access " + vol.realpath, c=1)
+                    self.log("cannot access %s: %r" % (vol.realpath, ex), c=1)
                     continue
 
                 if scan_vols and vol.vpath not in scan_vols:
@@ -1036,8 +1039,11 @@ class Up2k(object):
         return None
 
     def _verify_db_cache(self, cur , vpath )  :
-        # check if volume config changed since last use; drop caches if so
-        zsl = [vpath] + list(sorted(self.asrv.vfs.all_vols.keys()))
+        # check if list of intersecting volumes changed since last use; drop caches if so
+        prefix = (vpath + "/").lstrip("/")
+        zsl = [x for x in self.asrv.vfs.all_vols if x.startswith(prefix)]
+        zsl = [x[len(prefix) :] for x in zsl]
+        zsl.sort()
         zb = hashlib.sha1("\n".join(zsl).encode("utf-8", "replace")).digest()
         vcfg = base64.urlsafe_b64encode(zb[:18]).decode("ascii")
 
@@ -1647,7 +1653,7 @@ class Up2k(object):
 
         if e2vp and rewark:
             self.hub.retcode = 1
-            os.kill(os.getpid(), signal.SIGTERM)
+            Daemon(self.hub.sigterm)
             raise Exception("{} files have incorrect hashes".format(len(rewark)))
 
         if not e2vu or not rewark:

copyparty/util.py

@@ -337,6 +337,21 @@ APPLESAN_TXT = r"/(__MACOS|Icon\r\r)|/\.(_|DS_Store|AppleDouble|LSOverride|Docum
 APPLESAN_RE = re.compile(APPLESAN_TXT)
 
 
+HUMANSIZE_UNITS = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB")
+
+UNHUMANIZE_UNITS = {
+    "b": 1,
+    "k": 1024,
+    "m": 1024 * 1024,
+    "g": 1024 * 1024 * 1024,
+    "t": 1024 * 1024 * 1024 * 1024,
+    "p": 1024 * 1024 * 1024 * 1024 * 1024,
+    "e": 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
+}
+
+VF_CAREFUL = {"mv_re_t": 5, "rm_re_t": 5, "mv_re_r": 0.1, "rm_re_r": 0.1}
+
+
 pybin = sys.executable or ""
 if EXE:
     pybin = ""
@@ -442,13 +457,22 @@ class Daemon(threading.Thread):
         r  = True,
         ka   = None,
     )  :
-        threading.Thread.__init__(
-            self, target=target, name=name, args=a or (), kwargs=ka
-        )
+        threading.Thread.__init__(self, name=name)
+        self.a = a or ()
+        self.ka = ka or {}
+        self.fun = target
         self.daemon = True
         if r:
             self.start()
 
+    def run(self):
+        if not ANYWIN and not PY2:
+            signal.pthread_sigmask(
+                signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]
+            )
+
+        self.fun(*self.a, **self.ka)
+
 
 class Netdev(object):
     def __init__(self, ip , idx , name , desc ):
@@ -843,6 +867,7 @@ class ProgressPrinter(threading.Thread):
         self.start()
 
     def run(self)  :
+        sigblock()
         tp = 0
         msg = None
         no_stdout = self.args.q
@@ -1287,6 +1312,15 @@ def log_thrs(log    , ival , name )  :
         log(name, "\033[0m \033[33m".join(tv), 3)
 
 
+def sigblock():
+    if ANYWIN or PY2:
+        return
+
+    signal.pthread_sigmask(
+        signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM, signal.SIGUSR1]
+    )
+
+
 def vol_san(vols , txt )  :
     txt0 = txt
     for vol in vols:
@@ -1308,10 +1342,11 @@ def vol_san(vols , txt )  :
 
 def min_ex(max_lines  = 8, reverse  = False)  :
     et, ev, tb = sys.exc_info()
-    stb = traceback.extract_tb(tb)
+    stb = traceback.extract_tb(tb) if tb else traceback.extract_stack()[:-1]
     fmt = "%s @ %d <%s>: %s"
     ex = [fmt % (fp.split(os.sep)[-1], ln, fun, txt) for fp, ln, fun, txt in stb]
-    ex.append("[%s] %s" % (et.__name__ if et else "(anonymous)", ev))
+    if et or ev or tb:
+        ex.append("[%s] %s" % (et.__name__ if et else "(anonymous)", ev))
     return "\n".join(ex[-max_lines:][:: -1 if reverse else 1])
 
 
@@ -1764,7 +1799,7 @@ def gencookie(k , v , r , tls , dur  = 0, txt  = "")  :
 
 
 def humansize(sz , terse  = False)  :
-    for unit in ["B", "KiB", "MiB", "GiB", "TiB"]:
+    for unit in HUMANSIZE_UNITS:
         if sz < 1024:
             break
 
@@ -1785,12 +1820,7 @@ def unhumanize(sz )  :
         pass
 
     mc = sz[-1:].lower()
-    mi = {
-        "k": 1024,
-        "m": 1024 * 1024,
-        "g": 1024 * 1024 * 1024,
-        "t": 1024 * 1024 * 1024 * 1024,
-    }.get(mc, 1)
+    mi = UNHUMANIZE_UNITS.get(mc, 1)
     return int(float(sz[:-1]) * mi)
 
 
@@ -2473,6 +2503,7 @@ def sendfile_py(
     s ,
     bufsz ,
     slp ,
+    use_poll ,
 )  :
     remains = upper - lower
     f.seek(lower)
@@ -2501,22 +2532,31 @@ def sendfile_kern(
     s ,
     bufsz ,
     slp ,
+    use_poll ,
 )  :
     out_fd = s.fileno()
     in_fd = f.fileno()
     ofs = lower
     stuck = 0.0
+    if use_poll:
+        poll = select.poll()
+        poll.register(out_fd, select.POLLOUT)
+
     while ofs < upper:
         stuck = stuck or time.time()
         try:
             req = min(2 ** 30, upper - ofs)
-            select.select([], [out_fd], [], 10)
+            if use_poll:
+                poll.poll(10000)
+            else:
+                select.select([], [out_fd], [], 10)
             n = os.sendfile(out_fd, in_fd, ofs, req)
             stuck = 0
         except OSError as ex:
             # client stopped reading; do another select
             d = time.time() - stuck
             if d < 3600 and ex.errno == errno.EWOULDBLOCK:
+                time.sleep(0.02)
                 continue
 
             n = 0
@@ -2660,7 +2700,7 @@ def unescape_cookie(orig )  :
 
 def guess_mime(url , fallback  = "application/octet-stream")  :
     try:
-        _, ext = url.rsplit(".", 1)
+        ext = url.rsplit(".", 1)[1].lower()
     except:
         return fallback
 

copyparty/web/a/u2c.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from __future__ import print_function, unicode_literals
 
-S_VERSION = "1.16"
-S_BUILD_DT = "2024-04-20"
+S_VERSION = "1.18"
+S_BUILD_DT = "2024-06-01"
 
 """
 u2c.py: upload to copyparty
@@ -79,12 +79,21 @@ req_ses = requests.Session()
 
 
 class Daemon(threading.Thread):
-    def __init__(self, target, name=None, a=None):
-        # type: (Any, Any, Any) -> None
-        threading.Thread.__init__(self, target=target, args=a or (), name=name)
+    def __init__(self, target, name = None, a = None):
+        threading.Thread.__init__(self, name=name)
+        self.a = a or ()
+        self.fun = target
         self.daemon = True
         self.start()
 
+    def run(self):
+        try:
+            signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT, signal.SIGTERM])
+        except:
+            pass
+
+        self.fun(*self.a)
+
 
 class File(object):
     """an up2k upload task; represents a single file"""
@@ -1135,7 +1144,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("url", type=unicode, help="server url, including destination folder")
     ap.add_argument("files", type=unicode, nargs="+", help="files and/or folders to process")
     ap.add_argument("-v", action="store_true", help="verbose")
-    ap.add_argument("-a", metavar="PASSWORD", help="password or $filepath")
+    ap.add_argument("-a", metavar="PASSWD", help="password or $filepath")
     ap.add_argument("-s", action="store_true", help="file-search (disables upload)")
     ap.add_argument("-x", type=unicode, metavar="REGEX", default="", help="skip file if filesystem-abspath matches REGEX, example: '.*/\\.hist/.*'")
     ap.add_argument("--ok", action="store_true", help="continue even if some local files are inaccessible")
@@ -1153,8 +1162,8 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("--drd", action="store_true", help="delete remote files during upload instead of afterwards; reduces peak disk space usage, but will reupload instead of detecting renames")
 
     ap = app.add_argument_group("performance tweaks")
-    ap.add_argument("-j", type=int, metavar="THREADS", default=4, help="parallel connections")
-    ap.add_argument("-J", type=int, metavar="THREADS", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
+    ap.add_argument("-j", type=int, metavar="CONNS", default=2, help="parallel connections")
+    ap.add_argument("-J", type=int, metavar="CORES", default=hcores, help="num cpu-cores to use for hashing; set 0 or 1 for single-core hashing")
     ap.add_argument("-nh", action="store_true", help="disable hashing while uploading")
     ap.add_argument("-ns", action="store_true", help="no status panel (for slow consoles and macos)")
     ap.add_argument("--cd", type=float, metavar="SEC", default=5, help="delay before reattempting a failed handshake/upload")
@@ -1162,7 +1171,7 @@ source file/folder selection uses rsync syntax, meaning that:
     ap.add_argument("-z", action="store_true", help="ZOOMIN' (skip uploading files if they exist at the destination with the ~same last-modified timestamp, so same as yolo / turbo with date-chk but even faster)")
 
     ap = app.add_argument_group("tls")
-    ap.add_argument("-te", metavar="PEM_FILE", help="certificate to expect/verify")
+    ap.add_argument("-te", metavar="PATH", help="path to ca.pem or cert.pem to expect/verify")
     ap.add_argument("-td", action="store_true", help="disable certificate check")
     # fmt: on
 
@@ -1199,6 +1208,14 @@ source file/folder selection uses rsync syntax, meaning that:
     ar.url = ar.url.rstrip("/") + "/"
     if "://" not in ar.url:
         ar.url = "http://" + ar.url
+    
+    if "https://" in ar.url.lower():
+        try:
+            import ssl, zipfile
+        except:
+            t = "ERROR: https is not available for some reason; please use http"
+            print("\n\n   %s\n\n" % (t,))
+            raise
 
     if ar.a and ar.a.startswith("$"):
         fn = ar.a[1:]

{copyparty-1.13.1.dist-info → copyparty-1.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: copyparty
-Version: 1.13.1
+Version: 1.13.3
 Summary: Portable file server with accelerated resumable uploads, deduplication, WebDAV, FTP, zeroconf, media indexer, video thumbnails, audio transcoding, and write-only folders
 Author-email: ed <copyparty@ocv.me>
 License: MIT
@@ -284,6 +284,7 @@ also see [comparison to similar software](./docs/versus.md)
     * ☑ ...of videos using FFmpeg
     * ☑ ...of audio (spectrograms) using FFmpeg
     * ☑ cache eviction (max-age; maybe max-size eventually)
+  * ☑ multilingual UI (english, norwegian, [add your own](./docs/rice/#translations)))
   * ☑ SPA (browse while uploading)
 * server indexing
   * ☑ [locate files by contents](#file-search)
@@ -466,7 +467,7 @@ configuring accounts/volumes with arguments:
     `-v .::r,usr1,usr2:rw,usr3,usr4` = usr1/2 read-only, 3/4 read-write
 
 permissions:
-* `r` (read): browse folder contents, download files, download as zip/tar
+* `r` (read): browse folder contents, download files, download as zip/tar, see filekeys/dirkeys
 * `w` (write): upload files, move files *into* this folder
 * `m` (move): move files/folders *from* this folder
 * `d` (delete): delete files/folders
@@ -668,7 +669,7 @@ you can also zip a selection of files or folders by clicking them in the browser
 
 cool trick: download a folder by appending url-params `?tar&opus` or `?tar&mp3` to transcode all audio files (except aac|m4a|mp3|ogg|opus|wma) to opus/mp3 before they're added to the archive
 * super useful if you're 5 minutes away from takeoff and realize you don't have any music on your phone but your server only has flac files and downloading those will burn through all your data + there wouldn't be enough time anyways
-* and url-params `&j` / `&w` produce jpeg/webm thumbnails/spectrograms instead of the original audio/video/images
+* and url-params `&j` / `&w` produce jpeg/webm thumbnails/spectrograms instead of the original audio/video/images (`&p` for audio waveforms)
   * can also be used to pregenerate thumbnails; combine with `--th-maxage=9999999` or `--th-clean=0`
 
 
@@ -964,6 +965,8 @@ using arguments or config files, or a mix of both:
 
 **NB:** as humongous as this readme is, there is also a lot of undocumented features. Run copyparty with `--help` to see all available global options; all of those can be used in the `[global]` section of config files, and everything listed in `--help-flags` can be used in volumes as volflags.
 * if running in docker/podman, try this: `docker run --rm -it copyparty/ac --help`
+* or see this (probably outdated): https://ocv.me/copyparty/helptext.html
+* or if you prefer plaintext, https://ocv.me/copyparty/helptext.txt
 
 
 ## zeroconf
@@ -1140,7 +1143,7 @@ tweaking the ui
   * to sort in music order (album, track, artist, title) with filename as fallback, you could `--sort tags/Cirle,tags/.tn,tags/Artist,tags/Title,href`
   * to sort by upload date, first enable showing the upload date in the listing with `-e2d -mte +.up_at` and then `--sort tags/.up_at`
 
-see [./docs/rice](./docs/rice) for more, including how to add stuff (css/`<meta>`/...) to the html `<head>` tag
+see [./docs/rice](./docs/rice) for more, including how to add stuff (css/`<meta>`/...) to the html `<head>` tag, or to add your own translation
 
 
 ## opengraph