copyparty 1.19.0__py3-none-any.whl → 1.19.2__py3-none-any.whl

copyparty/th_srv.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import hashlib
+import io
 import logging
 import os
 import re
@@ -82,7 +83,10 @@ try:
         if os.environ.get("PRTY_NO_PIL_HEIF"):
             raise Exception()
 
-        from pyheif_pillow_opener import register_heif_opener
+        try:
+            from pillow_heif import register_heif_opener
+        except ImportError:
+            from pyheif_pillow_opener import register_heif_opener
 
         register_heif_opener()
         HAVE_HEIF = True
@@ -109,14 +113,28 @@ except:
 
 try:
     if os.environ.get("PRTY_NO_VIPS"):
-        raise Exception()
+        raise ImportError()
 
     HAVE_VIPS = True
     import pyvips
 
     logging.getLogger("pyvips").setLevel(logging.WARNING)
-except:
+except Exception as e:
     HAVE_VIPS = False
+    if not isinstance(e, ImportError):
+        logging.warning("libvips found, but failed to load: " + str(e))
+
+
+try:
+    if os.environ.get("PRTY_NO_RAW"):
+        raise Exception()
+
+    HAVE_RAW = True
+    import rawpy
+
+    logging.getLogger("rawpy").setLevel(logging.WARNING)
+except:
+    HAVE_RAW = False
 
 
 th_dir_cache = {}
@@ -202,11 +220,19 @@ class ThumbSrv(object):
         if self.args.th_clean:
             Daemon(self.cleaner, "thumb.cln")
 
-        self.fmt_pil, self.fmt_vips, self.fmt_ffi, self.fmt_ffv, self.fmt_ffa = [
+        (
+            self.fmt_pil,
+            self.fmt_vips,
+            self.fmt_raw,
+            self.fmt_ffi,
+            self.fmt_ffv,
+            self.fmt_ffa,
+        ) = [
             set(y.split(","))
             for y in [
                 self.args.th_r_pil,
                 self.args.th_r_vips,
+                self.args.th_r_raw,
                 self.args.th_r_ffi,
                 self.args.th_r_ffv,
                 self.args.th_r_ffa,
@@ -229,6 +255,9 @@ class ThumbSrv(object):
         if "vips" in self.args.th_dec:
             self.thumbable |= self.fmt_vips
 
+        if "raw" in self.args.th_dec:
+            self.thumbable |= self.fmt_raw
+
         if "ff" in self.args.th_dec:
             for zss in [self.fmt_ffi, self.fmt_ffv, self.fmt_ffa]:
                 self.thumbable |= zss
@@ -310,6 +339,7 @@ class ThumbSrv(object):
             "thumbable": self.thumbable,
             "pil": self.fmt_pil,
             "vips": self.fmt_vips,
+            "raw": self.fmt_raw,
             "ffi": self.fmt_ffi,
             "ffv": self.fmt_ffv,
             "ffa": self.fmt_ffa,
@@ -365,6 +395,8 @@ class ThumbSrv(object):
                         funs.append(self.conv_pil)
                     elif lib == "vips" and ext in self.fmt_vips:
                         funs.append(self.conv_vips)
+                    elif lib == "raw" and ext in self.fmt_raw:
+                        funs.append(self.conv_raw)
                     elif can_au and (want_png or want_au):
                         if want_opus:
                             funs.append(self.conv_opus)
@@ -477,35 +509,38 @@ class ThumbSrv(object):
 
         return im
 
+    def conv_image_pil(self, im , tpath , fmt , vn )  :
+        try:
+            im = self.fancy_pillow(im, fmt, vn)
+        except Exception as ex:
+            self.log("fancy_pillow {}".format(ex), "90")
+            im.thumbnail(self.getres(vn, fmt))
+
+        fmts = ["RGB", "L"]
+        args = {"quality": 40}
+
+        if tpath.endswith(".webp"):
+            # quality 80 = pillow-default
+            # quality 75 = ffmpeg-default
+            # method 0 = pillow-default, fast
+            # method 4 = ffmpeg-default
+            # method 6 = max, slow
+            fmts.extend(("RGBA", "LA"))
+            args["method"] = 6
+        else:
+            # default q = 75
+            args["progressive"] = True
+
+        if im.mode not in fmts:
+            # print("conv {}".format(im.mode))
+            im = im.convert("RGB")
+
+        im.save(tpath, **args)
+
     def conv_pil(self, abspath , tpath , fmt , vn )  :
         self.wait4ram(0.2, tpath)
         with Image.open(fsenc(abspath)) as im:
-            try:
-                im = self.fancy_pillow(im, fmt, vn)
-            except Exception as ex:
-                self.log("fancy_pillow {}".format(ex), "90")
-                im.thumbnail(self.getres(vn, fmt))
-
-            fmts = ["RGB", "L"]
-            args = {"quality": 40}
-
-            if tpath.endswith(".webp"):
-                # quality 80 = pillow-default
-                # quality 75 = ffmpeg-default
-                # method 0 = pillow-default, fast
-                # method 4 = ffmpeg-default
-                # method 6 = max, slow
-                fmts.extend(("RGBA", "LA"))
-                args["method"] = 6
-            else:
-                # default q = 75
-                args["progressive"] = True
-
-            if im.mode not in fmts:
-                # print("conv {}".format(im.mode))
-                im = im.convert("RGB")
-
-            im.save(tpath, **args)
+            self.conv_image_pil(im, tpath, fmt, vn)
 
     def conv_vips(self, abspath , tpath , fmt , vn )  :
         self.wait4ram(0.2, tpath)
@@ -527,9 +562,52 @@ class ThumbSrv(object):
 
         img.write_to_file(tpath, Q=40)
 
+    def conv_raw(self, abspath , tpath , fmt , vn )  :
+        self.wait4ram(0.2, tpath)
+        with rawpy.imread(abspath) as raw:
+            thumb = raw.extract_thumb()
+        if thumb.format == rawpy.ThumbFormat.JPEG and tpath.endswith(".jpg"):
+            # if we have a jpg thumbnail and no webp output is available,
+            # just write the jpg directly (it'll be the wrong size, but it's fast)
+            with open(tpath, "wb") as f:
+                f.write(thumb.data)
+        if HAVE_VIPS:
+            crops = ["centre", "none"]
+            if "f" in fmt:
+                crops = ["none"]
+            w, h = self.getres(vn, fmt)
+            kw = {"height": h, "size": "down", "intent": "relative"}
+
+            for c in crops:
+                try:
+                    kw["crop"] = c
+                    if thumb.format == rawpy.ThumbFormat.BITMAP:
+                        img = pyvips.Image.new_from_array(
+                            thumb.data, interpretation="rgb"
+                        )
+                        img = img.thumbnail_image(w, **kw)
+                    else:
+                        img = pyvips.Image.thumbnail_buffer(thumb.data, w, **kw)
+                    break
+                except:
+                    if c == crops[-1]:
+                        raise
+
+            img.write_to_file(tpath, Q=40)
+        elif HAVE_PIL:
+            if thumb.format == rawpy.ThumbFormat.BITMAP:
+                im = Image.fromarray(thumb.data, "RGB")
+            else:
+                im = Image.open(io.BytesIO(thumb.data))
+            self.conv_image_pil(im, tpath, fmt, vn)
+        else:
+            raise Exception(
+                "either pil or vips is needed to process embedded bitmap thumbnails in raw files"
+            )
+
     def conv_ffmpeg(self, abspath , tpath , fmt , vn )  :
         self.wait4ram(0.2, tpath)
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if not ret:
             return
 
@@ -540,6 +618,17 @@ class ThumbSrv(object):
             dur = ret[".dur"][1] if ".dur" in ret else 4
             seek = [b"-ss", "{:.0f}".format(dur / 3).encode("utf-8")]
 
+        self._ffmpeg_im(abspath, tpath, fmt, vn, seek, b"0:v:0")
+
+    def _ffmpeg_im(
+        self,
+        abspath ,
+        tpath ,
+        fmt ,
+        vn ,
+        seek ,
+        imap ,
+    )  :
         scale = "scale={0}:{1}:force_original_aspect_ratio="
         if "f" in fmt:
             scale += "decrease,setsar=1:1"
@@ -558,7 +647,7 @@ class ThumbSrv(object):
         cmd += seek
         cmd += [
             b"-i", fsenc(abspath),
-            b"-map", b"0:v:0",
+            b"-map", imap,
             b"-vf", bscale,
             b"-frames:v", b"1",
             b"-metadata:s:v:0", b"rotate=0",
@@ -579,11 +668,11 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
-    def _run_ff(self, cmd , vn , oom  = 400)  :
+    def _run_ff(self, cmd , vn , kto , oom  = 400)  :
         # self.log((b" ".join(cmd)).decode("utf-8"))
-        ret, _, serr = runcmd(cmd, timeout=vn.flags["convt"], nice=True, oom=oom)
+        ret, _, serr = runcmd(cmd, timeout=vn.flags[kto], nice=True, oom=oom)
         if not ret:
             return
 
@@ -627,7 +716,7 @@ class ThumbSrv(object):
         raise sp.CalledProcessError(ret, (cmd[0], b"...", cmd[-1]))
 
     def conv_waves(self, abspath , tpath , fmt , vn )  :
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
@@ -665,7 +754,7 @@ class ThumbSrv(object):
         # fmt: on
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
         if "pngquant" in vn.flags:
             wtpath = tpath + ".png"
@@ -686,11 +775,31 @@ class ThumbSrv(object):
             else:
                 atomic_move(self.log, wtpath, tpath, vn.flags)
 
+    def conv_emb_cv(
+        self, abspath , tpath , fmt , vn , strm  
+    )  :
+        self.wait4ram(0.2, tpath)
+        self._ffmpeg_im(
+            abspath, tpath, fmt, vn, [], b"0:" + strm["index"].encode("ascii")
+        )
+
     def conv_spec(self, abspath , tpath , fmt , vn )  :
-        ret, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        ret, raw, strms, ctnr = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in ret:
             raise Exception("not audio")
 
+        want_spec = vn.flags.get("th_spec_p", 1)
+        if want_spec < 2:
+            for strm in strms:
+                if (
+                    strm.get("codec_type") == "video"
+                    and strm.get("DISPOSITION:attached_pic") == "1"
+                ):
+                    return self.conv_emb_cv(abspath, tpath, fmt, vn, strm)
+
+        if not want_spec:
+            raise Exception("spectrograms forbidden by volflag")
+
         fext = abspath.split(".")[-1].lower()
 
         # https://trac.ffmpeg.org/ticket/10797
@@ -726,7 +835,7 @@ class ThumbSrv(object):
                 b"-y", fsenc(infile),
             ]
             # fmt: on
-            self._run_ff(cmd, vn)
+            self._run_ff(cmd, vn, "convt")
 
         fc = "[0:a:0]aresample=48000{},showspectrumpic=s="
         if "3" in fmt:
@@ -768,7 +877,7 @@ class ThumbSrv(object):
             ]
 
         cmd += [fsenc(tpath)]
-        self._run_ff(cmd, vn)
+        self._run_ff(cmd, vn, "convt")
 
     def conv_mp3(self, abspath , tpath , fmt , vn )  :
         quality = self.args.q_mp3.lower()
@@ -776,7 +885,7 @@ class ThumbSrv(object):
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, rawtags, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -807,14 +916,14 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def conv_flac(self, abspath , tpath , fmt , vn )  :
         if self.args.no_acode or not self.args.allow_flac:
             raise Exception("flac not permitted in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -832,14 +941,14 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def conv_wav(self, abspath , tpath , fmt , vn )  :
         if self.args.no_acode or not self.args.allow_wav:
             raise Exception("wav not permitted in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, _, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -867,14 +976,14 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def conv_opus(self, abspath , tpath , fmt , vn )  :
         if self.args.no_acode or not self.args.q_opus:
             raise Exception("disabled in server config")
 
         self.wait4ram(0.2, tpath)
-        tags, rawtags = ffprobe(abspath, int(vn.flags["convt"] / 2))
+        tags, rawtags, _, _ = ffprobe(abspath, int(vn.flags["convt"] / 2))
         if "ac" not in tags:
             raise Exception("not audio")
 
@@ -923,7 +1032,7 @@ class ThumbSrv(object):
             fsenc(tpath)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
     def _conv_caf(
         self,
@@ -963,7 +1072,7 @@ class ThumbSrv(object):
             fsenc(tmp_opus)
         ]
         # fmt: on
-        self._run_ff(cmd, vn, oom=300)
+        self._run_ff(cmd, vn, "aconvt", oom=300)
 
         # iOS fails to play some "insufficiently complex" files
         # (average file shorter than 8 seconds), so of course we
@@ -990,7 +1099,7 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn, oom=300)
+            self._run_ff(cmd, vn, "aconvt", oom=300)
 
         else:
             # simple remux should be safe
@@ -1009,7 +1118,7 @@ class ThumbSrv(object):
                 fsenc(tpath)
             ]
             # fmt: on
-            self._run_ff(cmd, vn, oom=300)
+            self._run_ff(cmd, vn, "aconvt", oom=300)
 
         try:
             wunlink(self.log, tmp_opus, vn.flags)

copyparty/u2idx.py

@@ -385,7 +385,7 @@ class U2idx(object):
             fk_alg = 2 if "fka" in flags else 1
             c = cur.execute(uq, tuple(vuv))
             for hit in c:
-                w, ts, sz, rd, fn, ip, at = hit[:7]
+                w, ts, sz, rd, fn = hit[:5]
 
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)