copyparty 1.14.4__py3-none-any.whl → 1.15.1__py3-none-any.whl

copyparty/svchub.py

@@ -3,7 +3,6 @@ from __future__ import print_function, unicode_literals
 
 import argparse
 import base64
-import calendar
 import errno
 import gzip
 import logging
@@ -16,7 +15,7 @@ import string
 import sys
 import threading
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 
 # from inspect import currentframe
 # print(currentframe().f_lineno)
@@ -98,6 +97,7 @@ class SvcHub(object):
         self.argv = argv
         self.E  = args.E
         self.no_ansi = args.no_ansi
+        self.tz = UTC if args.log_utc else None
         self.logf  = None
         self.logf_base_fn = ""
         self.is_dut = False  # running in unittest; always False
@@ -112,7 +112,8 @@ class SvcHub(object):
         self.httpsrv_up = 0
 
         self.log_mutex = threading.Lock()
-        self.next_day = 0
+        self.cday = 0
+        self.cmon = 0
         self.tstack = 0.0
 
         self.iphash = HMaccas(os.path.join(self.E.cfg, "iphash"), 8)
@@ -214,6 +215,9 @@ class SvcHub(object):
             noch.update([x for x in zsl if x])
         args.chpw_no = noch
 
+        if not self.args.no_ses:
+            self.setup_session_db()
+
         if args.shr:
             self.setup_share_db()
 
@@ -362,6 +366,64 @@ class SvcHub(object):
 
         self.broker = Broker(self)
 
+    def setup_session_db(self)  :
+        if not HAVE_SQLITE3:
+            self.args.no_ses = True
+            t = "WARNING: sqlite3 not available; disabling sessions, will use plaintext passwords in cookies"
+            self.log("root", t, 3)
+            return
+
+        import sqlite3
+
+        create = True
+        db_path = self.args.ses_db
+        self.log("root", "opening sessions-db %s" % (db_path,))
+        for n in range(2):
+            try:
+                db = sqlite3.connect(db_path)
+                cur = db.cursor()
+                try:
+                    cur.execute("select count(*) from us").fetchone()
+                    create = False
+                    break
+                except:
+                    pass
+            except Exception as ex:
+                if n:
+                    raise
+                t = "sessions-db corrupt; deleting and recreating: %r"
+                self.log("root", t % (ex,), 3)
+                try:
+                    cur.close()  # type: ignore
+                except:
+                    pass
+                try:
+                    db.close()  # type: ignore
+                except:
+                    pass
+                os.unlink(db_path)
+
+        sch = [
+            r"create table kv (k text, v int)",
+            r"create table us (un text, si text, t0 int)",
+            # username, session-id, creation-time
+            r"create index us_un on us(un)",
+            r"create index us_si on us(si)",
+            r"create index us_t0 on us(t0)",
+            r"insert into kv values ('sver', 1)",
+        ]
+
+        assert db  # type: ignore
+        assert cur  # type: ignore
+        if create:
+            for cmd in sch:
+                cur.execute(cmd)
+            self.log("root", "created new sessions-db")
+            db.commit()
+
+        cur.close()
+        db.close()
+
     def setup_share_db(self)  :
         al = self.args
         if not HAVE_SQLITE3:
@@ -538,7 +600,7 @@ class SvcHub(object):
         fng = []
         t_ff = "transcode audio, create spectrograms, video thumbnails"
         to_check = [
-            (HAVE_SQLITE3, "sqlite", "file and media indexing"),
+            (HAVE_SQLITE3, "sqlite", "sessions and file/media indexing"),
             (HAVE_PIL, "pillow", "image thumbnails (plenty fast)"),
             (HAVE_VIPS, "vips", "image thumbnails (faster, eats more ram)"),
             (HAVE_WEBP, "pillow-webp", "create thumbnails as webp files"),
@@ -785,7 +847,7 @@ class SvcHub(object):
             self.args.nc = min(self.args.nc, soft // 2)
 
     def _logname(self)  :
-        dt = datetime.now(UTC)
+        dt = datetime.now(self.tz)
         fn = str(self.args.lo)
         for fs in "YmdHMS":
             fs = "%" + fs
@@ -938,6 +1000,11 @@ class SvcHub(object):
 
         self._reload(rescan_all_vols=rescan_all_vols, up2k=up2k)
 
+    def _reload_sessions(self)  :
+        with self.asrv.mutex:
+            self.asrv.load_sessions(True)
+        self.broker.reload_sessions()
+
     def stop_thr(self)  :
         while not self.stop_req:
             with self.stop_cond:
@@ -1058,12 +1125,12 @@ class SvcHub(object):
             return
 
         with self.log_mutex:
-            zd = datetime.now(UTC)
+            dt = datetime.now(self.tz)
             ts = self.log_dfmt % (
-                zd.year,
-                zd.month * 100 + zd.day,
-                (zd.hour * 100 + zd.minute) * 100 + zd.second,
-                zd.microsecond // self.log_div,
+                dt.year,
+                dt.month * 100 + dt.day,
+                (dt.hour * 100 + dt.minute) * 100 + dt.second,
+                dt.microsecond // self.log_div,
             )
 
             if c and not self.args.no_ansi:
@@ -1084,41 +1151,26 @@ class SvcHub(object):
             if not self.args.no_logflush:
                 self.logf.flush()
 
-            now = time.time()
-            if int(now) >= self.next_day:
-                self._set_next_day()
+            if dt.day != self.cday or dt.month != self.cmon:
+                self._set_next_day(dt)
 
-    def _set_next_day(self)  :
-        if self.next_day and self.logf and self.logf_base_fn != self._logname():
+    def _set_next_day(self, dt )  :
+        if self.cday and self.logf and self.logf_base_fn != self._logname():
             self.logf.close()
             self._setup_logfile("")
 
-        dt = datetime.now(UTC)
-
-        # unix timestamp of next 00:00:00 (leap-seconds safe)
-        day_now = dt.day
-        while dt.day == day_now:
-            dt += timedelta(hours=12)
-
-        dt = dt.replace(hour=0, minute=0, second=0)
-        try:
-            tt = dt.utctimetuple()
-        except:
-            # still makes me hella uncomfortable
-            tt = dt.timetuple()
-
-        self.next_day = calendar.timegm(tt)
+        self.cday = dt.day
+        self.cmon = dt.month
 
     def _log_enabled(self, src , msg , c   = 0)  :
         """handles logging from all components"""
         with self.log_mutex:
-            now = time.time()
-            if int(now) >= self.next_day:
-                dt = datetime.fromtimestamp(now, UTC)
+            dt = datetime.now(self.tz)
+            if dt.day != self.cday or dt.month != self.cmon:
                 zs = "{}\n" if self.no_ansi else "\033[36m{}\033[0m\n"
                 zs = zs.format(dt.strftime("%Y-%m-%d"))
                 print(zs, end="")
-                self._set_next_day()
+                self._set_next_day(dt)
                 if self.logf:
                     self.logf.write(zs)
 
@@ -1137,12 +1189,11 @@ class SvcHub(object):
                 else:
                     msg = "%s%s\033[0m" % (c, msg)
 
-            zd = datetime.fromtimestamp(now, UTC)
             ts = self.log_efmt % (
-                zd.hour,
-                zd.minute,
-                zd.second,
-                zd.microsecond // self.log_div,
+                dt.hour,
+                dt.minute,
+                dt.second,
+                dt.microsecond // self.log_div,
             )
             msg = fmt % (ts, src, msg)
             try:

copyparty/up2k.py

@@ -1459,7 +1459,7 @@ class Up2k(object):
                     self.log("file: {}".format(abspath))
 
                 try:
-                    hashes = self._hashlist_from_file(
+                    hashes, _ = self._hashlist_from_file(
                         abspath, "a{}, ".format(self.pp.n)
                     )
                 except Exception as ex:
@@ -1653,6 +1653,7 @@ class Up2k(object):
             qex = " where " + qex
 
         rewark      = []
+        f404    = []
 
         with self.mutex:
             b_left = 0
@@ -1669,7 +1670,8 @@ class Up2k(object):
                 if self.stop:
                     return -1
 
-                w, drd, dfn = zb[:-1].decode("utf-8").split("\x00")
+                zs = zb[:-1].decode("utf-8").replace("\x00\x02", "\n")
+                w, drd, dfn = zs.split("\x00\x01")
                 with self.mutex:
                     q = "select mt, sz from up where rd=? and fn=? and +w=?"
                     try:
@@ -1695,9 +1697,14 @@ class Up2k(object):
                 pf = "v{}, {:.0f}+".format(n_left, b_left / 1024 / 1024)
                 self.pp.msg = pf + abspath
 
-                # throws on broken symlinks (always did)
-                stl = bos.lstat(abspath)
-                st = bos.stat(abspath) if stat.S_ISLNK(stl.st_mode) else stl
+                try:
+                    stl = bos.lstat(abspath)
+                    st = bos.stat(abspath) if stat.S_ISLNK(stl.st_mode) else stl
+                except Exception as ex:
+                    self.log("missing file: %s" % (abspath,), 3)
+                    f404.append((drd, dfn, w))
+                    continue
+
                 mt2 = int(stl.st_mtime)
                 sz2 = st.st_size
 
@@ -1708,7 +1715,7 @@ class Up2k(object):
                         self.log("file: {}".format(abspath))
 
                     try:
-                        hashes = self._hashlist_from_file(abspath, pf)
+                        hashes, _ = self._hashlist_from_file(abspath, pf)
                     except Exception as ex:
                         self.log("hash: {} @ [{}]".format(repr(ex), abspath))
                         continue
@@ -1734,12 +1741,15 @@ class Up2k(object):
                 t = t.format(abspath, w, sz, mt, w2, sz2, mt2)
                 self.log(t, 1)
 
-        if e2vp and rewark:
+        if e2vp and (rewark or f404):
             self.hub.retcode = 1
             Daemon(self.hub.sigterm)
-            raise Exception("{} files have incorrect hashes".format(len(rewark)))
+            t = "in volume /%s:  %s files missing, %s files have incorrect hashes"
+            t = t % (vol.vpath, len(f404), len(rewark))
+            self.log(t, 1)
+            raise Exception(t)
 
-        if not e2vu or not rewark:
+        if not e2vu or (not rewark and not f404):
             return 0
 
         with self.mutex:
@@ -1747,9 +1757,13 @@ class Up2k(object):
                 q = "update up set w = ?, sz = ?, mt = ? where rd = ? and fn = ? limit 1"
                 cur.execute(q, (w, sz, int(mt), rd, fn))
 
+            for _, _, w in f404:
+                q = "delete from up where w = ? limit 1"
+                cur.execute(q, (w,))
+
             cur.connection.commit()
 
-        return len(rewark)
+        return len(rewark) + len(f404)
 
     def _build_tags_index(self, vol )    :
         ptop = vol.realpath
@@ -1964,7 +1978,8 @@ class Up2k(object):
                     if c2.execute(q, (row[0][:16],)).fetchone():
                         continue
 
-                gf.write(("%s\n" % ("\x00".join(row),)).encode("utf-8"))
+                zs = "\x00\x01".join(row).replace("\n", "\x00\x02")
+                gf.write((zs + "\n").encode("utf-8"))
                 n += 1
 
         c2.close()
@@ -2663,10 +2678,13 @@ class Up2k(object):
             jcur = self.cur.get(ptop)
             reg = self.registry[ptop]
             vfs = self.asrv.vfs.all_vols[cj["vtop"]]
-            n4g = vfs.flags.get("noforget")
+            n4g = bool(vfs.flags.get("noforget"))
             rand = vfs.flags.get("rand") or cj.get("rand")
             lost    = []
 
+            safe_dedup = vfs.flags.get("safededup") or 50
+            data_ok = safe_dedup < 10 or n4g
+
             vols = [(ptop, jcur)] if jcur else []
             if vfs.flags.get("xlink"):
                 vols += [(k, v) for k, v in self.cur.items() if k != ptop]
@@ -2674,7 +2692,7 @@ class Up2k(object):
                 # force upload time rather than last-modified
                 cj["lmod"] = int(time.time())
 
-            alts     = []
+            alts        = []
             for ptop, cur in vols:
                 allv = self.asrv.vfs.all_vols
                 cvfs = next((v for v in allv.values() if v.realpath == ptop), vfs)
@@ -2704,13 +2722,12 @@ class Up2k(object):
                                 wark, st.st_size, dsize, st.st_mtime, dtime, dp_abs
                             )
                             self.log(t)
-                            raise Exception("desync")
+                            raise Exception()
                     except Exception as ex:
                         if n4g:
                             st = os.stat_result((0, -1, -1, 0, 0, 0, 0, 0, 0, 0))
                         else:
-                            if str(ex) != "desync":
-                                lost.append((cur, dp_dir, dp_fn))
+                            lost.append((cur, dp_dir, dp_fn))
                             continue
 
                     j = {
@@ -2733,18 +2750,42 @@ class Up2k(object):
                         if k in cj:
                             j[k] = cj[k]
 
+                    # offset of 1st diff in vpaths
+                    zig = (
+                        n + 1
+                        for n, (c1, c2) in enumerate(
+                            zip(dp_dir + "\r", cj["prel"] + "\n")
+                        )
+                        if c1 != c2
+                    )
                     score = (
-                        (3 if st.st_dev == dev else 0)
-                        + (2 if dp_dir == cj["prel"] else 0)
+                        (6969 if st.st_dev == dev else 0)
+                        + (3210 if dp_dir == cj["prel"] else next(zig))
                         + (1 if dp_fn == cj["name"] else 0)
                     )
-                    alts.append((score, -len(alts), j))
-
-            if alts:
-                best = sorted(alts, reverse=True)[0]
-                job = best[2]
-            else:
-                job = None
+                    alts.append((score, -len(alts), j, cur, dp_dir, dp_fn))
+
+            job = None
+            inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
+            for dupe in sorted(alts, reverse=True):
+                rj = dupe[2]
+                orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
+                if data_ok or inc_ap == orig_ap:
+                    data_ok = True
+                    job = rj
+                    break
+                else:
+                    self.log("asserting contents of %s" % (orig_ap,))
+                    dhashes, st = self._hashlist_from_file(orig_ap)
+                    dwark = up2k_wark_from_hashlist(self.salt, st.st_size, dhashes)
+                    if wark != dwark:
+                        t = "will not dedup (fs index desync): fs=%s, db=%s, file: %s"
+                        self.log(t % (dwark, wark, orig_ap))
+                        lost.append(dupe[3:])
+                        continue
+                    data_ok = True
+                    job = rj
+                    break
 
             if job and wark in reg:
                 # self.log("pop " + wark + "  " + job["name"] + " handle_json db", 4)
@@ -2753,7 +2794,7 @@ class Up2k(object):
             if lost:
                 c2 = None
                 for cur, dp_dir, dp_fn in lost:
-                    t = "forgetting deleted file: /{}"
+                    t = "forgetting desynced db entry: /{}"
                     self.log(t.format(vjoin(vjoin(vfs.vpath, dp_dir), dp_fn)))
                     self.db_rm(cur, dp_dir, dp_fn, cj["size"])
                     if c2 and c2 != cur:
@@ -2788,7 +2829,13 @@ class Up2k(object):
                             del reg[wark]
                         break
 
-                if st and not self.args.nw and not n4g and st.st_size != rj["size"]:
+                inc_ap = djoin(cj["ptop"], cj["prel"], cj["name"])
+                orig_ap = djoin(rj["ptop"], rj["prel"], rj["name"])
+
+                if self.args.nw or n4g or not st:
+                    pass
+
+                elif st.st_size != rj["size"]:
                     t = "will not dedup (fs index desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}"
                     t = t.format(
                         wark, st.st_size, rj["size"], st.st_mtime, rj["lmod"], path
@@ -2796,6 +2843,15 @@ class Up2k(object):
                     self.log(t)
                     del reg[wark]
 
+                elif inc_ap != orig_ap and not data_ok and "done" in reg[wark]:
+                    self.log("asserting contents of %s" % (orig_ap,))
+                    dhashes, _ = self._hashlist_from_file(orig_ap)
+                    dwark = up2k_wark_from_hashlist(self.salt, st.st_size, dhashes)
+                    if wark != dwark:
+                        t = "will not dedup (fs index desync): fs=%s, idx=%s, file: %s"
+                        self.log(t % (dwark, wark, orig_ap))
+                        del reg[wark]
+
             if job or wark in reg:
                 job = job or reg[wark]
                 if (
@@ -3048,7 +3104,22 @@ class Up2k(object):
         fp = djoin(fdir, fname)
         if job.get("replace") and bos.path.exists(fp):
             self.log("replacing existing file at {}".format(fp))
-            wunlink(self.log, fp, self.flags.get(job["ptop"]) or {})
+            cur = None
+            ptop = job["ptop"]
+            vf = self.flags.get(ptop) or {}
+            st = bos.stat(fp)
+            try:
+                vrel = vjoin(job["prel"], fname)
+                xlink = bool(vf.get("xlink"))
+                cur, wark, _, _, _, _ = self._find_from_vpath(ptop, vrel)
+                self._forget_file(ptop, vrel, cur, wark, True, st.st_size, xlink)
+            except Exception as ex:
+                self.log("skipping replace-relink: %r" % (ex,))
+            finally:
+                if cur:
+                    cur.connection.commit()
+
+            wunlink(self.log, fp, vf)
 
         if self.args.plain_ip:
             dip = ip.replace(":", ".")
@@ -3067,17 +3138,25 @@ class Up2k(object):
         verbose  = True,
         rm  = False,
         lmod  = 0,
+        fsrc  = None,
     )  :
+        if src == dst or (fsrc and fsrc == dst):
+            t = "symlinking a file to itself?? orig(%s) fsrc(%s) link(%s)"
+            raise Exception(t % (src, fsrc, dst))
+
         if verbose:
-            self.log("linking dupe:\n  {0}\n  {1}".format(src, dst))
+            t = "linking dupe:\n  point-to: {0}\n  link-loc: {1}"
+            if fsrc:
+                t += "\n  data-src: {2}"
+            self.log(t.format(src, dst, fsrc))
 
         if self.args.nw:
             return
 
         linked = False
         try:
-            if "copydupes" in flags:
-                raise Exception("disabled in config")
+            if not flags.get("dedup"):
+                raise Exception("dedup is disabled in config")
 
             lsrc = src
             ldst = dst
@@ -3114,7 +3193,7 @@ class Up2k(object):
                     linked = True
             except Exception as ex:
                 self.log("cannot hardlink: " + repr(ex))
-                if "neversymlink" in flags:
+                if "hardlinkonly" in flags:
                     raise Exception("symlink-fallback disabled in cfg")
 
             if not linked:
@@ -3133,7 +3212,15 @@ class Up2k(object):
                 linked = True
         except Exception as ex:
             self.log("cannot link; creating copy: " + repr(ex))
-            shutil.copy2(fsenc(src), fsenc(dst))
+            if bos.path.isfile(src):
+                csrc = src
+            elif fsrc and bos.path.isfile(fsrc):
+                csrc = fsrc
+            else:
+                t = "BUG: no valid sources to link from! orig(%s) fsrc(%s) link(%s)"
+                self.log(t, 1)
+                raise Exception(t % (src, fsrc, dst))
+            shutil.copy2(fsenc(csrc), fsenc(dst))
 
         if lmod and (not linked or SYMTIME):
             times = (int(time.time()), int(lmod))
@@ -3695,8 +3782,11 @@ class Up2k(object):
                     cur = None
                     try:
                         ptop = dbv.realpath
+                        xlink = bool(dbv.flags.get("xlink"))
                         cur, wark, _, _, _, _ = self._find_from_vpath(ptop, volpath)
-                        self._forget_file(ptop, volpath, cur, wark, True, st.st_size)
+                        self._forget_file(
+                            ptop, volpath, cur, wark, True, st.st_size, xlink
+                        )
                     finally:
                         if cur:
                             cur.connection.commit()
@@ -3920,13 +4010,15 @@ class Up2k(object):
             if c2 and c2 != c1:
                 self._copy_tags(c1, c2, w)
 
+            xlink = bool(svn.flags.get("xlink"))
+
             with self.reg_mutex:
                 has_dupes = self._forget_file(
-                    svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize
+                    svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize, xlink
                 )
 
             if not is_xvol:
-                has_dupes = self._relink(w, svn.realpath, srem, dabs)
+                has_dupes = self._relink(w, svn.realpath, srem, dabs, c1, xlink)
 
             curs.add(c1)
 
@@ -4069,6 +4161,7 @@ class Up2k(object):
         wark ,
         drop_tags ,
         sz ,
+        xlink ,
     )  :
         """
         mutex(main,reg) me
@@ -4080,7 +4173,7 @@ class Up2k(object):
         if wark and cur:
             self.log("found {} in db".format(wark))
             if drop_tags:
-                if self._relink(wark, ptop, vrem, ""):
+                if self._relink(wark, ptop, vrem, "", cur, xlink):
                     has_dupes = True
                     drop_tags = False
 
@@ -4112,7 +4205,15 @@ class Up2k(object):
 
         return has_dupes
 
-    def _relink(self, wark , sptop , srem , dabs )  :
+    def _relink(
+        self,
+        wark ,
+        sptop ,
+        srem ,
+        dabs ,
+        vcur ,
+        xlink ,
+    )  :
         """
         update symlinks from file at svn/srem to dabs (rename),
         or to first remaining full if no dabs (delete)
@@ -4128,6 +4229,8 @@ class Up2k(object):
             argv = (wark[:16], wark)
 
         for ptop, cur in self.cur.items():
+            if not xlink and cur and cur != vcur:
+                continue
             for rd, fn in cur.execute(q, argv):
                 if rd.startswith("//") or fn.startswith("//"):
                     rd, fn = s3dec(rd, fn)
@@ -4214,7 +4317,13 @@ class Up2k(object):
             except:
                 pass
 
-            self._symlink(dabs, alink, flags, False, lmod=lmod or 0)
+            # this creates a link pointing from dabs to alink; alink may
+            # not exist yet, which becomes problematic if the symlinking
+            # fails and it has to fall back on hardlinking/copying files
+            # (for example a volume with symlinked dupes but no --dedup);
+            # fsrc=sabs is then a source that currently resolves to copy
+
+            self._symlink(dabs, alink, flags, False, lmod=lmod or 0, fsrc=sabs)
 
         return len(full) + len(links)
 
@@ -4243,8 +4352,11 @@ class Up2k(object):
 
         return wark
 
-    def _hashlist_from_file(self, path , prefix  = "")  :
-        fsz = bos.path.getsize(path)
+    def _hashlist_from_file(
+        self, path , prefix  = ""
+    )   :
+        st = bos.stat(path)
+        fsz = st.st_size
         csz = up2k_chunksize(fsz)
         ret = []
         suffix = " MB, {}".format(path)
@@ -4257,7 +4369,7 @@ class Up2k(object):
             while fsz > 0:
                 # same as `hash_at` except for `imutex` / bufsz
                 if self.stop:
-                    return []
+                    return [], st
 
                 if self.pp:
                     mb = fsz // (1024 * 1024)
@@ -4278,7 +4390,7 @@ class Up2k(object):
                 digest = base64.urlsafe_b64encode(digest)
                 ret.append(digest.decode("utf-8"))
 
-        return ret
+        return ret, st
 
     def _new_upload(self, job  , vfs , depth )   :
         pdir = djoin(job["ptop"], job["prel"])
@@ -4579,7 +4691,7 @@ class Up2k(object):
                 self.salt, inf.st_size, int(inf.st_mtime), rd, fn
             )
         else:
-            hashes = self._hashlist_from_file(abspath)
+            hashes, _ = self._hashlist_from_file(abspath)
             if not hashes:
                 return False
 

copyparty/util.py

@@ -239,6 +239,8 @@ IMPLICATIONS = [
     ["e2vu", "e2v"],
     ["e2vp", "e2v"],
     ["e2v", "e2d"],
+    ["hardlink_only", "hardlink"],
+    ["hardlink", "dedup"],
     ["tftpvv", "tftpv"],
     ["smbw", "smb"],
     ["smb1", "smb"],