distbackup 1.0.19__tar.gz → 1.0.21__tar.gz

{distbackup-1.0.19 → distbackup-1.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: distbackup
-Version: 1.0.19
+Version: 1.0.21
 Summary: A backup manager which can back up a large set of files to a number of offline disks
 Home-page: https://ktpanda.org/software/distbackup
 Author: Katie Rust
@@ -9,9 +9,14 @@ License: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: argcomplete>=2.0.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: ktpanda_modules>=1.0.28
+Requires-Dist: hashcopy>=1.0.5; platform_system == "Linux"
+Dynamic: license-file
 
 Distbackup
 ==========

distbackup-1.0.21/distbackup/__init__.py

@@ -0,0 +1 @@
+VERSION = "1.0.21"

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup/cli.py

@@ -432,7 +432,6 @@ def cmd_disk_list(db:BackupDB, parser, args):
 @arg('name', metavar='NAME', help='Name of new disk')
 @arg('path', metavar='PATH', type=Path, help='Path to data files')
 @arg('-s', '--size', help='Set size of selected disk')
-@arg('-p', '--path', metavar='PATH', help='Path to data files')
 def cmd_disk_add(db:BackupDB, parser, args):
     args.path.mkdir(parents=True, exist_ok=True)
     path = args.path.resolve()

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup/database.py

@@ -54,75 +54,11 @@ class BackupDB(SQLiteDB):
     # Upgrades
     ##########################################################################################
 
-    def _upgrade_to_13(self, oldvers):
-        self.backend.execute('ALTER TABLE disks ADD COLUMN relative_path TEXT')
-
-    def _upgrade_to_16(self, oldvers):
-        self.backend.execute("ALTER TABLE file_names ADD COLUMN metadata TEXT NOT NULL DEFAULT ''")
-
-    def _upgrade_to_18(self, oldvers):
-        for column in ('file_names.path', 'path_map.virtual_path', 'path_config.virtual_path', 'links.path', 'files.last_path'):
-            table, column = column.split('.')
-            sql = f"UPDATE {table} SET {column} = '/' || {column} WHERE {column} NOT LIKE '/%'"
-            self.backend.execute(sql)
-
-    def _upgrade_to_20(self, oldvers):
-        if self.backend.execute("SELECT name FROM sqlite_master WHERE name = 'source_path'"):
-            self.backend.execute('ALTER TABLE source_path RENAME TO path_map')
-
-        if self.backend.execute("SELECT name FROM sqlite_master WHERE name = 'source_config'"):
-            self.backend.execute('ALTER TABLE source_config RENAME TO path_config')
-
     def _drop_triggers(self):
         triggers = [row[0] for row in self.backend.execute("SELECT name FROM sqlite_master WHERE type = 'trigger'")]
         for trigger in triggers:
             self.backend.execute(f'DROP TRIGGER {trigger}')
 
-    def _upgrade_to_21(self, oldvers):
-        '''*BIG UPDATE* - drop all triggers, create the new tables, copy
-        the data over, then let it create the new triggers'''
-
-        self._drop_triggers()
-
-        self.exec_schema(TABLES_INDICES)
-
-        self.backend.create_function('newid', 0, new_uuid)
-        self.backend.execute('INSERT INTO object(hash, size, nexus, refs, last_path, last_modtime) SELECT hash, size, nexus, refs, last_path, last_lastmod FROM files')
-        self.backend.execute('INSERT INTO file_tree(virtual_path, last_modified, hash, ino, size, metadata) SELECT path, lastmod, hash, ino, size, metadata FROM file_names')
-        self.backend.execute('INSERT INTO symbolic_link(virtual_path, target) SELECT path, dest FROM links')
-        self.backend.execute('INSERT INTO disk(uuid, name, nexus_index, relative_path, size) SELECT IFNULL(uuid, newid()), name, id - 1, relative_path, size FROM disks')
-        self.backend.execute('INSERT INTO nexus(nexus, refs, totalsize) SELECT nexus, refs, totalsize FROM dist_nexus')
-        self.backend.execute('DROP TABLE files')
-        self.backend.execute('DROP TABLE file_names')
-        self.backend.execute('DROP TABLE links')
-        self.backend.execute('DROP TABLE disks')
-        self.backend.execute('DROP TABLE dist_nexus')
-        self.backend.execute('DROP TABLE IF EXISTS vars')
-        self.backend.execute('DROP TABLE IF EXISTS source_lock')
-
-    def _upgrade_to_22(self, oldvers):
-        self._drop_triggers()
-        self.backend.execute("ALTER TABLE file_tree ADD COLUMN maxcopies INTEGER")
-        self.backend.execute("ALTER TABLE object ADD COLUMN maxcopies INTEGER")
-
-    def _upgrade_to_23(self, oldvers):
-        self._drop_triggers()
-        self.backend.execute("ALTER TABLE nexus RENAME COLUMN count TO level")
-
-    def _upgrade_to_24(self, oldvers):
-        self.backend.execute("ALTER TABLE nexus ADD COLUMN disks TEXT NOT NULL DEFAULT ''")
-        self.update_nexus_disk_text()
-
-    def _upgrade_to_25(self, oldvers):
-        self._drop_triggers()
-        self.backend.execute('ALTER TABLE object RENAME TO old_object')
-        self.backend.execute('ALTER TABLE file_tree RENAME TO old_file_tree')
-        self.exec_schema(TABLES_INDICES)
-        self.backend.execute('INSERT INTO object(hash, size, nexus, refs, priority, last_path, last_modtime) SELECT hash, size, nexus, refs, priority, last_path, last_modtime FROM old_object')
-        self.backend.execute('INSERT INTO file_tree(virtual_path, last_modified, hash, ino, size, priority, metadata) SELECT virtual_path, last_modified, hash, ino, size, priority, metadata FROM old_file_tree')
-        self.backend.execute('DROP TABLE old_object')
-        self.backend.execute('DROP TABLE old_file_tree')
-
     def _upgrade_to_26(self, oldvers):
         self._drop_triggers()
 
@@ -158,8 +94,7 @@ class BackupDB(SQLiteDB):
             ("type = 'table' AND name = 'object'", (), lambda text: (
                 re.sub(r'(last_path TEXT)(\s*,)', r"\1 NOT NULL DEFAULT ''\2",
                        re.sub(r'(last_modtime INTEGER)(\s*,)', r"\1 NOT NULL DEFAULT 0\2", text))
-            )),
-            debug=True
+            ))
         )
 
     ##########################################################################################
@@ -193,15 +128,15 @@ class BackupDB(SQLiteDB):
     @in_transaction
     def copy_database(self, dest_dir=None):
         if dest_dir is None:
+            if not self.current_disk or not self.current_disk.data_path:
+                return
+
             dest_dir = self.current_disk.data_path
 
         self.execute_nonquery('PRAGMA wal_checkpoint(FULL)')
         if self.nocopydb:
             return
 
-        if not self.current_disk or not self.current_disk.data_path:
-            return
-
         progress = Progress.get_global()
         progress.set_status('copying database')
         tmppath = dest_dir / (DBNAME + '.new')
@@ -327,7 +262,7 @@ class BackupDB(SQLiteDB):
     @in_transaction('IMMEDIATE')
     def add_disk(self, name, path, size=None):
         uuid = new_uuid()
-        new_disk = BackupDisk(uuid, name, None, size)
+        new_disk = BackupDisk(uuid=uuid, name=name, size=size)
         new_disk.set_from_path(path)
 
         try:
@@ -402,7 +337,7 @@ class BackupDB(SQLiteDB):
     def drop_disk_objects(self):
         index = self.current_disk.nexus_index
         with self.execute('SELECT nexus, level, totalsize FROM nexus') as curs:
-            for nexus, copies, totalsize in curs:
+            for nexus, _, _ in curs:
                 if disk_in_nexus(nexus, index):
                     new_nexus = nexus_without_disk(nexus, index)
                     self.execute_nonquery('UPDATE object SET nexus = ? WHERE nexus = ?', (new_nexus, nexus))
@@ -561,7 +496,7 @@ class BackupDB(SQLiteDB):
             for uuid, config in data['disks'].items():
                 realuuid = self.query_scalar('SELECT uuid FROM disk WHERE uuid = ?', (canonical_uuid(uuid),))
                 if realuuid is None:
-                    raise ValueError(f'Disk with UUID {realuuid} not in database (disks cannot be added with import)')
+                    raise ValueError(f'Disk with UUID {uuid} not in database (disks cannot be added with import)')
 
                 if 'size_bytes' in config:
                     size = int(str(config['size_bytes']).replace(',', ''))
@@ -627,10 +562,10 @@ class BackupDB(SQLiteDB):
         with self.execute(sql, args) as curs:
             for row in curs:
                 path = row[0]
+                relpath = path.lstrip('/')
                 if rx and not rx.match(path):
                     continue
-
-                if any((base / path).exists() for base in existing):
+                if any((base / relpath).exists() for base in existing):
                     continue
 
                 yield row

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup/disk.py

@@ -19,7 +19,7 @@ log = logging.getLogger(__name__)
 
 RX_HEX = re.compile(r'^[0-9a-fA-F]+$')
 
-def find_mountpoint(path, partition_by_mountpoint):
+def find_mountpoint(path:Path, partition_by_mountpoint:dict[str, 'Partition']) -> tuple[Path, str, int, Optional['Partition']]:
     path = path.resolve()
     cpath = path
     cstat = path.stat()
@@ -46,33 +46,37 @@ class Partition:
     fsuuid: Optional[str] = None
 
     @classmethod
-    def list(cls):
-        rtn = []
-        if sys.platform == 'linux':
-            try:
-                p = subprocess.run(
-                    'lsblk --json -p -b -o name,size,label,uuid,mountpoint,fstype,fsavail,fssize --list'.split(),
-                    encoding='ascii', stdout=subprocess.PIPE, check=True
+    def _list_linux(cls):
+        try:
+            p = subprocess.run(
+                'lsblk --json -p -b -o name,size,label,uuid,mountpoint,fstype,fsavail,fssize --list'.split(),
+                encoding='ascii', stdout=subprocess.PIPE, check=True
+            )
+            jsdata = json.loads(p.stdout)
+            rtn = []
+            for dev in jsdata['blockdevices']:
+                mountpoint = dev.get('mountpoint')
+                part = cls(
+                    device=dev.get('name'),
+                    mountpoint=Path(mountpoint) if mountpoint else None,
+                    size=dev.get('fssize') or dev.get('size'),
+                    label=dev.get('label'),
+                    fstype=dev.get('fstype'),
+                    fsuuid=dev.get('uuid'),
                 )
-                jsdata = json.loads(p.stdout)
-                for dev in jsdata['blockdevices']:
-                    mountpoint = dev.get('mountpoint')
-                    part = cls(
-                        device=dev.get('name'),
-                        mountpoint=Path(mountpoint) if mountpoint else None,
-                        size=dev.get('fssize') or dev.get('size'),
-                        label=dev.get('label'),
-                        fstype=dev.get('fstype'),
-                        fsuuid=dev.get('uuid'),
-                    )
-                    if part.mountpoint or part.fsuuid:
-                        rtn.append(part)
-
-                return rtn
-
-            except (OSError, ValueError, KeyError) as e:
-                log.warning(f'Unable to use `lsblk`: {e}', file=sys.stderr)
+                if part.mountpoint or part.fsuuid:
+                    rtn.append(part)
 
+            return rtn
+
+        except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as e:
+            log.warning(f'Unable to use `lsblk`: {e}')
+            # Fallback to using psutil if lsblk fails
+            return cls._list_psutil()
+
+    @classmethod
+    def _list_psutil(cls):
+        rtn = []
         for disk in psutil.disk_partitions():
             if not disk.mountpoint:
                 continue
@@ -80,6 +84,13 @@ class Partition:
             rtn.append(cls(device=disk.device, mountpoint=Path(disk.mountpoint), fstype=disk.fstype))
         return rtn
 
+    @classmethod
+    def list(cls):
+        if sys.platform == 'linux':
+            return cls._list_linux()
+
+        return cls._list_psutil()
+
 @dataclass
 class BackupDisk:
     uuid: str
@@ -126,7 +137,7 @@ class BackupDisk:
 
     def set_from_path(self, path: Path):
         partitions = Partition.list()
-        partition_by_mountpoint = {part.mountpoint: part for part in partitions if part.mountpoint}
+        partition_by_mountpoint = {str(part.mountpoint): part for part in partitions if part.mountpoint}
         mountpt, relpath, totalsize, partition = find_mountpoint(path, partition_by_mountpoint)
 
         if not self.size:
@@ -199,7 +210,7 @@ class BackupDisk:
                     rdisk = BackupDisk.read_config(part.mountpoint / disk.relative_path)
                     if rdisk:
                         if rdisk.uuid != disk.uuid:
-                            log.warning(f'Disk on partition {part.fsuuid} ({rdisk.data_path}) does not match expected UUID!', file=sys.stderr)
+                            log.warning(f'Disk on partition {part.fsuuid} ({rdisk.data_path}) does not match expected UUID!')
                             log.warning(f'  Expected: {disk.name} {disk.uuid}')
                             log.warning(f'  Got     : {rdisk.name} {rdisk.uuid}')
                             continue

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup/restore.py

@@ -116,24 +116,38 @@ class Restore:
         for z in sorted(all_file_nexus):
             log.info(' '.join((disk_names.get(j, '?') if (z&(1<<j)) else '--').ljust(10) for j in disk_seq))
 
+        # Start with no disks chosen with all groups remaining
         found_list = None
         search_queue = deque()
         search_queue.append((0, all_file_nexus))
+
         while search_queue:
             chosen_disks, sets = search_queue.popleft()
+
+            # All disk IDs that haven't been chosen at this step - candidates for choosing next
             disk_set = [(disk_id, set()) for disk_id in disk_seq if not chosen_disks & (1<<disk_id)]
             for disk_id, next_set in disk_set:
                 mask = 1 << disk_id
-                if mask & chosen_disks:
-                    continue
+                # needed? filtered out in list comp above
+                #if mask & chosen_disks:
+                #    continue
+                assert (mask & chosen_disks) == 0
+
+                # Remove any nexus groups that reference disk_id, giving us the remaining disks
                 for nexus_set in sets:
                     if not nexus_set & mask:
                         next_set.add(nexus_set & ~mask)
 
+            # Sort the list of candidates by the size of the remaining set, smallest first
             disk_set.sort(key=lambda v: len(v[1]))
+
+            # Keep up to 4 candidates to reduce the search space
             del disk_set[4:]
+
+            # Queue up the next level of candidates
             for disk_id, next_set in disk_set:
                 new_chosen = chosen_disks | (1 << disk_id)
+                # If the next set is empty, then new_chosen covers all files.
                 if not next_set:
                     found_list = new_chosen
                     break

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup/update.py

@@ -169,13 +169,18 @@ class Updater:
     def lock_path(self, vpath):
         log.info(f'{vpath} is locked')
 
+        vpath_without_slash = vpath.rstrip('/')
+        vpath_with_slash = vpath_without_slash + '/'
+
         links = self.symlink_unseen
-        locked_links = [k for k in links if k.startswith(vpath)]
+        links.pop(vpath_without_slash, None)
+        locked_links = [k for k in links if k.startswith(vpath_with_slash)]
         for k in locked_links:
             del links[k]
 
         bypath = self.row_by_path
-        locked_files = [k for k in bypath if k.startswith(vpath)]
+        bypath.pop(vpath_without_slash, None)
+        locked_files = [k for k in bypath if k.startswith(vpath_with_slash)]
         for k in locked_files:
             del bypath[k]
 
@@ -246,23 +251,13 @@ class Updater:
                     continue
 
                 try:
-                    fstat = lstat = f.lstat()
+                    lstat = f.stat() if follow_links else f.lstat()
                 except OSError as e:
-                    log.info(f'WARNING: cannot stat {f} ({e})')
+                    log.info(f'WARNING: cannot stat {f} ({e}), locking')
+                    self.lock_path(virtual_path)
                     continue
 
-                if follow_links and S_ISLNK(lstat.st_mode):
-                    try:
-                        fstat = f.stat()
-                    except OSError as e:
-                        # If we encounter a broken symbolic link when 'follow_symlinks' is
-                        # on, assume the destination is not mounted and lock it to prevent
-                        # its contents from being removed from the tree.
-                        log.info(f'WARNING: cannot follow link {f} ({e}), locking')
-                        self.lock_path(virtual_path)
-                        continue
-
-                srcf = SourceFile(f, fstat)
+                srcf = SourceFile(f, lstat)
 
                 srcf.read_meta()
                 if srcf.priority is None:

distbackup-1.0.21/distbackup/utils.py

@@ -0,0 +1,395 @@
+import sys
+import os
+import json
+import re
+import uuid
+import fnmatch
+import traceback
+from pathlib import Path
+
+# Third-party imports
+from ktpanda import fileutils
+from ktpanda.textcolor import Colorizer
+
+class CacheDict(dict):
+    '''Works like `defaultdict`, except it passes the key to the factory function'''
+    def __init__(self, factory):
+        super().__init__()
+        self.factory = factory
+
+    def __missing__(self, key):
+        r = self[key] = self.factory(key)
+        return r
+
+class Progress:
+    '''
+    Class to track and display progress of an operation.
+    Provides a global instance for convenience.
+    '''
+    _global_instance = None
+
+    def __init__(self):
+        self.current_percent:float = 0.0
+        self.converted_progress:int = 0
+        self.status:str = ''
+
+    @classmethod
+    def set_global(cls, progress):
+        cls._global_instance = progress
+
+    @classmethod
+    def get_global(cls):
+        return cls._global_instance or NullProgress()
+
+    def set_status(self, status:str):
+        self.status = status
+        self.display()
+
+    def _convert_percent(self, percent:float):
+        return int(percent * 10)
+
+    def progress(self, percent:float):
+        if percent is None:
+            self.reset()
+            return
+
+        new_progress = self._convert_percent(percent)
+        if new_progress != self.converted_progress:
+            self.current_percent = percent
+            self.converted_progress = new_progress
+            self.display()
+
+    def reset(self):
+        self.status = ''
+        self.current_percent = 0.0
+        self.converted_progress = self._convert_percent(0.0)
+        self.clear()
+
+    def display(self):
+        raise NotImplementedError
+
+    def clear(self):
+        raise NotImplementedError
+
+class NullProgress(Progress):
+    def display(self):
+        pass
+
+    def clear(self):
+        pass
+
+class ProgressBar(Progress):
+    '''
+    Progress bar implementation of the `Progress` class.
+    Displays a textual progress bar in the terminal.
+    '''
+    def display(self):
+        barsz = 90 - len(self.status) - 12
+        barfill = int(self.current_percent * barsz / 100)
+        text = f'\r\033[?7l {self.status} {self.current_percent:5.1f}% |{"=" * barfill}{" " * (barsz - barfill)}|\r\033[?7h'
+        sys.stdout.write(text)
+        sys.stdout.flush()
+
+    def clear(self):
+        sys.stdout.write('\r\033[K')
+        sys.stdout.flush()
+
+def progress(pct=None):
+    '''Convenience function to update the global progress instance.'''
+    if Progress._global_instance:
+        Progress._global_instance.progress(pct)
+
+_unit = list("BKMGTPE")
+_unit_color = [
+    'fff',
+    '05f',
+    '3f3',
+    'fc0',
+    'c0f',
+    '0f9',
+    '0f0',
+]
+
+def _prettysize(n):
+    '''
+    Helper function for `prettysize` and `prettysize_clr`.
+    Returns a tuple `(prefix, number, magnitude)` where `prefix` is '-' for negative numbers or '' for non-negative,
+    `number` is the formatted number as a string, and `magnitude` is the index of the unit in `_unit`.
+    '''
+    neg = n < 0
+    if neg:
+        n = -n
+    un = 0
+    while n >= 1000.0:
+        n /= 1000.0
+        un += 1
+    pfx = ('-' if neg else '')
+    if un:
+        return pfx, format(n, '.2f'), un
+    return pfx, format(n, 'd'), 0
+
+def prettysize(n):
+    '''
+    Convert a size in bytes to a human-readable string with appropriate unit suffix, in SI units (powers of 1000, not 1024)
+    '''
+    pfx, num, magnitude = _prettysize(n)
+    return f'{pfx}{num}{_unit[magnitude]}'
+
+def size_color(n):
+    '''
+    Return the color code associated with the magnitude of the given size `n`.
+    '''
+    _, _, magnitude = _prettysize(n)
+    return _unit_color[magnitude]
+
+def prettysize_clr(n, padding=0):
+    '''
+    Convert a size in bytes to a human-readable string with appropriate unit suffix and color.
+    The result is right-justified to the specified `padding` width.
+    Returns a `Colorizer` instance with the text.
+    '''
+    pfx, num, magnitude = _prettysize(n)
+    return Colorizer().t(f'{pfx}{num}{_unit[magnitude]}'.rjust(padding), fg=_unit_color[magnitude])
+
+RX_COMMA = re.compile(r'(\d\d\d)(?=\d)')
+def addcomma(n:int|float|str) -> str:
+    '''
+    Format a number with commas as thousands separators. Leaves the decimal part unchanged.
+    Can be called with an integer, float, or string representing a number (without any separators).
+    '''
+    # Admittedly, this is very US-centric. I verified that str(<float>) always uses '.' as the decimal separator,
+    # even in locales like de-DE where ',' is used as the decimal separator. Other locales may use commas, but
+    # place them differently. Standard convention would be to use '_' as a separator, but I personally prefer ','
+    # since I made this tool for my own use.
+
+    intpart, sep, fracpart = str(n).partition('.')
+    # Reverse the integer part, insert commas every three digits, then reverse it back.
+    return RX_COMMA.sub(r'\1,', intpart[::-1])[::-1] + sep + fracpart
+
+mul = dict(k=1000, m=1000000, g=1000000000, t=1000000000000)
+def parsesize(s):
+    '''
+    Parse a human-readable size string (e.g., '10k', '5M', '2G') and return the size in bytes as an integer.
+    Supports 'k', 'm', 'g', 't' suffixes for kilo, mega, giga, and tera respectively.
+    '''
+    if s is None:
+        return None
+
+    s = s.lower().replace(',', '')
+    m = mul.get(s[-1:])
+    if m:
+        return int(float(s[:-1]) * m)
+    return int(s)
+
+def read_file(p:Path, default=None):
+    '''
+    Read the contents of a file at path `p` and return it as a string. Any whitespace at the beginning or end is stripped.
+    If the file does not exist, return the `default` value.
+    '''
+    try:
+        r = p.read_text(encoding='utf-8').strip()
+        return r or default
+    except FileNotFoundError:
+        return default
+
+def write_file(p:Path, data: str):
+    '''
+    Write the string `data` to a file at path `p`, appending a newline at the end.
+    The file is written with UTF-8 encoding.
+    Uses a temporary file and atomic replace to ensure safe writes.
+    '''
+    with fileutils.save_file(p, mode='wb') as fp:
+        fp.write(data.encode('utf-8') + b'\n')
+        fp.flush()
+        os.fsync(fp.fileno())
+
+def try_unlink(p:str|Path):
+    '''
+    Try to unlink (delete) the file at path `p`.
+    Returns True if the file was successfully deleted, False if an error occurred.
+    '''
+    try:
+        os.unlink(p)
+        return True
+    except OSError:
+        return False
+
+def check_dir(f:Path):
+    '''
+    Ensure that the parent directory of the file `f` exists.
+    Creates the directory and any necessary parent directories if they do not exist.
+    '''
+    f.parent.mkdir(parents=True, exist_ok=True)
+
+def compact_json(data):
+    '''
+    Return a compact JSON representation of the given `data` object.
+    '''
+    return json.dumps(data, separators=(',',':'))
+
+def canonical_path(path:str, tslash:bool|None=False) -> str:
+    '''
+    Canonicalize the given file path `path`.
+    Collapses multiple slashes, and removes extra slashes from the beginning.
+    If `tslash` is False (default), the resulting path will not have a trailing slash.
+    If `tslash` is True, the resulting path will always have a trailing slash.
+    If `tslash` is None, the resulting path will have a trailing slash only if `path` has one.
+    '''
+    if path is None:
+        return None
+
+    # Collapse all multiple slashes into one
+    path = re.sub(r'[\\/]+', '/', path)
+
+    # Remove extra slashes from the beginning, and if `tslash` is not None, also remove trailing slashes.
+    path = '/' + (path.lstrip('/') if tslash is None else path.strip('/'))
+
+    # Add the trailing slash back if `tslash` is True, but not if the path is just '/'.
+    if tslash and path != '/':
+        path += '/'
+    return path
+
+##########################################################################################
+# Functions exported to SQLite
+##########################################################################################
+
+SQL_FUNCS = {}
+def sqlf(f):
+    '''
+    Decorator to register a function as an SQLite-exported function.
+    The decorated function will be added to the `SQL_FUNCS` dictionary.
+    '''
+    def ret(*args):
+        try:
+            return f(*args)
+        except Exception:
+            traceback.print_exc()
+            raise
+    SQL_FUNCS[f.__name__] = ret
+    return f
+
+@sqlf
+def disk_in_nexus(nexus, nexus_index):
+    '''
+    Returns True if the disk at the specified `nexus_index` is present in the `nexus` string, False otherwise.
+    '''
+    if nexus_index < 0 or nexus_index >= len(nexus):
+        return False
+    return nexus[nexus_index] == '1'
+
+@sqlf
+def nexus_with_disk(nexus, nexus_index):
+    '''
+    Returns a new `nexus` string with the disk at the specified `nexus_index` marked as present ('1').
+    If the `nexus_index` is beyond the current length of the `nexus` string, the string is extended with '0's as needed.
+    '''
+    if nexus_index >= len(nexus):
+        return nexus + '0' * (nexus_index - len(nexus)) + '1'
+    return nexus[:nexus_index] + '1' + nexus[nexus_index + 1:]
+
+@sqlf
+def nexus_without_disk(nexus, nexus_index):
+    '''
+    Returns a new `nexus` string with the disk at the specified `nexus_index` marked as absent ('0').
+    Trailing '0's are removed from the resulting string.
+    '''
+    if nexus_index >= len(nexus):
+        return nexus
+    return (nexus[:nexus_index] + '0' + nexus[nexus_index + 1:]).rstrip('0')
+
+@sqlf
+def nexus_level(nexus):
+    '''
+    Returns the number of disks present in the `nexus` string, i.e., the count of '1' characters.
+    '''
+    return nexus.count('1')
+
+@sqlf
+def new_uuid():
+    '''
+    Generate a new UUID and return it as a string.
+    '''
+    return str(uuid.uuid4())
+
+@sqlf
+def canonical_uuid(val):
+    '''
+    Convert the given UUID string `val` to its canonical form.
+    '''
+    return str(uuid.UUID(val))
+
+def prefix_upper(prefix):
+    '''
+    Return the next string after the given `prefix` in lexicographical order.
+    This is used to construct SQL range queries for prefix matching.
+    '''
+    return prefix[:-1] + chr(1 + ord(prefix[-1:]))
+
+def prefix_sql(column, prefix):
+    '''
+    Construct an SQL range query for the given `column` to match the specified `prefix`.
+    Returns a tuple `(query, params)` where `query` is the SQL condition string and `params` is a tuple of parameters.
+    '''
+    return f'({column} >= ? AND {column} < ?)', (prefix, prefix_upper(prefix))
+
+def escape_like(text):
+    '''
+    Escape special characters in the given `text` for use in an SQL LIKE query.
+    '''
+    return text.replace('\\', r'\\').replace('%', r'\%').replace('_', r'\_')
+
+def hash_relpath(hash):
+    '''
+    Returns the on-disk path for the given object `hash`.
+    '''
+    return f'distbackup-objects/{hash[:3]}/{hash}'
+
+def pattern_to_regex(pat):
+    '''
+    Convert a shell-style pattern `pat` such as '*.txt' to a compiled regular expression and a prefix string.
+    The returned regular expression will also match any files in subdirectories of the specified prefix.
+    Returns a tuple `(regex, prefix)`, where `regex` is a compiled regular expression matching the pattern,
+    and `prefix` is the longest literal prefix of the pattern.
+    If the pattern is empty, returns `(None, None)`.
+    '''
+    if not pat:
+        return None, None
+
+    pat = canonical_path(pat, None)
+
+    prefix = None
+    m = re.match(r'^([^\*\[\?]+)', pat)
+    if m:
+        prefix = m.group(1)
+
+    rxpattern = fnmatch.translate(pat)
+
+    # fnmatch.translate tacks on \Z or \z (depending on Python version) to match the end of the string
+    if rxpattern.endswith(r'\Z') or rxpattern.endswith(r'\z'):
+        rxpattern = rxpattern[:-2]
+
+    # Match either an exact file, or any file in a subdirectory
+    if pat.endswith('/'):
+        rxpattern += r'(.*)?\Z'
+    else:
+        rxpattern += r'(/.*)?\Z'
+
+    return re.compile(rxpattern, re.S), prefix
+
+def round_blocks(s):
+    '''
+    Round the given size `s` up to the nearest multiple of 4096, then adds one extra block of 4096 to account for metadata.
+    '''
+    return 4096 + ((s + 4095) & ~4095)
+
+def int_mtime(m):
+    '''
+    Convert the given modification time `m` to an integer representing milliseconds since the epoch.
+    '''
+    return int(m * 1000)
+
+def restore_mtime(m):
+    '''
+    Convert the given integer modification time `m` representing milliseconds since the epoch back to a floating-point number of seconds.
+    '''
+    return m / 1000.0

{distbackup-1.0.19 → distbackup-1.0.21}/distbackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: distbackup
-Version: 1.0.19
+Version: 1.0.21
 Summary: A backup manager which can back up a large set of files to a number of offline disks
 Home-page: https://ktpanda.org/software/distbackup
 Author: Katie Rust
@@ -9,9 +9,14 @@ License: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: argcomplete>=2.0.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: ktpanda_modules>=1.0.28
+Requires-Dist: hashcopy>=1.0.5; platform_system == "Linux"
+Dynamic: license-file
 
 Distbackup
 ==========

{distbackup-1.0.19 → distbackup-1.0.21}/setup.cfg

@@ -14,7 +14,7 @@ classifiers =
 	Operating System :: OS Independent
 
 [options]
-python_requires = >=3.8
+python_requires = >=3.10
 install_requires = 
 	argcomplete>=2.0.0
 	psutil>=5.9.0

distbackup-1.0.19/distbackup/__init__.py

@@ -1 +0,0 @@
-VERSION = "1.0.19"

distbackup-1.0.19/distbackup/utils.py

@@ -1,272 +0,0 @@
-import sys
-import os
-import json
-import re
-import uuid
-import argparse
-import fnmatch
-import traceback
-from ktpanda.textcolor import Colorizer
-
-class CacheDict(dict):
-    '''Works like `defaultdict`, except it passes the key to the factory function'''
-    def __init__(self, factory):
-        super().__init__()
-        self.factory = factory
-
-    def __missing__(self, key):
-        r = self[key] = self.factory(key)
-        return r
-
-class Progress:
-    _global_instance = None
-
-    def __init__(self):
-        self.current_percent:float = 0.0
-        self.converted_progress:int = 0
-        self.status:str = ''
-
-    @classmethod
-    def set_global(cls, progress):
-        cls._global_instance = progress
-
-    @classmethod
-    def get_global(cls):
-        return cls._global_instance or NullProgress()
-
-    def set_status(self, status:str):
-        self.status = status
-        self.display()
-
-    def _convert_percent(self, percent:float):
-        return int(percent * 10)
-
-    def progress(self, percent:float):
-        if percent is None:
-            self.reset()
-            return
-
-        new_progress = self._convert_percent(percent)
-        if new_progress != self.converted_progress:
-            self.current_percent = percent
-            self.converted_progress = new_progress
-            self.display()
-
-    def reset(self):
-        self.status = ''
-        self.current_percent = 0.0
-        self.converted_progress = self._convert_percent(0.0)
-        self.clear()
-
-    def display(self):
-        raise NotImplementedError
-
-    def clear(self):
-        raise NotImplementedError
-
-class NullProgress(Progress):
-    def display(self):
-        pass
-
-    def clear(self):
-        pass
-
-class ProgressBar(Progress):
-    def display(self):
-        barsz = 90 - len(self.status) - 12
-        barfill = int(self.current_percent * barsz / 100)
-        text = f'\r\033[?7l {self.status} {self.current_percent:5.1f}% |{"=" * barfill}{" " * (barsz - barfill)}|\r\033[?7h'
-        sys.stdout.write(text)
-        sys.stdout.flush()
-
-    def clear(self):
-        sys.stdout.write('\r\033[K')
-        sys.stdout.flush()
-
-def progress(pct=None):
-    if Progress._global_instance:
-        Progress._global_instance.progress(pct)
-
-_unit = list("BKMGTPE")
-_unit_color = [
-    'fff',
-    '05f',
-    '3f3',
-    'fc0',
-    'c0f',
-    '0f9',
-    '0f0',
-]
-
-def _prettysize(n):
-    neg = n < 0
-    if neg:
-        n = -n
-    un = 0
-    while n >= 1000.0:
-        n /= 1000.0
-        un += 1
-    pfx = ('-' if neg else '')
-    if un:
-        return pfx, format(n, '.2f'), un
-    return pfx, format(n, 'd'), 0
-
-def prettysize(n):
-    pfx, num, magnitude = _prettysize(n)
-    return f'{pfx}{num}{_unit[magnitude]}'
-
-def size_color(n):
-    _, _, magnitude = _prettysize(n)
-    return _unit_color[magnitude]
-
-def prettysize_clr(n, padding=0):
-    pfx, num, magnitude = _prettysize(n)
-    return Colorizer().t(f'{pfx}{num}{_unit[magnitude]}'.rjust(padding), fg=_unit_color[magnitude])
-
-RX_COMMA = re.compile(r'(\d\d\d)(?=\d)')
-def addcomma(n):
-    n, sep, d = str(n).partition('.')
-    return RX_COMMA.sub(r'\1,', n[::-1])[::-1] + sep + d
-
-mul = dict(k=1000, m=1000000, g=1000000000, t=1000000000000)
-def parsesize(s):
-    if s is None:
-        return None
-
-    s = s.lower().replace(',', '')
-    m = mul.get(s[-1:])
-    if m:
-        return int(float(s[:-1]) * m)
-    return int(s)
-
-def read_file(p, default=None):
-    try:
-        r = p.read_text().strip()
-        return r or default
-    except FileNotFoundError:
-        return default
-
-def write_file(p, data=None):
-    tmpf = p.with_name(p.name + '~')
-    with tmpf.open('w') as fp:
-        fp.write(data + '\n')
-        fp.flush()
-        os.fsync(fp.fileno())
-    tmpf.replace(p)
-
-def try_unlink(p):
-    try:
-        os.unlink(p)
-        return True
-    except OSError:
-        return False
-
-def check_dir(f):
-    f.parent.mkdir(parents=True, exist_ok=True)
-
-def compact_json(data):
-    return json.dumps(data, separators=(',',':'))
-
-def canonical_path(path, tslash=False):
-    if path is None:
-        return None
-
-    # Collapse all multiple slashes into one
-    path = re.sub(r'[\\/]+', '/', path)
-
-    # Remove extra slashes from the beginning
-    path = '/' + (path.lstrip('/') if tslash is None else path.strip('/'))
-    if tslash and path != '/':
-        path += '/'
-    return path
-
-##########################################################################################
-# Functions exported to SQLite
-##########################################################################################
-
-SQL_FUNCS = {}
-def sqlf(f):
-    def ret(*args):
-        try:
-            return f(*args)
-        except Exception:
-            traceback.print_exc()
-            raise
-    SQL_FUNCS[f.__name__] = ret
-    return f
-
-@sqlf
-def disk_in_nexus(nexus, nexus_index):
-    if nexus_index < 0 or nexus_index >= len(nexus):
-        return False
-    return nexus[nexus_index] == '1'
-
-@sqlf
-def nexus_with_disk(nexus, nexus_index):
-    if nexus_index >= len(nexus):
-        return nexus + '0' * (nexus_index - len(nexus)) + '1'
-    return nexus[:nexus_index] + '1' + nexus[nexus_index + 1:]
-
-@sqlf
-def nexus_without_disk(nexus, nexus_index):
-    if nexus_index >= len(nexus):
-        return nexus
-    return (nexus[:nexus_index] + '0' + nexus[nexus_index + 1:]).rstrip('0')
-
-@sqlf
-def nexus_level(nexus):
-    return nexus.count('1')
-
-@sqlf
-def new_uuid():
-    return str(uuid.uuid4())
-
-@sqlf
-def canonical_uuid(val):
-    return str(uuid.UUID(val))
-
-def prefix_upper(prefix):
-    return prefix[:-1] + chr(1 + ord(prefix[-1:]))
-
-def prefix_sql(column, prefix):
-    return f'({column} >= ? AND {column} < ?)', (prefix, prefix_upper(prefix))
-
-def escape_like(text):
-    return text.replace('\\', r'\\').replace('%', r'\%').replace('_', r'\_')
-
-def hash_relpath(hash):
-    return f'distbackup-objects/{hash[:3]}/{hash}'
-
-def pattern_to_regex(pat):
-    if not pat:
-        return None, None
-
-    pat = canonical_path(pat, None)
-
-    prefix = None
-    m = re.match(r'^([^\*\.\[\?]+)', pat)
-    if m:
-        prefix = m.group(1)
-
-    rxpattern = fnmatch.translate(pat)
-
-    # fnmatch.translate tacks on \Z to match the end of the string
-    if rxpattern.endswith(r'\Z'):
-        rxpattern = rxpattern[:-2]
-
-    # Match either an exact file, or any file in a subdirectory
-    if pat.endswith('/'):
-        rxpattern += r'(.*)?\Z'
-    else:
-        rxpattern += r'(/.*)?\Z'
-
-    return re.compile(rxpattern, re.S), prefix
-
-def round_blocks(s):
-    return 4096 + (s + 4095) & ~4095
-
-def int_mtime(m):
-    return int(m * 1000)
-
-def restore_mtime(m):
-    return m / 1000.0