batchmp 1.4__py3-none-any.whl

batchmp/fstools/fsutils.py

@@ -0,0 +1,272 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+import os, sys, fnmatch, shutil
+import hashlib
+
+class UniqueDirNamesChecker:
+    ''' Unique file names Helper
+    '''
+    def __init__(self, src_dir, *, unique_fnames = None):
+        self._uname_gen = unique_fnames() if unique_fnames else FSH.unique_fnames()
+
+        # init the generator function with file names from given source directory
+        fnames = [fname for fname in os.listdir(src_dir)]
+                                            #if os.path.isfile(os.path.join(src_dir, fname))]
+        for fname in fnames:
+            next(self._uname_gen)
+            self._uname_gen.send(fname)
+
+    def unique_name(self, fname):
+        ''' Returns unique file name
+        '''
+        next(self._uname_gen)
+        return self._uname_gen.send(fname)
+
+
+class FSH:
+    ''' FS helper utilities
+    '''
+    @staticmethod
+    def full_path(path, check_parent_path = False):
+        if path:
+            path = os.path.expanduser(path)
+            path = os.path.expandvars(path)
+            path = os.path.abspath(path)
+            path = os.path.realpath(path)
+
+        # for files, check that the parent dir exists
+        if check_parent_path:
+            if not os.access(os.path.dirname(path), os.W_OK):
+                print('Non-valid path:\n\t "{}"'.format(path))
+                sys.exit(1)
+
+        return path if path else None
+
+    @staticmethod
+    def path_components(path):
+        path = FSH.full_path(path)
+        return path.split(os.path.sep) if path else None        
+
+    @staticmethod
+    def path_extension(path):
+        components = FSH.path_components(path)
+        return os.path.splitext(components[-1])[1][1:] if components else None    
+
+    @staticmethod
+    def is_subdir(subdir_path, parent_path):
+        subdir_path = FSH.full_path(subdir_path)
+        parent_path = FSH.full_path(parent_path)
+
+        relative = os.path.relpath(subdir_path, start=parent_path)
+
+        return not relative.startswith(os.pardir)
+
+    @staticmethod
+    def level_from_root(root_path, nested_path):
+        ''' determines the level from root_path folder
+        '''
+        return FSH.full_path(nested_path).count(os.path.sep) - \
+                            FSH.full_path(root_path).count(os.path.sep)
+
+    @staticmethod
+    def folders_at_level(src_dir, target_level):
+        ''' generates a sequence of folders at given level from src_dir
+        '''
+        for r,d,f in os.walk(src_dir):
+           if FSH.level_from_root(src_dir, r) == target_level:
+                yield FSH.full_path(r)
+
+    @staticmethod
+    def remove_folders_below_target_level(src_dir, target_level=sys.maxsize, empty_only=True, non_empty_msg = None):
+        ''' removes folders below target level
+        '''
+        folders_removed = 0
+        for tpath in FSH.folders_at_level(src_dir, target_level):
+            for crpath, dnames, _ in os.walk(tpath, topdown = False):
+                for dname in dnames:
+                    dpath = os.path.join(crpath, dname)
+                    if not (empty_only and os.listdir(dpath)):
+                        folders_removed +=1
+                        shutil.rmtree(dpath)
+                    else:
+                        print('not empty: {}'.format(dpath))
+                        if non_empty_msg:
+                            print(non_empty_msg)
+        return folders_removed
+
+    @staticmethod
+    def unique_fnames():
+        ''' default unique file names generator method,
+            via appending a simple numbering pattern
+        '''
+        unique_names = {}
+        while True:
+            fname = yield
+            while True:
+                if fname in unique_names:
+                    unique_names[fname] += 1
+                    name_base, name_ext = os.path.splitext(fname)
+                    fname = '{0}_{1}{2}'.format(name_base, unique_names[fname], name_ext)
+                else:
+                    unique_names[fname] = 0
+                    yield fname
+                    break
+
+    @staticmethod
+    def fs_size(size, kb_1024=False):
+        ''' human readable file system entry size
+        '''
+        if size < 0:
+            raise ValueError('File size can not be negative')
+        unit_sfx = {1000: ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
+                    1024: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']}
+        div = 1024 if kb_1024 else 1000
+        for suffix in unit_sfx[div]:
+            size /= div
+            if size < div:
+                if suffix in ('KB', 'KiB'):
+                    return '{0:.0f}{1}'.format(size, suffix)
+                else:
+                    return '{0:.1f}{1}'.format(size, suffix)
+        raise ValueError('File is way too large')
+
+    @staticmethod
+    def dir_size(dir_path, shared_cache = None):
+        ''' Calculates directory size in bytes
+        '''
+        total_size = 0
+        use_shared_cache = isinstance(shared_cache, dict)
+
+        for r, _, fnames in os.walk(dir_path):   
+            # for repetitive calls, look to get from provided shared cache
+            if use_shared_cache:
+                r_size_from_cache = shared_cache.get(r)
+                if r_size_from_cache:
+                    total_size += r_size_from_cache
+                    continue
+
+            # caculate current root size
+            r_size = 0            
+            r_size += os.path.getsize(r)
+            for fname in fnames:
+                fpath = os.path.join(r, fname)
+                try:
+                    stat = os.stat(fpath)
+                except OSError:
+                    continue
+                r_size += stat.st_size
+
+            total_size += r_size
+            if use_shared_cache:
+                shared_cache[r] = r_size
+
+        return total_size
+
+    @staticmethod
+    def file_md5(fpath, block_size=0, hex=False):
+        ''' Calculates MD5 hash for a file at fpath
+        '''
+        md5 = hashlib.md5()
+        if block_size == 0:
+            block_size = 128 * md5.block_size
+        with open(fpath,'rb') as f:
+            for chunk in iter(lambda: f.read(block_size), b''):
+                md5.update(chunk)
+        return md5.hexdigest() if hex else md5.digest()
+
+    @staticmethod
+    def files(src_dir, *, recursive = False, pass_filter = None):
+        ''' list of files passing specified filter
+        '''
+        if not pass_filter:
+            pass_filter = lambda f: True
+        if recursive:
+            fpathes = [os.path.join(r,f) for r,d,files in os.walk(src_dir)
+                                                for f in files if pass_filter(f)]
+        else:
+            fpathes = (os.path.join(src_dir, fname) for fname in os.listdir(src_dir)
+                                                                    if pass_filter(fname))
+            fpathes = [f for f in fpathes if os.path.isfile(f)]
+
+        return fpathes
+
+    @staticmethod
+    def move_FS_entry(orig_path, target_path,
+                      check_unique = True,
+                      quiet = False, stop = False):
+        succeeded = False
+        try:
+            if check_unique and os.path.exists(target_path):
+                raise OSError('\nTarget path entry already exists')
+            shutil.move(orig_path, target_path)
+            succeeded = True
+        except OSError as e:
+            if not quiet:
+                print(str(e))
+                print('Failed to move entry:\n\t{0}\n\t{1}'.format(orig_path, target_path))
+                print('Exiting...') if stop else print('Skipping...')
+            if stop:
+                sys.exit(1)
+        return succeeded
+
+    @staticmethod
+    def remove_FS_entry(entry_path, include_read_only = False):
+        ''' Remove files / dirs,
+            can deal with with read-only files
+        '''
+        def check_writable(fpath):
+            if include_read_only and (not os.access(fpath, os.W_OK)):
+                os.chmod(fpath, stat.S_IWUSR)
+                return True
+            else:
+                return False
+
+        def onerror(func, fpath, exc_info):
+            if check_writable(fpath):
+                func(fpath)
+            else:
+                raise
+
+        entry_path = FSH.full_path(entry_path)
+        if os.path.isfile(entry_path):
+            check_writable(entry_path)
+            os.remove(entry_path)
+
+        elif os.path.isdir(entry_path):
+            shutil.rmtree(entry_path, onerror = onerror)
+
+    @staticmethod
+    def fs_entry(fpath):
+        pass
+
+
+# Quick dev test
+if __name__ == '__main__':
+    from batchmp.commons.utils import timed
+    path = os.path.realpath(os.path.dirname(__file__))
+
+    @timed
+    def dir_size_test(n):    
+        for i in range(n):
+            size = FSH.dir_size(path, shared_cache)
+        return size
+
+    repeat_cnt = 5000
+    shared_cache = {}
+    print(dir_size_test(repeat_cnt))    
+
+    shared_cache = None
+    print(dir_size_test(repeat_cnt))    
+

batchmp/fstools/rename.py

@@ -0,0 +1,390 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+import os, re, datetime, string
+from collections import namedtuple
+from string import Template
+from batchmp.fstools.dirtools import DHandler
+from batchmp.fstools.fsutils import FSH
+from batchmp.fstools.builders.fsentry import FSEntry, FSEntryType, FSEntryDefaults
+from batchmp.fstools.builders.fsprms import FSEntryParamsBase
+from batchmp.commons.utils import MiscHelpers
+from batchmp.tags.handlers.ffmphandler import FFmpegTagHandler
+from batchmp.tags.handlers.mtghandler import MutagenTagHandler
+from batchmp.tags.output.formatters import TagOutputFormatter
+
+
+# DirEntry Counters helper
+class DirCounters:
+    def __init__(self, dirs_cnt = 0, files_cnt = 0, num_files = 0, num_dirs = 0):
+        self.dirs_cnt = dirs_cnt
+        self.files_cnt = files_cnt
+        self.num_files = num_files
+        self.num_dirs = num_dirs
+
+    @staticmethod
+    def num_digits(number, min_digits):
+        return max(MiscHelpers.int_num_digits(number), min_digits)
+
+class Renamer:
+    ''' Renames FS entries
+    '''
+    @classmethod
+    def add_index(cls, fs_entry_params, 
+                        as_prefix = False, join_str = '_',
+                        start_from = 1, min_digits = 1,
+                        sequential = False, by_directory = False):
+        ''' adds indexing
+            automatically figures out right number of min_digits
+        '''
+        try:
+            start_from = abs(int(start_from))
+        except ValueError:
+            start_from = 1
+
+        counters = {fs_entry_params.src_dir : DirCounters(start_from, start_from, 0, 0)}
+        total_files = total_dirs = 0      
+
+        if (sequential or by_directory):                
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+            cnt_key = fs_entry_params.src_dir # one key to rule them all here
+
+            def index_sequential(entry):
+                nonlocal counters
+                addition = None
+
+                if entry.type == FSEntryType.DIR:        
+                    if fs_entry_params.include_dirs:
+                        addition = str(counters[cnt_key].dirs_cnt).zfill(DirCounters.num_digits(total_dirs, min_digits))
+                    # update the dirs counter
+                    if fs_entry_params.include_dirs or by_directory:
+                        counters[cnt_key].dirs_cnt += 1
+
+                elif entry.type == FSEntryType.FILE:
+                    if by_directory:
+                        # indexing via adding respective directory counter
+                        fcnt = counters[cnt_key].dirs_cnt - 1
+                        # do nothing for root files
+                        if fcnt >= 0:
+                            addition = str(fcnt).zfill(Counters.num_digits(total_files, min_digits))
+                    else:
+                        addition = str(counters[cnt_key].files_cnt).zfill(DirCounters.num_digits(total_files, min_digits))
+                        # need to update the files counter
+                        counters[cnt_key].files_cnt += 1
+
+                return addition
+        else:
+            # multilevel indexing
+            def index_multilevel(entry):
+                nonlocal counters, total_files, total_dirs
+                addition = None
+
+                if entry.scopeSwitchingEntry:
+                    counters[entry.realpath] = DirCounters(start_from, start_from, 
+                                                    len(fs_entry_params.fnames), len(fs_entry_params.dnames.passed))
+
+                cnt = counters[os.path.dirname(entry.realpath)]
+                if entry.type == FSEntryType.DIR and fs_entry_params.include_dirs:
+                    addition = str(cnt.dirs_cnt).zfill(DirCounters.num_digits(cnt.num_dirs, min_digits))
+                    cnt.dirs_cnt += 1
+                    total_dirs += 1
+
+                elif entry.type == FSEntryType.FILE:
+                    addition = str(cnt.files_cnt).zfill(DirCounters.num_digits(cnt.num_files, min_digits))
+                    cnt.files_cnt += 1
+                    total_files += 1
+
+                return addition
+
+        # set the index function
+        index_function = index_sequential if (sequential or by_directory) else index_multilevel
+        def add_index_transform(entry):
+            addition = None
+
+            # src dir
+            if entry.type == FSEntryType.ROOT:
+                pass
+            # dirs
+            elif entry.type == FSEntryType.DIR:
+                addition = index_function(entry)
+            # files
+            elif entry.type == FSEntryType.FILE:
+                if fs_entry_params.include_files:
+                    addition = index_function(entry)
+
+            if addition is None:
+                return entry.basename
+            if as_prefix:
+                return join_str.join((addition, entry.basename))
+            else:
+                name_base, name_ext = os.path.splitext(entry.basename)
+                return '{0}{1}{2}{3}'.format(name_base, join_str, addition, name_ext)
+
+        # visualise changes and proceed if confirmed
+        if fs_entry_params.quiet:
+            proceed = True
+        else: 
+            proceed, _, _ = DHandler.visualise_changes(fs_entry_params, formatter = add_index_transform)
+        
+        if proceed:
+            counters = {fs_entry_params.src_dir : DirCounters(start_from, start_from, 0, 0)}
+            if (total_dirs + total_files) == 0:
+                total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = add_index_transform)
+
+    @classmethod
+    def capitalize(cls, fs_entry_params):
+        ''' capitalizes names of FS entries
+        '''
+
+        def capitalize_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return entry.basename
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return entry.basename
+            return string.capwords(entry.basename)
+
+        # visualise changes and proceed if confirmed
+        if fs_entry_params.quiet:
+            proceed = True
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+        else: 
+            proceed, total_files, total_dirs = DHandler.visualise_changes(fs_entry_params, formatter = capitalize_transform)
+
+        if proceed:
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = capitalize_transform, check_unique = False)
+
+
+    @classmethod
+    def add_date(cls, fs_entry_params, as_prefix = False, join_str = '_', format = '%Y-%m-%d'):
+        ''' adds current date
+        '''
+        addition = datetime.datetime.now().strftime(format)
+        join_str = str(join_str)
+
+        def add_date_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return entry.basename
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return entry.basename
+
+            if as_prefix:
+                return join_str.join((addition, entry.basename))
+            else:
+                name_base, name_ext = os.path.splitext(entry.basename)
+                return '{0}{1}{2}{3}'.format(name_base, join_str, addition, name_ext)
+
+        # visualise changes and proceed if confirmed
+        if fs_entry_params.quiet:
+            proceed = True
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+        else: 
+            proceed, total_files, total_dirs = DHandler.visualise_changes(fs_entry_params, formatter = add_date_transform)
+
+        if proceed:
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = add_date_transform)
+
+    @classmethod
+    def add_text(cls, fs_entry_params, text, as_prefix = False, join_str = ' '):
+        ''' adds text
+        '''
+        addition = text
+        join_str = str(join_str)
+
+        def add_text_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return entry.basename
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return entry.basename
+
+            if as_prefix:
+                return join_str.join((addition, entry.basename))
+            else:
+                name_base, name_ext = os.path.splitext(entry.basename)
+                return '{0}{1}{2}{3}'.format(name_base, join_str, addition, name_ext)
+
+        if fs_entry_params.quiet:
+            proceed = True
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+        else: 
+            proceed, total_files, total_dirs = DHandler.visualise_changes(fs_entry_params, formatter = add_text_transform)
+
+        if proceed:
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = add_text_transform)
+
+    @classmethod
+    def remove_n_characters(cls, fs_entry_params, num_chars = 0, from_head = True):
+        ''' removes n first characters
+        '''
+        num_chars = abs(num_chars)
+
+        def remove_n_chars_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return entry.basename
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return entry.basename
+
+            name_base, name_ext = os.path.splitext(entry.basename)
+            if from_head:
+                name_base = name_base[num_chars:]
+            else:
+                name_base = name_base[:-num_chars]
+            return ''.join((name_base, name_ext))
+
+        # visualise changes and proceed if confirmed
+        if fs_entry_params.quiet:
+            proceed = True
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+        else: 
+            proceed, total_files, total_dirs = DHandler.visualise_changes(fs_entry_params, formatter = remove_n_chars_transform)
+
+        if proceed:
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = remove_n_chars_transform)
+
+    @classmethod
+    def replace(cls, fs_entry_params, find_str, replace_str, case_insensitive=False, include_extension = False):
+        ''' Regexp-base replace
+        '''
+        flags = re.UNICODE
+        if case_insensitive:
+            flags = flags | re.IGNORECASE
+        p = re.compile(find_str, flags)
+
+        def replace_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return entry.basename
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return entry.basename
+
+            name_base, name_ext = os.path.splitext(entry.basename)
+            match = p.search(entry.basename if include_extension else name_base)
+            if match:
+                if replace_str is not None:
+                    # expand templates
+                    replace_str_expanded = cls._expand_templates(entry, replace_str)
+                    res = p.sub(replace_str_expanded, entry.basename if include_extension else name_base)
+                else:
+                    res = match.group()
+                return '{0}{1}'.format(res, '' if include_extension else name_ext)
+            else:
+                return entry.basename
+
+        # visualise changes and proceed if confirmed
+        if fs_entry_params.quiet:
+            proceed = True
+            total_files, total_dirs, _ = DHandler.dir_stats(fs_entry_params)
+        else: 
+            proceed, total_files, total_dirs = DHandler.visualise_changes(fs_entry_params, formatter = replace_transform)
+
+        if proceed:
+            DHandler.rename_entries(fs_entry_params, total_files + total_dirs, formatter = replace_transform)
+
+    @classmethod
+    def delete(cls, fs_entry_params):
+
+        ''' Deletes selected files
+            Support detection of non-media files
+        '''        
+
+        if fs_entry_params.filter_dirs & fs_entry_params.include_dirs:
+            fs_entry_params.filter_files = False
+            fs_entry_params.include_files = True
+
+
+        def delete_transform(entry):
+            if entry.type == FSEntryType.ROOT:
+                return entry.basename
+            if entry.type == FSEntryType.DIR and not fs_entry_params.include_dirs:
+                return None
+            if entry.type == FSEntryType.FILE and not fs_entry_params.include_files:
+                return None
+
+            # these are to be gone soon...
+            return entry.basename
+
+        if fs_entry_params.quiet:
+            proceed = True
+        else: 
+            proceed, _, _ = DHandler.visualise_changes(fs_entry_params, 
+                                    formatter = delete_transform, 
+                                    after_msg = 'The following files / folders will be deleted')
+
+        if proceed:
+            DHandler.remove_entries(fs_entry_params, formatter = delete_transform)
+
+
+    @classmethod
+    def organize(cls, fs_entry_params):
+
+        ''' Organizes files by selected attributes
+            Support detection of non-media files
+        '''        
+        print('to be organized')
+
+
+
+    @classmethod
+    def _expand_templates(cls, entry, value):
+        ''' expands template values
+        '''
+        template = Template(value)
+        return template.safe_substitute(cls._substitute_dictionary(entry))
+
+    @classmethod
+    def _substitute_dictionary(cls, entry):
+        ''' internal template value substitution
+        '''
+        sd = {}
+        full_dir_name = os.path.dirname(entry.realpath)
+        sd['dirname'] = os.path.basename(full_dir_name)
+        sd['pardirname'] = os.path.basename(os.path.dirname(full_dir_name))
+
+        sd['adtime'] = datetime.datetime.fromtimestamp(os.path.getatime(entry.realpath))
+        sd['cdtime'] = datetime.datetime.fromtimestamp(os.path.getctime(entry.realpath))
+        sd['mdtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(entry.realpath))
+
+        sd['atime'] = datetime.datetime.fromtimestamp(os.path.getatime(entry.realpath)).time()
+        sd['ctime'] = datetime.datetime.fromtimestamp(os.path.getctime(entry.realpath)).time()
+        sd['mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(entry.realpath)).time()
+
+        sd['adate'] = datetime.datetime.fromtimestamp(os.path.getatime(entry.realpath)).date()
+        sd['cdate'] = datetime.datetime.fromtimestamp(os.path.getctime(entry.realpath)).date()
+        sd['mdate'] = datetime.datetime.fromtimestamp(os.path.getmtime(entry.realpath)).date()
+
+        # for media files, update with the base tags values
+        sd.update(cls._get_tags(entry))        
+
+        return sd
+
+    @classmethod
+    def _get_tags(cls, entry):
+        ''' media tags values for template value substitution
+        '''
+        tags = {}
+        handler = MutagenTagHandler() + FFmpegTagHandler()
+        if handler.can_handle(entry.realpath):
+            for field in TagOutputFormatter.COMPACT_FIELDS:
+                tags[field] = getattr(handler.tag_holder, field, '')
+        return tags
+