batchmp 1.4__py3-none-any.whl

batchmp/commons/progressbar.py

@@ -0,0 +1,154 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+''' A simple single-line console progress bar
+    Displays progress by fractions of 10%
+    Supports premature stops & info messages during execution
+'''
+import sys, time
+from bisect import bisect as bs
+import threading, queue
+from contextlib import contextmanager
+
+
+class CmdProgressBarRefreshRate:
+    SLOW = 0.2
+    MODERATE = 0.1
+    FAST = 0.02
+
+
+@contextmanager
+def progress_bar(starts_from = 0, refresh_rate = CmdProgressBarRefreshRate.MODERATE):
+    ''' Enables usage via a runtime context
+    '''
+    p_bar = CmdProgressBar(starts_from, refresh_rate)
+    p_bar.start()
+    try:
+        yield p_bar
+    finally:
+        p_bar.stop()
+
+
+class CmdProgressBarUpdateTypes:
+    UPDATE_PROGRESS = 0
+    UPDATE_MSG = 1
+
+
+class CmdProgressBar(object):
+    def __init__(self, start_from=0, refresh_rate = CmdProgressBarRefreshRate.MODERATE):
+        self._queue = queue.Queue(1)  # used to communicate with the worker thread
+        self._end_event = threading.Event()  # used to exit
+
+        self.progress = start_from # access via property, for validation $ enqueuing
+        self._info_msg = None
+
+        self._bar_thread = threading.Thread(target=self._show_progress,
+                                            args=(start_from, refresh_rate, self._end_event, self._queue,))
+        self._bar_thread.daemon = True
+
+    @property
+    def info_msg(self):
+        return self._info_msg
+    @info_msg.setter
+    def info_msg(self, value):
+        self._queue.put((CmdProgressBarUpdateTypes.UPDATE_MSG, value))
+        self._info_msg = value
+
+    @property
+    def progress(self):
+        return self._progress
+    @progress.setter
+    def progress(self, value):
+        # unless in stopping mode, set & enqueue
+        if not self._end_event.is_set():
+            if value < 0: value = 0
+            if value > 100: value = 100
+            self._queue.put((CmdProgressBarUpdateTypes.UPDATE_PROGRESS, value))
+            self._progress = value
+
+    # the worker thread method
+    @staticmethod
+    def _show_progress(last_known_progress, refresh_rate, end_event, queue):
+        progress_values = [i for i in range(0, 110, 10)]  # [0, 10, ..., 100]
+        chars = '|/-\\'
+        msg = None
+        while True:
+            if not queue.full():
+                # nothing in the queue yet, keep showing the last known progres
+                progress = last_known_progress
+            else:
+                update = queue.get()
+                # figure out what kind of update is being requested
+                if update[0] == CmdProgressBarUpdateTypes.UPDATE_PROGRESS:
+                    progress = update[1]
+                    last_known_progress = progress
+                else:
+                    msg = update[1]
+                # signal that the value has been consumed
+                queue.task_done()
+
+            num_progress_vals = bs(progress_values, progress)
+            progress_info = '..'.join([''.join((str(i), '%')) for i in progress_values[:num_progress_vals]])
+            progress_info = ''.join((progress_info, '.' * (53 - len(progress_info))))
+
+            # for info msg updates, display the message
+            if msg != None:
+                sys.stdout.write(''.join(('\r', ' ' * 70, '\r')))
+                sys.stdout.write(''.join((msg, '\n')))
+                msg = None
+
+            # show pogress
+            for c in chars:
+                sys.stdout.write('\r[ {0} ..{1}.. ]'.format(c, progress_info))
+                sys.stdout.flush()
+                time.sleep(refresh_rate)
+
+            if end_event.is_set():
+                break
+
+    # starts the worker thread
+    def start(self):
+        self._bar_thread.start()
+
+    # stops the worker thread
+    # handles premature exits (e.g., when the method is called at 70% progress)
+    def stop(self):
+        # check if a graceful stop is needed
+        if self._progress < 100:
+            # looks like a premature exit,
+            # set progress to max
+            self.progress = 100
+
+        # wait till the queue is processed
+        self._queue.join()
+
+        # OK to stop now
+        self._end_event.set()
+        self._bar_thread.join()
+        sys.stdout.write(''.join(('\r', ' ' * 70, '\r')))
+        sys.stdout.flush()
+
+# Quick Dev Test
+if __name__ == '__main__':
+    start_from, msg_target, progress_target = 30, 40, 60
+    with progress_bar(start_from) as p_bar:
+        while True:
+            if p_bar.progress == msg_target:
+                p_bar.info_msg = 'At {}%, and doing well'.format(msg_target)
+            if p_bar.progress == progress_target:
+                p_bar.info_msg = 'At {}%, and feel like finishing early'.format(progress_target)
+                break
+            p_bar.progress += 10
+    print('All Done')
+

batchmp/commons/taskprocessor.py

@@ -0,0 +1,149 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+import copyreg, types, multiprocessing
+from abc import ABCMeta, abstractmethod
+from batchmp.commons.progressbar import progress_bar, CmdProgressBarRefreshRate
+from batchmp.commons.utils import timed, MiscHelpers
+
+
+class Task(metaclass = ABCMeta):
+    ''' Abstract TasksProcessor task
+    '''
+    @abstractmethod
+    def execute(self):
+        return TaskResult()
+
+
+class TaskResult:
+    ''' TasksProcessor Task result
+    '''
+    def __init__(self):
+        self._task_steps_info_msgs = []
+        self._task_steps_durations = []
+        self._succeeded = False
+
+    def add_task_step_duration(self, step_duration):
+        self._task_steps_durations.append(step_duration)
+
+    def add_task_step_info_msg(self, step_info_msg):
+        self._task_steps_info_msgs.append(step_info_msg)
+
+    def add_report_msg(self, processed_fpath):
+        task_duration_str = MiscHelpers.time_delta_str(self.task_duration)
+        self.add_task_step_info_msg('Done processing\n {0}\n in {1}'.format(
+                                                        processed_fpath, task_duration_str))
+    @property
+    def succeeded(self):
+         return self._succeeded
+    @succeeded.setter
+    def succeeded(self, value):
+        self._succeeded = value
+
+    @property
+    def task_output(self):
+        task_output = None
+        for info_msg in self._task_steps_info_msgs:
+            task_output = '{0}{1}{2}'.format(task_output if task_output else '',
+                                                '\n' if task_output else '',
+                                                info_msg)
+        return task_output
+
+    @property
+    def task_duration(self):
+        task_duration = 0.0
+        for step_duration in self._task_steps_durations:
+            task_duration += step_duration
+        return task_duration
+
+
+class TasksProcessor:
+    ''' Runs cmd-line Tasks, sequentially or in a pool of processes
+        Displays progress / tasks done
+    '''
+    def _process_task(self, task):
+        result = task.execute()
+        return result
+
+    @timed
+    def process_tasks(self, tasks_queue, serial_exec = False, num_workers = None, quiet = False):
+        tasks_results = []
+        cpu_core_time = 0.0
+
+        num_tasks = len(tasks_queue)
+        serial_exec = serial_exec or num_tasks == 1
+        if num_tasks > 0:
+            # Pre-processing msgs
+            if serial_exec:
+                print('Processing {0} {1}'.format(num_tasks,
+                                                        'task' if num_tasks == 1 else 'tasks sequentially'))
+            else:
+                if not num_workers:
+                    num_workers = multiprocessing.cpu_count()
+                print('Processing {0} tasks with pool of {1} worker processes'.format(num_tasks, num_workers))
+
+            # start showing progress
+            with progress_bar(refresh_rate = CmdProgressBarRefreshRate.MODERATE) as p_bar:
+                def _make_progress(result):
+                    nonlocal cpu_core_time
+                    tasks_results.append(result)
+                    if not quiet:
+                        p_bar.info_msg = result.task_output
+                    cpu_core_time += result.task_duration
+                    p_bar.progress = len(tasks_results) / num_tasks * 100
+
+                if serial_exec:
+                    # just loop through the queue of tasks
+                    for task in tasks_queue:
+                        result = self._process_task(task)
+                        _make_progress(result)
+                else:
+                    # init the pool and kick it off
+                    with multiprocessing.Pool(num_workers) as pool:
+                        for result in pool.imap_unordered(self._process_task, tasks_queue):
+                            _make_progress(result)
+
+        # return tasks results, aggregate CPU cores time, and total time elapsed (via @timed)
+        return tasks_results, cpu_core_time
+
+
+"""
+    Python multiprocessing pickles stuff, and bound methods are not picklable
+    The code below serves as a workaround, more at:
+    http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods#edit2155350
+"""
+def _pickle_method(method):
+    func_name = method.im_func.__name__
+    obj = method.im_self
+    cls = method.im_class
+
+    if func_name.startswith('__') and not func_name.endswith('__'):
+        cls_name = cls.__name__.lstrip('_')
+        if cls_name:
+            func_name = '_' + cls_name + func_name
+
+    return _unpickle_method, (func_name, obj, cls)
+
+def _unpickle_method(func_name, obj, cls):
+    for cls in cls.mro():
+        try:
+            func = cls.__dict__[func_name]
+        except KeyError:
+            pass
+        else:
+            break
+    return func.__get__(obj, cls)
+
+copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
+

batchmp/commons/utils.py

@@ -0,0 +1,194 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import subprocess, shlex, time, tempfile, shutil
+import datetime, math, functools
+from functools import wraps
+from urllib.parse import urlparse
+import urllib.request, urllib.error
+from contextlib import contextmanager
+
+''' General-level utilities
+'''
+
+@contextmanager
+def temp_dir():
+    ''' Temp dir context manager
+    '''
+    tmp_dir = tempfile.mkdtemp()
+    try:
+        yield tmp_dir
+    finally:
+        # remove tmp dir
+        shutil.rmtree(tmp_dir)
+
+
+def timed(f):
+    """ A timing decorator
+    """
+    @wraps(f)
+    def wrapper(*args, **kwds):
+        start = time.time()
+        result = f(*args, **kwds)
+        elapsed = time.time() - start
+        return (result, elapsed)
+    return wrapper
+
+
+class CmdProcessingError(Exception):
+    pass
+
+@timed
+def run_cmd(cmd, shell = False):
+    ''' Runs shell commands in a separate process
+    '''
+    if not shell:
+        cmd = shlex.split(cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell = shell)
+    output = proc.communicate()[0].decode('utf-8')
+    if proc.returncode != 0:
+        raise CmdProcessingError(output)
+    return output
+
+
+class MiscHelpers:
+    @staticmethod
+    def int_num_digits(num):
+        ''' Number of digits in an int number
+        '''
+        if not isinstance(num, int):
+            try:
+                num = int(num)
+            except ValueError as e:
+                return 0
+        num_digits = 1
+        while (int(abs(num)/(10**num_digits)) > 0):
+            num_digits += 1
+        return num_digits
+
+    @staticmethod
+    def time_delta(td_str):
+        ''' Timedelta from the "hh:mm:ss[.xxx]" format
+        '''
+        hrs = mins = secs = None
+        td = td_str.split(':')
+        time_parts = range(len(td))
+        for _ in time_parts:
+            if secs is None:
+                secs = float(td.pop(-1))
+            elif mins is None:
+                mins = int(td.pop(-1))
+            elif hrs is None:
+                hrs = int(td.pop(-1))
+            else:
+                break
+        return  datetime.timedelta(hours = hrs if hrs else 0,
+                                      minutes = mins if mins else 0,
+                                      seconds = secs if secs else 0)
+
+    @staticmethod
+    def time_delta_str(secs, num_miliseconds = 2):
+        ''' Timedelta string with specified number of miliseconds
+        '''
+        div = 10**num_miliseconds if num_miliseconds > 0 else 1
+        td_str = str(datetime.timedelta(seconds = math.ceil(secs*div)/div)).rstrip('0')
+        if td_str.endswith(':'):
+            td_str = '{}00'.format(td_str)
+        return td_str
+
+    @staticmethod
+    def percentile(values, percent):
+        '''
+        Find the percentile for a list of values
+        '''
+        if not values:
+            return None
+        else:
+            values.sort()
+
+        dv = (len(values) - 1) * percent / 100
+        floor = math.floor(dv)
+
+        ceil = math.ceil(dv)
+        if floor == ceil:
+            return values[int(dv)]
+
+        dc = values[int(floor)] * (ceil - dv)
+        dt = values[int(ceil)] * (dv - floor)
+
+        return dc + dt
+
+    @staticmethod
+    def median(l):
+        # the 50th percentile
+        return functools.partial(MiscHelpers.percentile, percent=50)(l)
+
+
+class ImageLoader:
+    @staticmethod
+    def load_image_from_url(url):
+        ''' Loads image from an URL
+        '''
+        img = None
+        try:
+            responce = urllib.request.urlopen(url, timeout = 5)
+        except urllib.error.URLError as e:
+            print('A problem while retrieving image: "{}"'.format(e))
+        else:
+            content_type = dict(responce.getheaders())['Content-Type']
+            if content_type.split('/')[0] != 'image':
+                print('URL did not seem to return a valid image')
+                print('Received Content-Type: {}'.format(content_type))
+            else:
+                img = responce.read()
+
+        return img
+
+    @staticmethod
+    def load_image_from_file(fpath):
+        ''' Loads an image from disk via file path
+        '''
+        img = None
+        if fpath:
+            with open(fpath, 'rb') as f:
+                img = f.read()
+
+        return img
+
+    @staticmethod
+    def load_image(path_or_url):
+        ''' Loads an image from an URL or a file path
+        '''
+        url_parts = urlparse(path_or_url)
+        if url_parts.scheme in (None, '') and url_parts.netloc in (None, ''):
+            return ImageLoader.load_image_from_file(path_or_url)
+
+        if url_parts.scheme == 'file':
+            fpath = format(url_parts.path)
+            if url_parts.netloc == '~':
+                fpath = '~{}'.format(fpath)
+            return ImageLoader.load_image_from_file(path_or_url)
+
+        return ImageLoader.load_image_from_url(path_or_url)
+
+
+# Quick dev test
+if __name__ == '__main__':
+    td = MiscHelpers.time_delta('00:24.5764654645464')
+    print(td)
+    print(MiscHelpers.time_delta_str(td.total_seconds(), num_miliseconds = -1))
+
+    a = [2.08, 2.11, 2.18, 2.18, 2.2, 2.2, 2.21, 2.21, 2.3, 2.8, 2.97, 3.21, 3.4, 3.71, 3.8, 3.9, 4.21, 4.3, 4.4]
+    print(MiscHelpers.median(a))
+
+    print(MiscHelpers.percentile(a, 25))

batchmp/ffmptools/ffcommands/cmdopt.py

@@ -0,0 +1,115 @@
+# coding=utf8
+## Copyright (c) 2014 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from enum import IntEnum
+
+
+class FFmpegCommands:
+    ''' Common FFmpeg commands / options
+    '''
+    MAP_ALL_STREAMS = ' -map 0'
+    COPY_CODECS = ' -c copy'
+
+    # queue size
+    MUXING_QUEUE_SIZE = ' -max_muxing_queue_size 1024'
+
+    # excluding streams
+    DISABLE_VIDEO = ' -vn'
+    DISABLE_AUDIO = ' -an'
+    DISABLE_SUBTITLES = ' -sn'
+
+    # Conversion options
+    CONVERT_COPY_VBR_QUALITY = ' -q:v 0 -q:a 0'
+    CONVERT_LOSSLESS = ' CONVERT_LOSSLESS_IF_POSSIBLE'
+    CONVERT_LOSSLESS_ALAC = ' -q:v 0 -acodec alac'
+    CONVERT_LOSSLESS_FLAC = ' -q:v 0 -acodec flac'
+    CONVERT_CHANGE_CONTAINER = ' -c copy -copyts'
+
+    # Log level
+    LOG_LEVEL_ERROR = ' -v error'
+    LOG_LEVEL_QUIET = ' -v quiet'
+
+    # Segment
+    SEGMENT = ' -f segment'
+    SEGMENT_TIME = ' -segment_time'
+    SEGMENT_TIMES = ' -segment_times'
+    SEGMENT_RESET_TIMESTAMPS = ' -reset_timestamps 1'
+
+    @staticmethod
+    def exclude_input_stream(stream_idx):
+        return ' -map -0:{}'.format(stream_idx)
+
+    @staticmethod
+    def include_input_stream(stream_idx):
+        return ' -map 0:{}'.format(stream_idx)
+
+
+class FFmpegBitMaskOptions(IntEnum):
+    ''' FFmpeg commands / options bitmasks
+    '''
+    MAP_ALL_STREAMS = (1<<0)
+    COPY_CODECS = (1<<1)
+
+    MUXING_QUEUE_SIZE = (1<<4)
+
+    DISABLE_VIDEO = (1<<5)
+    DISABLE_AUDIO = (1<<6)
+    DISABLE_SUBTITLES = (1<<7)
+
+    @classmethod
+    def ff_general_options(cls, ff_gbm_options):
+        options_str = ''
+        if ff_gbm_options:
+            for bm_option in cls:
+                if bm_option & ff_gbm_options == bm_option:
+                    options_str = ''.join((options_str, cls._option_str_value(bm_option)))
+        return options_str
+
+    @staticmethod
+    def _option_str_value(bm_option):
+        if bm_option == FFmpegBitMaskOptions.MAP_ALL_STREAMS:
+            return FFmpegCommands.MAP_ALL_STREAMS
+        elif bm_option == FFmpegBitMaskOptions.COPY_CODECS:
+            return FFmpegCommands.COPY_CODECS
+
+        elif bm_option == FFmpegBitMaskOptions.DISABLE_VIDEO:
+            return FFmpegCommands.DISABLE_VIDEO
+        elif bm_option == FFmpegBitMaskOptions.DISABLE_AUDIO:
+            return FFmpegCommands.DISABLE_AUDIO
+        elif bm_option == FFmpegBitMaskOptions.DISABLE_SUBTITLES:
+            return FFmpegCommands.DISABLE_SUBTITLES
+        elif bm_option == FFmpegBitMaskOptions.MUXING_QUEUE_SIZE:
+            return FFmpegCommands.MUXING_QUEUE_SIZE
+
+        else:
+            return ''
+
+# Quick dev test
+if __name__ == '__main__':
+    copy_codecs = True
+    options = 0
+    options |= FFmpegBitMaskOptions.DISABLE_SUBTITLES
+    if copy_codecs:
+        options |= FFmpegBitMaskOptions.COPY_CODECS
+    options |= FFmpegBitMaskOptions.DISABLE_VIDEO
+    options |= FFmpegBitMaskOptions.MUXING_QUEUE_SIZE
+    print(FFmpegBitMaskOptions.ff_general_options(options))
+
+    exclude_stream = 1
+    print('exclude video stream {}: {}'.format(exclude_stream, FFmpegCommands.exclude_input_stream(exclude_stream)))
+
+
+
+
+