shinestacker 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

shinestacker/algorithms/stack_framework.py

@@ -1,30 +1,49 @@
-# pylint: disable=C0114, C0115, C0116, W0102, R0902, R0903
+# pylint: disable=C0114, C0115, C0116, W0102, R0902, R0903, E1128
 # pylint: disable=R0917, R0913, R1702, R0912, E1111, E1121, W0613
 import logging
 import os
 from .. config.constants import constants
 from .. core.colors import color_str
-from .. core.framework import Job, ActionList
+from .. core.framework import Job, SequentialTask
 from .. core.core_utils import check_path_exists
-from .. core.exceptions import ShapeError, BitDepthError, RunStopException
-from .utils import read_img, write_img, extension_tif_jpg
+from .. core.exceptions import RunStopException
+from .utils import read_img, write_img, extension_tif_jpg, get_img_metadata, validate_image
 
 
 class StackJob(Job):
-    def __init__(self, name, working_path, input_path='', **kwargs):
+    def __init__(self, name, working_path, input_path='', input_filepaths=[], **kwargs):
         check_path_exists(working_path)
         self.working_path = working_path
-        if input_path == '':
-            self.paths = []
-        else:
-            self.paths = [input_path]
+        self._input_path = input_path
+        self._action_paths = [] if input_path == '' else [input_path]
+        self._input_filepaths = []
+        self._input_full_path = None
+        self._input_filepaths = input_filepaths
         Job.__init__(self, name, **kwargs)
 
     def init(self, a):
         a.init(self)
 
+    def input_filepaths(self):
+        return self._input_filepaths
+
+    def num_input_filepaths(self):
+        return len(self._input_filepaths)
+
+    def action_paths(self):
+        return self._action_paths
+
+    def add_action_path(self, path):
+        self._action_paths.append(path)
+
+    def num_action_paths(self):
+        return len(self._action_paths)
+
+    def action_path(self, i):
+        return self._action_paths[i]
+
 
-class FramePaths:
+class ImageSequenceManager:
     def __init__(self, name, input_path='', output_path='', working_path='',
                  plot_path=constants.DEFAULT_PLOTS_PATH,
                  scratch_output_dir=True, resample=1,
@@ -33,26 +52,73 @@ class FramePaths:
         self.working_path = working_path
         self.plot_path = plot_path
         self.input_path = input_path
-        self.output_path = output_path
-        self.output_dir = None
-        self.resample = resample
+        self.output_path = self.name if output_path == '' else output_path
+        self._resample = resample
         self.reverse_order = reverse_order
         self.scratch_output_dir = scratch_output_dir
-        self.input_full_path = None
         self.enabled = None
-        self.filenames = None
         self.base_message = ''
-
-    def folder_filelist(self):
-        assert False, "this method should be overwritten"
+        self._input_full_path = None
+        self._output_full_path = None
+        self._input_filepaths = None
+
+    def output_full_path(self):
+        if self._output_full_path is None:
+            self._output_full_path = os.path.join(self.working_path, self.output_path)
+        return self._output_full_path
+
+    def input_full_path(self):
+        if self._input_full_path is None:
+            if isinstance(self.input_path, str):
+                self._input_full_path = os.path.join(self.working_path, self.input_path)
+                check_path_exists(self._input_full_path)
+            elif hasattr(self.input_path, "__len__"):
+                self._input_full_path = [os.path.join(self.working_path, path)
+                                         for path in self.input_path]
+                for path in self._input_full_path:
+                    check_path_exists(path)
+        return self._input_full_path
+
+    def input_filepaths(self):
+        if self._input_filepaths is None:
+            if isinstance(self.input_full_path(), str):
+                dirs = [self.input_full_path()]
+            elif hasattr(self.input_full_path(), "__len__"):
+                dirs = self.input_full_path()
+            else:
+                raise RuntimeError("input_full_path option must contain "
+                                   "a path or an array of paths")
+            files = []
+            for d in dirs:
+                filelist = []
+                for _dirpath, _, filenames in os.walk(d):
+                    filelist = [os.path.join(_dirpath, name)
+                                for name in filenames if extension_tif_jpg(name)]
+                    filelist.sort()
+                    if self.reverse_order:
+                        filelist.reverse()
+                    if self._resample > 1:
+                        filelist = filelist[0::self._resample]
+                    files += filelist
+                if len(files) == 0:
+                    self.print_message(color_str(f"input folder {d} does not contain any image",
+                                                 constants.LOG_COLOR_WARNING),
+                                       level=logging.WARNING)
+            self._input_filepaths = files
+        return self._input_filepaths
+
+    def input_filepath(self, index):
+        return self.input_filepaths()[index]
+
+    def num_input_filepaths(self):
+        return len(self.input_filepaths())
 
     def print_message(self, msg='', level=logging.INFO, end=None, begin='', tqdm=False):
         assert False, "this method should be overwritten"
 
     def set_filelist(self):
-        self.filenames = self.folder_filelist()
-        file_folder = self.input_full_path.replace(self.working_path, '').lstrip('/')
-        self.print_message(color_str(f"{len(self.filenames)} files in folder: {file_folder}",
+        file_folder = self.input_full_path().replace(self.working_path, '').lstrip('/')
+        self.print_message(color_str(f"{self.num_input_filepaths()} files in folder: {file_folder}",
                                      constants.LOG_COLOR_LEVEL_2))
         self.base_message = color_str(self.name, constants.LOG_COLOR_LEVEL_1, "bold")
 
@@ -60,20 +126,16 @@ class FramePaths:
         if self.working_path == '':
             self.working_path = job.working_path
         check_path_exists(self.working_path)
-        if self.output_path == '':
-            self.output_path = self.name
-        self.output_dir = self.working_path + \
-            ('' if self.working_path[-1] == '/' else '/') + \
-            self.output_path
-        if not os.path.exists(self.output_dir):
-            os.makedirs(self.output_dir)
+        output_dir = self.output_full_path()
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
         else:
-            list_dir = os.listdir(self.output_dir)
+            list_dir = os.listdir(output_dir)
             if len(list_dir) > 0:
                 if self.scratch_output_dir:
                     if self.enabled:
                         for filename in list_dir:
-                            file_path = os.path.join(self.output_dir, filename)
+                            file_path = os.path.join(output_dir, filename)
                             if os.path.isfile(file_path):
                                 os.remove(file_path)
                         self.print_message(
@@ -95,141 +157,91 @@ class FramePaths:
             if not os.path.exists(self.plot_path):
                 os.makedirs(self.plot_path)
         if self.input_path in ['', []]:
-            if len(job.paths) == 0:
+            if job.num_action_paths() == 0:
                 raise RuntimeError(f"Job {job.name} does not have any configured path")
-            self.input_path = job.paths[-1]
-        job.paths.append(self.output_path)
-
-
-class FrameDirectory(FramePaths):
-    def __init__(self, name, **kwargs):
-        FramePaths.__init__(self, name, **kwargs)
+            self.input_path = job.action_path(-1)
+            if job.num_input_filepaths() > 0:
+                self._input_filepaths = []
+                for filepath in job.input_filepaths():
+                    if not os.path.isabs(filepath):
+                        filepath = os.path.join(self.input_full_path(), filepath)
+                    self._input_filepaths.append(filepath)
+        job.add_action_path(self.output_path)
 
     def folder_list_str(self):
-        if isinstance(self.input_full_path, list):
+        if isinstance(self.input_full_path(), list):
             file_list = ", ".join(
-                list(self.input_full_path.replace(self.working_path, '').lstrip('/')))
-            return "folder" + ('s' if len(self.input_full_path) > 1 else '') + f": {file_list}"
-        return "folder: " + self.input_full_path.replace(self.working_path, '').lstrip('/')
-
-    def folder_filelist(self):
-        src_contents = os.walk(self.input_full_path)
-        _dirpath, _, filenames = next(src_contents)
-        filelist = [name for name in filenames if extension_tif_jpg(name)]
-        filelist.sort()
-        if self.reverse_order:
-            filelist.reverse()
-        if self.resample > 1:
-            filelist = filelist[0::self.resample]
-        return filelist
-
-    def init(self, job, _working_path=''):
-        FramePaths.init(self, job)
-        self.input_full_path = self.working_path + \
-            ('' if self.working_path[-1] == '/' else '/') + self.input_path
-        check_path_exists(self.input_full_path)
-        job.paths.append(self.output_path)
-
-
-class FrameMultiDirectory(FramePaths):
-    def __init__(self, name, input_path='', output_path='', working_path='',
-                 plot_path=constants.DEFAULT_PLOTS_PATH,
-                 scratch_output_dir=True, resample=1,
-                 reverse_order=constants.DEFAULT_FILE_REVERSE_ORDER, **_kwargs):
-        FramePaths.__init__(self, name, input_path, output_path, working_path, plot_path,
-                            scratch_output_dir, resample, reverse_order)
-        self.input_full_path = None
+                [path.replace(self.working_path, '').lstrip('/')
+                 for path in self.input_full_path()])
+            return "folder" + ('s' if len(self.input_full_path()) > 1 else '') + f": {file_list}"
+        return "folder: " + self.input_full_path().replace(self.working_path, '').lstrip('/')
 
-    def folder_list_str(self):
-        if isinstance(self.input_full_path, list):
-            file_list = ", ".join([d.replace(self.working_path, '').lstrip('/')
-                                   for d in self.input_full_path])
-            return "folder" + ('s' if len(self.input_full_path) > 1 else '') + f": {file_list}"
-        return "folder: " + self.input_full_path.replace(self.working_path, '').lstrip('/')
-
-    def folder_filelist(self):
-        if isinstance(self.input_full_path, str):
-            dirs = [self.input_full_path]
-            paths = [self.input_path]
-        elif hasattr(self.input_full_path, "__len__"):
-            dirs = self.input_full_path
-            paths = self.input_path
-        else:
-            raise RuntimeError("input_full_path option must contain a path or an array of paths")
-        files = []
-        for d, p in zip(dirs, paths):
-            filelist = []
-            for _dirpath, _, filenames in os.walk(d):
-                filelist = [f"{p}/{name}" for name in filenames if extension_tif_jpg(name)]
-                if self.reverse_order:
-                    filelist.reverse()
-                if self.resample > 1:
-                    filelist = filelist[0::self.resample]
-                files += filelist
-            if len(files) == 0:
-                self.print_message(color_str(f"input folder {p} does not contain any image", "red"),
-                                   level=logging.WARNING)
-        return files
 
-    def init(self, job):
-        FramePaths.init(self, job)
-        if isinstance(self.input_path, str):
-            self.input_full_path = self.working_path + \
-                ('' if self.working_path[-1] == '/' else '/') + \
-                self.input_path
-            check_path_exists(self.input_full_path)
-        elif hasattr(self.input_path, "__len__"):
-            self.input_full_path = []
-            for path in self.input_path:
-                self.input_full_path.append(self.working_path +
-                                            ('' if self.working_path[-1] == '/' else '/') +
-                                            path)
-        job.paths.append(self.output_path)
-
-
-class FramesRefActions(ActionList, FrameDirectory):
-    def __init__(self, name, enabled=True, ref_idx=-1, step_process=False, **kwargs):
-        FrameDirectory.__init__(self, name, **kwargs)
-        ActionList.__init__(self, name, enabled)
-        self.ref_idx = ref_idx
+class ReferenceFrameTask(SequentialTask, ImageSequenceManager):
+    def __init__(self, name, enabled=True, reference_index=0, step_process=False, **kwargs):
+        ImageSequenceManager.__init__(self, name, **kwargs)
+        SequentialTask.__init__(self, name, enabled)
+        self.ref_idx = reference_index
         self.step_process = step_process
-        self._idx = None
-        self._ref_idx = None
-        self._idx_step = None
+        self.current_idx = None
+        self.current_ref_idx = None
+        self.current_idx_step = None
 
     def begin(self):
-        ActionList.begin(self)
+        SequentialTask.begin(self)
         self.set_filelist()
-        self.set_counts(len(self.filenames))
-        if self.ref_idx == -1:
-            self.ref_idx = len(self.filenames) // 2
+        n = self.num_input_filepaths()
+        self.set_counts(n)
+        if self.ref_idx == 0:
+            self.ref_idx = n // 2
+        elif self.ref_idx == -1:
+            self.ref_idx = n - 1
+        else:
+            self.ref_idx -= 1
+            if not 0 <= self.ref_idx < n:
+                msg = f"reference index {self.ref_idx} out of range [1, {n}]"
+                self.print_message_r(color_str(msg, constants.LOG_COLOR_LEVEL_2))
+                raise IndexError(msg)
 
     def end(self):
-        ActionList.end(self)
+        SequentialTask.end(self)
 
     def run_frame(self, _idx, _ref_idx):
-        pass
-
-    def run_step(self):
-        if self.count == 0:
-            self._idx = self.ref_idx if self.step_process else 0
-            self._ref_idx = self.ref_idx
-            self._idx_step = +1
-        ll = len(self.filenames)
-        self.print_message_r(
-            color_str(f"step {self.count + 1}/{ll}: process file: {self.filenames[self._idx]}, "
-                      f"reference: {self.filenames[self._ref_idx]}", constants.LOG_COLOR_LEVEL_2))
+        return None
+
+    def run_step(self, action_count=-1):
+        num_files = self.num_input_filepaths()
+        if self.run_sequential():
+            if action_count == 0:
+                self.current_idx = self.ref_idx if self.step_process else 0
+                self.current_ref_idx = self.ref_idx
+                self.current_idx_step = +1
+            idx, ref_idx = self.current_idx, self.current_ref_idx
+            self.print_message_r(
+                color_str(f"step {action_count + 1}/{num_files}: process file: "
+                          f"{os.path.basename(self.input_filepath(idx))}, "
+                          f"reference: "
+                          f"{os.path.basename(self.input_filepath(self.current_ref_idx))}",
+                          constants.LOG_COLOR_LEVEL_2))
+        else:
+            idx, ref_idx = action_count, -1
+            self.print_message_r(
+                color_str(f"step {idx + 1}/{num_files}: process file: "
+                          f"{os.path.basename(self.input_filepath(idx))}, "
+                          "parallel thread", constants.LOG_COLOR_LEVEL_2))
         self.base_message = color_str(self.name, constants.LOG_COLOR_LEVEL_1, "bold")
-        self.run_frame(self._idx, self._ref_idx)
-        if self._idx < ll:
-            if self.step_process:
-                self._ref_idx = self._idx
-            self._idx += self._idx_step
-        if self._idx == ll:
-            self._idx = self.ref_idx - 1
-            if self.step_process:
-                self._ref_idx = self.ref_idx
-            self._idx_step = -1
+        img = self.run_frame(idx, ref_idx)
+        if self.run_sequential():
+            if self.current_idx < num_files:
+                if self.step_process and img is not None:
+                    self.current_ref_idx = self.current_idx
+                self.current_idx += self.current_idx_step
+            if self.current_idx == num_files:
+                self.current_idx = self.ref_idx - 1
+                if self.step_process:
+                    self.current_ref_idx = self.ref_idx
+                self.current_idx_step = -1
+        return img is not None
 
 
 class SubAction:
@@ -242,49 +254,49 @@ class SubAction:
     def end(self):
         pass
 
+    def sequential_processing(self):
+        return False
+
 
-class CombinedActions(FramesRefActions):
+class CombinedActions(ReferenceFrameTask):
     def __init__(self, name, actions=[], enabled=True, **kwargs):
-        FramesRefActions.__init__(self, name, enabled, **kwargs)
+        ReferenceFrameTask.__init__(self, name, enabled, **kwargs)
         self._actions = actions
-        self.dtype = None
-        self.shape = None
+        self._metadata = (None, None)
 
     def begin(self):
-        FramesRefActions.begin(self)
+        ReferenceFrameTask.begin(self)
         for a in self._actions:
             if a.enabled:
                 a.begin(self)
 
     def img_ref(self, idx):
-        filename = self.filenames[idx]
-        img = read_img((self.output_dir
-                        if self.step_process else self.input_full_path) + f"/{filename}")
+        input_path = self.input_filepath(idx)
+        img = read_img(input_path)
         if img is None:
-            raise RuntimeError(f"Invalid file: {self.input_full_path}/{filename}")
-        self.dtype = img.dtype
-        self.shape = img.shape
+            raise RuntimeError(f"Invalid file: {os.path.basename(input_path)}")
+        self._metadata = get_img_metadata(img)
         return img
 
     def run_frame(self, idx, ref_idx):
-        filename = self.filenames[idx]
-        self.sub_message_r(color_str(': read input image', constants.LOG_COLOR_LEVEL_3))
-        img = read_img(f"{self.input_full_path}/{filename}")
-        if self.dtype is not None and img.dtype != self.dtype:
-            raise BitDepthError(self.dtype, img.dtype, )
-        if self.shape is not None and img.shape != self.shape:
-            raise ShapeError(self.shape, img.shape)
+        input_path = self.input_filepath(idx)
+        self.print_message(
+            color_str(f'read input image '
+                      f'{idx + 1}/{self.total_action_counts}, '
+                      f'{os.path.basename(input_path)}', constants.LOG_COLOR_LEVEL_3))
+        img = read_img(input_path)
+        validate_image(img, *(self._metadata))
         if img is None:
-            raise RuntimeError(f"Invalid file: {self.input_full_path}/{filename}")
+            raise RuntimeError(f"Invalid file:  {os.path.basename(input_path)}")
         if len(self._actions) == 0:
             self.sub_message(color_str(": no actions specified", constants.LOG_COLOR_ALERT),
                              level=logging.WARNING)
         for a in self._actions:
             if not a.enabled:
                 self.get_logger().warning(color_str(f"{self.base_message}: sub-action disabled",
-                                                    'red'))
+                                                    constants.LOG_COLOR_ALERT))
             else:
-                if self.callback('check_running', self.id, self.name) is False:
+                if self.callback(constants.CALLBACK_CHECK_RUNNING, self.id, self.name) is False:
                     raise RunStopException(self.name)
                 if img is not None:
                     img = a.run_frame(idx, ref_idx, img)
@@ -293,16 +305,26 @@ class CombinedActions(FramesRefActions):
                         color_str(": null input received, action skipped",
                                   constants.LOG_COLOR_ALERT),
                         level=logging.WARNING)
-        self.sub_message_r(color_str(': write output image', constants.LOG_COLOR_LEVEL_3))
         if img is not None:
-            write_img(self.output_dir + "/" + filename, img)
-        else:
-            self.print_message(color_str(
-                "no output file resulted from processing input file: "
-                f"{self.input_full_path}/{filename}",
-                constants.LOG_COLOR_ALERT), level=logging.WARNING)
+            output_path = os.path.join(self.output_full_path(), os.path.basename(input_path))
+            self.print_message(
+                color_str(f'write output image '
+                          f'{idx + 1}/{self.total_action_counts}, '
+                          f'{os.path.basename(output_path)}', constants.LOG_COLOR_LEVEL_3))
+            write_img(output_path, img)
+            return img
+        self.print_message(color_str(
+            f"no output file resulted from processing input file: {os.path.basename(input_path)}",
+            constants.LOG_COLOR_ALERT), level=logging.WARNING)
+        return None
 
     def end(self):
         for a in self._actions:
             if a.enabled:
                 a.end()
+
+    def sequential_processing(self):
+        for a in self._actions:
+            if a.sequential_processing():
+                return True
+        return False

shinestacker/algorithms/vignetting.py

@@ -134,7 +134,8 @@ class Vignetting(SubAction):
         self.corrections = None
 
     def run_frame(self, idx, _ref_idx, img_0):
-        self.process.sub_message_r(color_str(": compute vignetting", "cyan"))
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: compute vignetting", "cyan"))
         h, w = img_0.shape[:2]
         self.w_2, self.h_2 = w / 2, h / 2
         self.r_max = np.sqrt((w / 2)**2 + (h / 2)**2)
@@ -153,14 +154,15 @@ class Vignetting(SubAction):
             return img_0
         self.v0 = sigmoid_model(0, *params)
         i0_fit, k_fit, r0_fit = params
-        self.process.sub_message(color_str(": vignetting model parameters: ", "cyan") +
-                                 color_str(f"i0={i0_fit / 2:.4f}, "
-                                           f"k={k_fit * self.r_max:.4f}, "
-                                           f"r0={r0_fit / self.r_max:.4f}",
-                                           "light_blue"),
-                                 level=logging.DEBUG)
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: vignetting model parameters: ", "cyan") +
+            color_str(f"i0={i0_fit / 2:.4f}, "
+                      f"k={k_fit * self.r_max:.4f}, "
+                      f"r0={r0_fit / self.r_max:.4f}",
+                      "light_blue"),
+            level=logging.DEBUG)
         if self.plot_correction:
-            plt.figure(figsize=(10, 5))
+            plt.figure(figsize=constants.PLT_FIG_SIZE)
             plt.plot(radii, intensities, label="image mean intensity")
             plt.plot(radii, sigmoid_model(radii * subsample, *params), label="sigmoid fit")
             plt.xlabel('radius (pixels)')
@@ -175,24 +177,25 @@ class Vignetting(SubAction):
             save_plot(plot_path)
             plt.close('all')
             self.process.callback(
-                'save_plot', self.process.id,
+                constants.CALLBACK_SAVE_PLOT, self.process.id,
                 f"{self.process.name}: intensity\nframe {idx_str}", plot_path)
         for i, p in enumerate(self.percentiles):
             self.corrections[i][idx] = fsolve(lambda x: sigmoid_model(x, *params) /
                                               self.v0 - p, r0_fit)[0]
-        self.process.sub_message_r(color_str(": correct vignetting", "cyan"))
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: correct vignetting", "cyan"))
         return correct_vignetting(
             img_0, self.max_correction, self.black_threshold, None, params, self.v0,
             subsample, self.fast_subsampling)
 
     def begin(self, process):
         self.process = process
-        self.corrections = [np.full(self.process.counts, None, dtype=float)
+        self.corrections = [np.full(self.process.total_action_counts, None, dtype=float)
                             for p in self.percentiles]
 
     def end(self):
         if self.plot_summary:
-            plt.figure(figsize=(10, 5))
+            plt.figure(figsize=constants.PLT_FIG_SIZE)
             xs = np.arange(1, len(self.corrections[0]) + 1, dtype=int)
             for i, p in enumerate(self.percentiles):
                 linestyle = 'solid'
@@ -224,5 +227,5 @@ class Vignetting(SubAction):
                         f"{self.process.name}-r0.pdf"
             save_plot(plot_path)
             plt.close('all')
-            self.process.callback('save_plot', self.process.id,
+            self.process.callback(constants.CALLBACK_SAVE_PLOT, self.process.id,
                                   f"{self.process.name}: vignetting", plot_path)

shinestacker/app/main.py

@@ -142,9 +142,12 @@ class MainApp(QMainWindow):
         return app_menu
 
     def quit(self):
-        self.retouch_window.quit()
-        self.project_window.quit()
+        if not self.retouch_window.quit():
+            return False
+        if not self.project_window.quit():
+            return False
         self.close()
+        return True
 
     def switch_app(self, index):
         self.stacked_widget.setCurrentIndex(index)
@@ -192,7 +195,8 @@ class MainApp(QMainWindow):
 class Application(QApplication):
     def event(self, event):
         if event.type() == QEvent.Quit and event.spontaneous():
-            self.main_app.quit()
+            if not self.quit():
+                return True
         return super().event(event)