shinestacker 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

shinestacker/algorithms/vignetting.py

@@ -134,7 +134,8 @@ class Vignetting(SubAction):
         self.corrections = None
 
     def run_frame(self, idx, _ref_idx, img_0):
-        self.process.sub_message_r(color_str(": compute vignetting", "cyan"))
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: compute vignetting", "cyan"))
         h, w = img_0.shape[:2]
         self.w_2, self.h_2 = w / 2, h / 2
         self.r_max = np.sqrt((w / 2)**2 + (h / 2)**2)
@@ -153,12 +154,13 @@ class Vignetting(SubAction):
             return img_0
         self.v0 = sigmoid_model(0, *params)
         i0_fit, k_fit, r0_fit = params
-        self.process.sub_message(color_str(": vignetting model parameters: ", "cyan") +
-                                 color_str(f"i0={i0_fit / 2:.4f}, "
-                                           f"k={k_fit * self.r_max:.4f}, "
-                                           f"r0={r0_fit / self.r_max:.4f}",
-                                           "light_blue"),
-                                 level=logging.DEBUG)
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: vignetting model parameters: ", "cyan") +
+            color_str(f"i0={i0_fit / 2:.4f}, "
+                      f"k={k_fit * self.r_max:.4f}, "
+                      f"r0={r0_fit / self.r_max:.4f}",
+                      "light_blue"),
+            level=logging.DEBUG)
         if self.plot_correction:
             plt.figure(figsize=constants.PLT_FIG_SIZE)
             plt.plot(radii, intensities, label="image mean intensity")
@@ -175,12 +177,13 @@ class Vignetting(SubAction):
             save_plot(plot_path)
             plt.close('all')
             self.process.callback(
-                'save_plot', self.process.id,
+                constants.CALLBACK_SAVE_PLOT, self.process.id,
                 f"{self.process.name}: intensity\nframe {idx_str}", plot_path)
         for i, p in enumerate(self.percentiles):
             self.corrections[i][idx] = fsolve(lambda x: sigmoid_model(x, *params) /
                                               self.v0 - p, r0_fit)[0]
-        self.process.sub_message_r(color_str(": correct vignetting", "cyan"))
+        self.process.print_message(
+            color_str(f"{self.process.idx_tot_str(idx)}: correct vignetting", "cyan"))
         return correct_vignetting(
             img_0, self.max_correction, self.black_threshold, None, params, self.v0,
             subsample, self.fast_subsampling)
@@ -224,5 +227,5 @@ class Vignetting(SubAction):
                         f"{self.process.name}-r0.pdf"
             save_plot(plot_path)
             plt.close('all')
-            self.process.callback('save_plot', self.process.id,
+            self.process.callback(constants.CALLBACK_SAVE_PLOT, self.process.id,
                                   f"{self.process.name}: vignetting", plot_path)

shinestacker/app/main.py

@@ -142,9 +142,12 @@ class MainApp(QMainWindow):
         return app_menu
 
     def quit(self):
-        self.retouch_window.quit()
-        self.project_window.quit()
+        if not self.retouch_window.quit():
+            return False
+        if not self.project_window.quit():
+            return False
         self.close()
+        return True
 
     def switch_app(self, index):
         self.stacked_widget.setCurrentIndex(index)
@@ -192,7 +195,8 @@ class MainApp(QMainWindow):
 class Application(QApplication):
     def event(self, event):
         if event.type() == QEvent.Quit and event.spontaneous():
-            self.main_app.quit()
+            if not self.quit():
+                return True
         return super().event(event)
 
 

shinestacker/config/constants.py

@@ -37,19 +37,25 @@ class _Constants:
     ACTION_VIGNETTING = "Vignetting"
     ACTION_ALIGNFRAMES = "AlignFrames"
     ACTION_BALANCEFRAMES = "BalanceFrames"
-    SUB_ACTION_TYPES = [ACTION_MASKNOISE, ACTION_VIGNETTING, ACTION_ALIGNFRAMES,
-                        ACTION_BALANCEFRAMES]
+    SUB_ACTION_TYPES = [ACTION_ALIGNFRAMES, ACTION_BALANCEFRAMES,
+                        ACTION_VIGNETTING, ACTION_MASKNOISE]
     STACK_ALGO_PYRAMID = 'Pyramid'
     STACK_ALGO_DEPTH_MAP = 'Depth map'
     STACK_ALGO_OPTIONS = [STACK_ALGO_PYRAMID, STACK_ALGO_DEPTH_MAP]
     STACK_ALGO_DEFAULT = STACK_ALGO_PYRAMID
-    DEFAULT_PLOTS_PATH = 'plots'
 
-    FIELD_SUBSAMPLE_VALUES_1 = [2, 3, 4, 6, 8, 12, 16, 24, 32]
-    FIELD_SUBSAMPLE_OPTIONS_1 = [f"1/{n} × 1/{n}" for n in FIELD_SUBSAMPLE_VALUES_1]
-    FIELD_SUBSAMPLE_VALUES = [0, 1] + FIELD_SUBSAMPLE_VALUES_1
-    FIELD_SUBSAMPLE_OPTIONS = ['Auto', 'Full resolution'] + FIELD_SUBSAMPLE_OPTIONS_1
-    FIELD_SUBSAMPLE_DEFAULT = FIELD_SUBSAMPLE_VALUES[0]
+    ACTION_ICONS = {
+        ACTION_JOB: '',
+        ACTION_COMBO: '',
+        ACTION_NOISEDETECTION: '',
+        ACTION_FOCUSSTACK: '',
+        ACTION_FOCUSSTACKBUNCH: '',
+        ACTION_MULTILAYER: '',
+        ACTION_MASKNOISE: '',
+        ACTION_VIGNETTING: '',
+        ACTION_ALIGNFRAMES: '',
+        ACTION_BALANCEFRAMES: ''
+    }
 
     PATH_SEPARATOR = ';'
 
@@ -60,10 +66,39 @@ class _Constants:
     LOG_COLOR_LEVEL_2 = 'magenta'
     LOG_COLOR_LEVEL_3 = 'cyan'
 
+    STATUS_RUNNING = 1
+    STATUS_PAUSED = 2
+    STATUS_STOPPED = 3
+
+    RUN_COMPLETED = 0
+    RUN_ONGOING = 1
+    RUN_FAILED = 2
+    RUN_STOPPED = 3
+
+    CALLBACK_BEFORE_ACTION = 'before_action'
+    CALLBACK_AFTER_ACTION = 'after_action'
+    CALLBACK_STEP_COUNTS = 'step_counts'
+    CALLBACK_BEGIN_STEPS = 'begin_steps'
+    CALLBACK_END_STEPS = 'end_steps'
+    CALLBACK_AFTER_STEP = 'after_step'
+    CALLBACK_CHECK_RUNNING = 'check_running'
+    CALLBACK_SAVE_PLOT = 'save_plot'
+    CALLBACK_OPEN_APP = 'open_app'
+
     DEFAULT_FILE_REVERSE_ORDER = False
     DEFAULT_MULTILAYER_FILE_REVERSE_ORDER = True
     MULTILAYER_WARNING_MEM_GB = 1
 
+    DEFAULT_PLOTS_PATH = 'plots'
+    DEFAULT_MAX_FWK_THREADS = 8
+    DEFAULT_MAX_FWK_CHUNK_SUBMIT = True
+
+    FIELD_SUBSAMPLE_VALUES_1 = [2, 3, 4, 6, 8, 12, 16, 24, 32]
+    FIELD_SUBSAMPLE_OPTIONS_1 = [f"1/{n} × 1/{n}" for n in FIELD_SUBSAMPLE_VALUES_1]
+    FIELD_SUBSAMPLE_VALUES = [0, 1] + FIELD_SUBSAMPLE_VALUES_1
+    FIELD_SUBSAMPLE_OPTIONS = ['Auto', 'Full resolution'] + FIELD_SUBSAMPLE_OPTIONS_1
+    FIELD_SUBSAMPLE_DEFAULT = FIELD_SUBSAMPLE_VALUES[0]
+
     DEFAULT_NOISE_MAP_FILENAME = "noise-map/hot_pixels.png"
     DEFAULT_NOISE_MAX_FRAMES = 10
     DEFAULT_MN_KERNEL_SIZE = 3
@@ -104,9 +139,9 @@ class _Constants:
     NOKNN_METHODS = {'detectors': [DETECTOR_ORB, DETECTOR_SURF, DETECTOR_AKAZE, DETECTOR_BRISK],
                      'descriptors': [DESCRIPTOR_ORB, DESCRIPTOR_AKAZE, DESCRIPTOR_BRISK]}
 
-    DEFAULT_DETECTOR = DETECTOR_SIFT
-    DEFAULT_DESCRIPTOR = DESCRIPTOR_SIFT
-    DEFAULT_MATCHING_METHOD = MATCHING_KNN
+    DEFAULT_DETECTOR = DETECTOR_ORB
+    DEFAULT_DESCRIPTOR = DESCRIPTOR_ORB
+    DEFAULT_MATCHING_METHOD = MATCHING_NORM_HAMMING
     DEFAULT_FLANN_IDX_KDTREE = 2
     DEFAULT_FLANN_TREES = 5
     DEFAULT_FLANN_CHECKS = 50
@@ -124,7 +159,13 @@ class _Constants:
     DEFAULT_ALIGN_SUBSAMPLE = 0
     DEFAULT_ALIGN_RES_TARGET_MPX = 2
     DEFAULT_ALIGN_FAST_SUBSAMPLING = False
-    DEFAULT_ALIGN_MIN_GOOD_MATCHES = 50
+    DEFAULT_ALIGN_MIN_GOOD_MATCHES = 20
+    ALIGN_VALID_MODES = ['auto', 'sequential', 'parallel']
+    DEFAULT_ALIGN_MODE = 'auto'
+    DEFAULT_ALIGN_MEMORY_LIMIT_GB = 8
+    DEFAULT_ALIGN_MAX_THREADS = min(os.cpu_count() or 4, 8)
+    DEFAULT_ALIGN_CHUNK_SUBMIT = True
+    DEFAULT_ALIGN_BW_MATCHING = False
 
     BALANCE_LINEAR = "LINEAR"
     BALANCE_GAMMA = "GAMMA"
@@ -162,13 +203,13 @@ class _Constants:
     DEFAULT_STACK_PREFIX = "stack_"
     DEFAULT_BUNCH_PREFIX = "bunch_"
 
-    DEFAULT_DM_FLOAT = FLOAT_32
     DM_ENERGY_LAPLACIAN = "laplacian"
     DM_ENERGY_SOBEL = "sobel"
     DM_MAP_AVERAGE = "average"
     DM_MAP_MAX = "max"
     VALID_DM_MAP = [DM_MAP_AVERAGE, DM_MAP_MAX]
     VALID_DM_ENERGY = [DM_ENERGY_LAPLACIAN, DM_ENERGY_SOBEL]
+    DEFAULT_DM_FLOAT = FLOAT_32
     DEFAULT_DM_MAP = DM_MAP_AVERAGE
     DEFAULT_DM_ENERGY = DM_ENERGY_LAPLACIAN
     DEFAULT_DM_KERNEL_SIZE = 5
@@ -185,22 +226,16 @@ class _Constants:
     DEFAULT_PY_N_TILED_LAYERS = 2
     DEFAULT_PY_MEMORY_LIMIT_GB = 8
     DEFAULT_PY_MAX_THREADS = min(os.cpu_count() or 4, 8)
-    DEFAULT_PY_MODE = 'auto'
     PY_VALID_MODES = ['auto', 'memory', 'tiled']
-    MIN_PY_TILE_SIZE = 256
+    DEFAULT_PY_MODE = 'auto'
+    DEFAULT_PY_MAX_TILE_SIZE = 4096
+    DEFAULT_PY_MIN_TILE_SIZE = 128
+    DEFAULT_PY_MIN_N_TILED_LAYERS = 1
+    PY_MEMORY_OVERHEAD = 2.5
 
-    DEFAULT_PLOT_STACK_BUNCH = False
+    DEFAULT_PLOT_STACK_BUNCH = True
     DEFAULT_PLOT_STACK = True
 
-    STATUS_RUNNING = 1
-    STATUS_PAUSED = 2
-    STATUS_STOPPED = 3
-
-    RUN_COMPLETED = 0
-    RUN_ONGOING = 1
-    RUN_FAILED = 2
-    RUN_STOPPED = 3
-
     def __setattr__aux(self, name, value):
         raise AttributeError(f"Can't reassign constant '{name}'")
 

shinestacker/config/gui_constants.py

@@ -16,7 +16,7 @@ class _GuiConstants:
     NEW_PROJECT_ALIGN_FRAMES = True
     NEW_PROJECT_BALANCE_FRAMES = True
     NEW_PROJECT_BUNCH_STACK = False
-    NEW_PROJECT_BUNCH_FRAMES = {'min': 2, 'max': 20}
+    NEW_PROJECT_BUNCH_FRAMES = {'min': 1, 'max': 20}
     NEW_PROJECT_BUNCH_OVERLAP = {'min': 0, 'max': 10}
     NEW_PROJECT_FOCUS_STACK_PYRAMID = True
     NEW_PROJECT_FOCUS_STACK_DEPTH_MAP = False

shinestacker/core/core_utils.py

@@ -50,3 +50,7 @@ def running_under_macos() -> bool:
 
 def running_under_linux() -> bool:
     return platform.system().lower() == 'linux'
+
+
+def make_chunks(ll, max_size):
+    return [ll[i:i + max_size] for i in range(0, len(ll), max_size)]

shinestacker/core/framework.py

@@ -1,15 +1,17 @@
-# pylint: disable=C0114, C0115, C0116, R0917, R0913, R0902
+# pylint: disable=C0114, C0115, C0116, R0917, R0913, R0902, W0718
 import time
 import logging
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from .. config.constants import constants
 from .. config.config import config
 from .colors import color_str
 from .logging import setup_logging
-from .core_utils import make_tqdm_bar
+from .core_utils import make_tqdm_bar, make_chunks
 from .exceptions import RunStopException
 
 LINE_UP = "\r\033[A"
-TRAILING_SPACES = " " * 30
+TRAILING_SPACES = " " * 50
 
 
 class TqdmCallbacks:
@@ -62,7 +64,7 @@ def elapsed_time_str(start):
     return f"{hh:02d}:{mm:02d}:{ss:05.2f}s"
 
 
-class JobBase:
+class TaskBase:
     def __init__(self, name, enabled=True):
         self.id = -1
         self.name = name
@@ -77,8 +79,7 @@ class JobBase:
             self.begin_r, self.end_r = LINE_UP, None
 
     def callback(self, key, *args):
-        has_callbacks = hasattr(self, 'callbacks')
-        if has_callbacks and self.callbacks is not None:
+        if self.callbacks is not None:
             callback = self.callbacks.get(key, None)
             if callback:
                 return callback(*args)
@@ -92,9 +93,9 @@ class JobBase:
         if not self.enabled:
             self.get_logger().warning(color_str(self.name + ": entire job disabled",
                                                 constants.LOG_COLOR_ALERT))
-        self.callback('before_action', self.id, self.name)
+        self.callback(constants.CALLBACK_BEFORE_ACTION, self.id, self.name)
         self.run_core()
-        self.callback('after_action', self.id, self.name)
+        self.callback(constants.CALLBACK_AFTER_ACTION, self.id, self.name)
         msg_name = color_str(self.name + ":", constants.LOG_COLOR_LEVEL_JOB, "bold")
         msg_time = color_str(f"elapsed time: {elapsed_time_str(self._t0)}",
                              constants.LOG_COLOR_LEVEL_JOB)
@@ -146,9 +147,9 @@ class JobBase:
         self.sub_message(msg, level, self.end_r, self.begin_r, False)
 
 
-class Job(JobBase):
+class Job(TaskBase):
     def __init__(self, name, logger_name=None, log_file='', callbacks=None, **kwargs):
-        JobBase.__init__(self, name, **kwargs)
+        TaskBase.__init__(self, name, **kwargs)
         self.action_counter = 0
         self.__actions = []
         if logger_name is None:
@@ -163,7 +164,7 @@ class Job(JobBase):
     def init(self, a):
         pass
 
-    def add_action(self, a: JobBase):
+    def add_action(self, a: TaskBase):
         a.id = self.action_counter
         self.action_counter += 1
         a.logger = self.logger
@@ -183,47 +184,127 @@ class Job(JobBase):
                 self.get_logger().warning(color_str(a.name + f": {msg} disabled",
                                                     constants.LOG_COLOR_ALERT))
             else:
-                if self.callback('check_running', self.id, self.name) is False:
+                if self.callback(constants.CALLBACK_CHECK_RUNNING,
+                                 self.id, self.name) is False:
                     raise RunStopException(self.name)
                 a.run()
 
 
-class ActionList(JobBase):
+class SequentialTask(TaskBase):
     def __init__(self, name, enabled=True, **kwargs):
-        JobBase.__init__(self, name, enabled, **kwargs)
+        self.max_threads = kwargs.pop('max_threads', constants.DEFAULT_MAX_FWK_THREADS)
+        self.chunk_submit = kwargs.pop('chunk_submit', constants.DEFAULT_MAX_FWK_CHUNK_SUBMIT)
+        TaskBase.__init__(self, name, enabled, **kwargs)
         self.total_action_counts = None
         self.current_action_count = None
+        self.begin_steps = 0
 
     def set_counts(self, counts):
         self.total_action_counts = counts
-        self.callback('step_counts', self.id, self.name, self.total_action_counts)
+        self.callback(constants.CALLBACK_STEP_COUNTS,
+                      self.id, self.name, self.total_action_counts)
+
+    def add_begin_steps(self, steps):
+        self.begin_steps += steps
 
     def begin(self):
-        self.callback('begin_steps', self.id, self.name)
+        self.callback(constants.CALLBACK_BEGIN_STEPS, self.id, self.name)
 
     def end(self):
-        self.callback('end_steps', self.id, self.name)
+        self.callback(constants.CALLBACK_END_STEPS, self.id, self.name)
 
     def __iter__(self):
         self.current_action_count = 0
         return self
 
-    def run_step(self):
+    def run_step(self, action_count=-1):
         pass
 
     def __next__(self):
         if self.current_action_count < self.total_action_counts:
-            self.run_step()
+            self.run_step(self.current_action_count)
             x = self.current_action_count
             self.current_action_count += 1
             return x
         raise StopIteration
 
+    def check_running(self):
+        if self.callback(constants.CALLBACK_CHECK_RUNNING,
+                         self.id, self.name) is False:
+            raise RunStopException(self.name)
+
+    def after_step(self, step=-1):
+        if step == -1:
+            step = self.current_action_count + self.begin_steps
+        self.callback(constants.CALLBACK_AFTER_STEP, self.id, self.name, step)
+
+    def run_core_serial(self):
+        for _ in iter(self):
+            self.after_step()
+            self.check_running()
+
+    def idx_tot_str(self, idx):
+        return f"{idx + 1}/{self.total_action_counts}"
+
+    def run_core_parallel_single_chunk(self, idx_chunk):
+        with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+            future_to_index = {}
+            for idx in idx_chunk:
+                self.print_message(color_str(
+                    f"submit processing step: {self.idx_tot_str(idx)}",
+                    constants.LOG_COLOR_LEVEL_1))
+                future = executor.submit(self.run_step, idx)
+                future_to_index[future] = idx
+                self.check_running()
+            for future in as_completed(future_to_index):
+                idx = future_to_index[future]
+                try:
+                    result = future.result()
+                    if result:
+                        self.print_message(color_str(
+                            f"completed processing step: {self.idx_tot_str(idx)}",
+                            constants.LOG_COLOR_LEVEL_1))
+                    else:
+                        self.print_message(color_str(
+                            f"failed processing step: {self.idx_tot_str(idx)}",
+                            constants.LOG_COLOR_WARNING))
+                    self.current_action_count += 1
+                    self.after_step()
+                    self.check_running()
+                except RunStopException as e:
+                    raise e
+                except Exception as e:
+                    traceback.print_tb(e.__traceback__)
+                    self.print_message(color_str(
+                        f"failed processing step: {idx + 1}: {str(e)}",
+                        constants.LOG_COLOR_ALERT))
+
+    def run_core_parallel(self):
+        self.current_action_count = 0
+        self.run_core_parallel_single_chunk(list(range(self.total_action_counts)))
+
+    def run_core_parallel_chunks(self):
+        self.current_action_count = 0
+        action_idx_list = list(range(self.total_action_counts))
+        max_chunck_size = self.max_threads
+        action_idx_chunks = make_chunks(action_idx_list, max_chunck_size)
+        for idx_chunk in action_idx_chunks:
+            self.run_core_parallel_single_chunk(idx_chunk)
+
     def run_core(self):
         self.print_message(color_str('begin run', constants.LOG_COLOR_LEVEL_2), end='\n')
         self.begin()
-        for _ in iter(self):
-            self.callback('after_step', self.id, self.name, self.current_action_count)
-            if self.callback('check_running', self.id, self.name) is False:
-                raise RunStopException(self.name)
+        if self.run_sequential():
+            self.run_core_serial()
+        else:
+            if self.chunk_submit:
+                self.run_core_parallel_chunks()
+            else:
+                self.run_core_parallel()
         self.end()
+
+    def sequential_processing(self):
+        return False
+
+    def run_sequential(self):
+        return self.sequential_processing() or self.max_threads == 1

shinestacker/gui/action_config.py

@@ -1,8 +1,7 @@
 # pylint: disable=C0114, C0115, C0116, E0611, R0913, R0917, R0915, R0912
-# pylint: disable=E0606, W0718, R1702, W0102, W0221
+# pylint: disable=E0606, W0718, R1702, W0102, W0221, R0914, E1121
 import traceback
 from abc import ABC, abstractmethod
-from typing import Dict, Any
 import os.path
 from PySide6.QtWidgets import (QPushButton, QHBoxLayout, QFileDialog, QLabel, QComboBox,
                                QMessageBox, QSizePolicy, QLineEdit, QSpinBox,
@@ -128,7 +127,7 @@ class FieldBuilder:
                         spinbox = widget.layout().itemAt(1 + i * 2).widget()
                         spinbox.setValue(default[i])
                 elif field['type'] == FIELD_COMBO:
-                    widget.setCurrentText(default)
+                    widget.setCurrentText(str(default))
 
     def get_path_widget(self, widget):
         return widget.layout().itemAt(0).widget()
@@ -145,7 +144,7 @@ class FieldBuilder:
             parent = parent.parent
         return ''
 
-    def update_params(self, params: Dict[str, Any]) -> bool:
+    def update_params(self, params):
         for tag, field in self.fields.items():
             if field['type'] == FIELD_TEXT:
                 params[tag] = field['widget'].text()
@@ -224,7 +223,7 @@ class FieldBuilder:
             self.action.params.get(tag, ''),
             kwargs.get('placeholder', ''),
             tag.replace('_', ' ')
-        )[1]
+        )
 
     def create_rel_path_field(self, tag, **kwargs):
         value = self.action.params.get(tag, kwargs.get('default', ''))