xmipp3-installer 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

xmipp3_installer/installer/modes/mode_sync/mode_add_model_executor.py

@@ -1,3 +1,9 @@
+"""
+### Mode Add Model Executor Module.
+
+This module contains the class to upload models to the remote server.
+"""
+
 import os
 import tarfile
 from typing import Dict, Tuple
@@ -11,94 +17,102 @@ from xmipp3_installer.installer.handlers import shell_handler
 from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSyncExecutor
 
 class ModeAddModelExecutor(ModeSyncExecutor):
-	def __init__(self, context: Dict):
-		"""
-		### Constructor.
-		
-		#### Params:
-		- context (dict): Dictionary containing the installation context variables.
-		"""
-		super().__init__(context)
-		self.login = context.pop(params.PARAM_LOGIN)
-		self.model_path = context.pop(params.PARAM_MODEL_PATH)
-		self.update = context.pop(params.PARAM_UPDATE)
-		self.model_dir = os.path.dirname(self.model_path)
-		self.model_name = os.path.basename(self.model_path)
-		self.tar_file_name = f"xmipp_model_{self.model_name}.tgz"
-		self.tar_file_path = os.path.join(self.model_dir, self.tar_file_name)
+  """
+  ### Mode Add Model Executor.
+
+  Uploads models to the remote server.
+  """
+
+  def __init__(self, context: Dict):
+    """
+    ### Constructor.
+    
+    #### Params:
+    - context (dict): Dictionary containing the installation context variables.
+    """
+    super().__init__(context)
+    self.login = context.pop(params.PARAM_LOGIN)
+    self.model_path = context.pop(params.PARAM_MODEL_PATH)
+    self.update = context.pop(params.PARAM_UPDATE)
+    self.model_dir = os.path.dirname(self.model_path)
+    self.model_name = os.path.basename(self.model_path)
+    self.tar_file_name = f"xmipp_model_{self.model_name}.tgz"
+    self.tar_file_path = os.path.join(self.model_dir, self.tar_file_name)
 
-	def _sync_operation(self) -> Tuple[int, str]:
-		"""
-		### Uploads the model to the remote server.
+  def _sync_operation(self) -> Tuple[int, str]:
+    """
+    ### Uploads the model to the remote server.
 
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		if not os.path.isdir(self.model_path):
-			logger('\n'.join([
-				logger.red(f"{self.model_path} is not a directory. Please, check the path."),
-				logger.red("The name of the model will be the name of that folder.")
-			]))
-			return errors.IO_ERROR, ""
-		
-		ret_code, output = self.__generate_compressed_file()
-		if ret_code:
-			return ret_code, output
-		
-		if not self.__get_confirmation():
-			return errors.INTERRUPTED_ERROR, ""
-		
-		return self.__upload_model()
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    if not os.path.isdir(self.model_path):
+      logger('\n'.join([
+        logger.red(f"{self.model_path} is not a directory. Please, check the path."),
+        logger.red("The name of the model will be the name of that folder.")
+      ]))
+      return errors.IO_ERROR, ""
+    
+    ret_code, output = self.__generate_compressed_file()
+    if ret_code:
+      return ret_code, output
+    
+    if not self.__get_confirmation():
+      return errors.INTERRUPTED_ERROR, ""
+    
+    ret_code, output = self.__upload_model()
+    ret_code = 1 if ret_code else ret_code
+    return ret_code, output
 
-	def __generate_compressed_file(self) -> Tuple[int, str]:
-		"""
-		### Generates the model's compressed file.
+  def __generate_compressed_file(self) -> Tuple[int, str]:
+    """
+    ### Generates the model's compressed file.
 
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		logger(f"Creating the {self.tar_file_name} model.")
-		try:
-			with tarfile.open(self.tar_file_path, "w:gz") as tar:
-				tar.add(self.model_path, arcname=self.model_name)
-		except (tarfile.ReadError, tarfile.CompressionError) as ex:
-			return errors.IO_ERROR, str(ex)
-		return 0, ""
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    logger(f"Creating the {self.tar_file_name} model.")
+    try:
+      with tarfile.open(self.tar_file_path, "w:gz") as tar:
+        tar.add(self.model_path, arcname=self.model_name)
+    except (tarfile.ReadError, tarfile.CompressionError) as ex:
+      return errors.IO_ERROR, str(ex)
+    return 0, ""
 
-	def __get_confirmation(self) -> bool:
-		"""
-		### Asks the user for confirmation.
+  def __get_confirmation(self) -> bool:
+    """
+    ### Asks the user for confirmation.
 
-		#### Returns:
-		- (bool): True if the user confirms, False otherwise.
-		"""
-		logger('\n'.join([
-			logger.yellow("Warning: Uploading, please BE CAREFUL! This can be dangerous."),
-			f"You are going to be connected to {self.login} to write in folder {paths.SCIPION_SOFTWARE_EM}.",
-			"Continue? YES/no (case sensitive)"
-		]))
-		return user_interactions.get_user_confirmation("YES")
+    #### Returns:
+    - (bool): True if the user confirms, False otherwise.
+    """
+    logger('\n'.join([
+      logger.yellow("Warning: Uploading, please BE CAREFUL! This can be dangerous."),
+      f"You are going to be connected to {self.login} to write in folder {paths.SCIPION_SOFTWARE_EM}.",
+      "Continue? YES/no (case sensitive)"
+    ]))
+    return user_interactions.get_user_confirmation("YES")
 
-	def __upload_model(self) -> Tuple[int, str]:
-		"""
-		### Uploads the model to the remote server.
+  def __upload_model(self) -> Tuple[int, str]:
+    """
+    ### Uploads the model to the remote server.
 
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		update = '--update' if self.update else ''
-		args = f"{self.login}, {os.path.abspath(self.tar_file_path)}, {paths.SCIPION_SOFTWARE_EM}, {update}"    
-		logger(f"Trying to upload the model using {self.login} as login")
-		sync_program_relative_call = os.path.join(
-			".",
-			os.path.basename(self.sync_program_path)
-		)
-		ret_code, output = shell_handler.run_shell_command(
-			f"{sync_program_relative_call} upload {args}",
-			cwd=os.path.dirname(self.sync_program_path)
-		)
-		if not ret_code:
-			output = ""
-			logger(logger.green(f"{self.model_name} model successfully uploaded! Removing the local .tgz"))
-			os.remove(self.tar_file_path)
-		return ret_code, output
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    update = '--update' if self.update else ''
+    args = f"{self.login}, {os.path.abspath(self.tar_file_path)}, {paths.SCIPION_SOFTWARE_EM}, {update}"    
+    logger(f"Trying to upload the model using {self.login} as login")
+    sync_program_relative_call = os.path.join(
+      ".",
+      os.path.basename(self.sync_program_path)
+    )
+    ret_code, output = shell_handler.run_shell_command(
+      f"{sync_program_relative_call} upload {args}",
+      cwd=os.path.dirname(self.sync_program_path)
+    )
+    if not ret_code:
+      output = ""
+      logger(logger.green(f"{self.model_name} model successfully uploaded! Removing the local .tgz"))
+      os.remove(self.tar_file_path)
+    return ret_code, output

xmipp3_installer/installer/modes/mode_sync/mode_get_models_executor.py

@@ -1,51 +1,63 @@
+"""
+### Mode Get Models Executor Module.
+
+This module contains the class to download deep learning models.
+"""
+
 import os
 from typing import Dict, Tuple
 
 from xmipp3_installer.application.cli.arguments import params
 from xmipp3_installer.application.logger.logger import logger
-from xmipp3_installer.installer import constants, urls
+from xmipp3_installer.installer import urls
 from xmipp3_installer.installer.constants import paths
 from xmipp3_installer.installer.handlers import shell_handler
 from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSyncExecutor
 
 class ModeGetModelsExecutor(ModeSyncExecutor):
-	def __init__(self, context: Dict):
-		"""
-		### Constructor.
-		
-		#### Params:
-		- context (dict): Dictionary containing the installation context variables.
-		"""
-		super().__init__(context)
-		self.models_directory = context.pop(params.PARAM_MODELS_DIRECTORY)
-		self.dist_path = paths.get_source_path(constants.XMIPP)
-		if self.models_directory == self.dist_path:
-			self.models_directory = os.path.join(self.models_directory, 'models')
-
-	def _sync_operation(self) -> Tuple[int, str]:
-		"""
-		### Downloads deep learning models.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		if os.path.isdir(self.models_directory):
-			task = "update"
-			in_progress_task = "Updating"
-			completed_task = "updated"
-		else:
-			task = "download"
-			in_progress_task = "Downloading"
-			completed_task = "downloaded"
-		
-		logger(f"{in_progress_task} Deep Learning models (in background)")
-		ret_code, output = shell_handler.run_shell_command(
-			f"{self.sync_program_path} {task} {self.models_directory} {urls.MODELS_URL} DLmodels",
-			show_command=True,
-			show_output=True,
-			show_error=True
-		)
-		if not ret_code:
-			logger(logger.green(f"Models successfully {completed_task}!"))
-
-		return ret_code, output
+  """
+  ### Mode Get Models Executor.
+
+  Downloads deep learning models for the installation.
+  """
+  
+  def __init__(self, context: Dict):
+    """
+    ### Constructor.
+    
+    #### Params:
+    - context (dict): Dictionary containing the installation context variables.
+    """
+    super().__init__(context)
+    self.models_directory = context.pop(params.PARAM_MODELS_DIRECTORY)
+    if self.models_directory == os.path.abspath(paths.INSTALL_PATH):
+      self.models_directory = os.path.join(self.models_directory, 'models')
+
+  def _sync_operation(self) -> Tuple[int, str]:
+    """
+    ### Downloads deep learning models.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    if os.path.isdir(self.models_directory):
+      task = "update"
+      in_progress_task = "Updating"
+      completed_task = "updated"
+    else:
+      task = "download"
+      in_progress_task = "Downloading"
+      completed_task = "downloaded"
+    
+    logger(f"{in_progress_task} Deep Learning models (in background)")
+    ret_code = shell_handler.run_shell_command_in_streaming(
+      f"{self.sync_program_path} {task} {self.models_directory} {urls.MODELS_URL} DLmodels",
+      show_output=True,
+      show_error=True,
+      substitute=True
+    )
+    if not ret_code:
+      logger(logger.green(f"Models successfully {completed_task}!"))
+
+    ret_code = 1 if ret_code else ret_code
+    return ret_code, ""

xmipp3_installer/installer/modes/mode_sync/mode_sync_executor.py

@@ -1,3 +1,8 @@
+"""
+### Mode Sync Executor Module.
+
+This module contains the base class for mode executors with sync operations.
+"""
 import os
 from abc import abstractmethod
 from typing import Dict, Tuple
@@ -11,38 +16,39 @@ _SYNC_PROGRAM_PATH = os.path.join(".", paths.BINARIES_PATH)
 _SYNC_PROGRAM_NAME = "xmipp_sync_data"
 
 class ModeSyncExecutor(mode_executor.ModeExecutor):
-	"""Base class for mode executors with sync operations."""
-
-	def __init__(self, context: Dict):
-		"""
-		### Constructor.
-		
-		#### Params:
-		- context (dict): Dictionary containing the installation context variables.
-		"""
-		super().__init__(context)
-		self.sync_program_path = os.path.join(_SYNC_PROGRAM_PATH, _SYNC_PROGRAM_NAME)
-
-	def run(self) -> Tuple[int, str]:
-		"""
-		### Executes the sync operation.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		if not os.path.exists(self.sync_program_path):
-			logger('\n'.join([
-				logger.red(f"{self.sync_program_path} does not exist."),
-				logger.red("Xmipp needs to be compiled successfully before running this command!")
-			]))
-			return errors.IO_ERROR, ""
-		return self._sync_operation()
-
-	@abstractmethod
-	def _sync_operation(self) -> Tuple[int, str]:
-		"""
-		### Executes the specific sync operation.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
+  """Base class for mode executors with sync operations."""
+
+  def __init__(self, context: Dict):
+    """
+    ### Constructor.
+    
+    #### Params:
+    - context (dict): Dictionary containing the installation context variables.
+    """
+    super().__init__(context)
+    self.sync_program_name = _SYNC_PROGRAM_NAME
+    self.sync_program_path = os.path.join(_SYNC_PROGRAM_PATH, _SYNC_PROGRAM_NAME)
+
+  def run(self) -> Tuple[int, str]:
+    """
+    ### Executes the sync operation.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    if not os.path.exists(self.sync_program_path):
+      logger('\n'.join([
+        logger.red(f"{self.sync_program_path} does not exist."),
+        logger.red("Xmipp needs to be compiled successfully before running this command!")
+      ]))
+      return errors.IO_ERROR, ""
+    return self._sync_operation()
+
+  @abstractmethod
+  def _sync_operation(self) -> Tuple[int, str]:
+    """
+    ### Executes the specific sync operation.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """

xmipp3_installer/installer/modes/mode_sync/mode_test_executor.py

@@ -1,8 +1,13 @@
+"""
+### Mode Test Executor Module.
+
+This module contains the class to execute Xmipp tests.
+"""
 import os
-from typing import Dict, Tuple
+from typing import Dict, Tuple, Any
 
 from xmipp3_installer.application.cli.arguments import params
-from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.application.logger.logger import logger, errors
 from xmipp3_installer.installer import urls
 from xmipp3_installer.installer.constants import paths
 from xmipp3_installer.installer.handlers import shell_handler
@@ -10,82 +15,144 @@ from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSy
 from xmipp3_installer.repository.config_vars import variables
 
 _DATASET_NAME = "xmipp_programs"
+_PYTHON_TEST_SCRIPT_PATH = os.path.join(paths.SOURCES_PATH, "xmipp")
+_PYTHON_TEST_SCRIPT_INTERNAL_FOLDER = "tests"
 _PYTHON_TEST_SCRIPT_NAME = "test.py"
-_PYTHON_TEST_SCRIPT_PATH = os.path.join(paths.BINARIES_PATH, "tests")
+_PYTHON_DATA_FOLDER = 'data'
+_PYTHON_TEST_SCRIPT_INTERNAL_PATH = os.path.join(
+  _PYTHON_TEST_SCRIPT_INTERNAL_FOLDER,
+  _PYTHON_TEST_SCRIPT_NAME
+)
 _DEFAULT_PYTHON_HOME = "python3"
-_DATASET_PATH = os.path.join(_PYTHON_TEST_SCRIPT_PATH, 'data')
+_DATASET_PATH = os.path.join(_PYTHON_TEST_SCRIPT_PATH, _PYTHON_TEST_SCRIPT_INTERNAL_FOLDER,  _PYTHON_DATA_FOLDER)
+_TEST_DATA = os.path.join(_PYTHON_TEST_SCRIPT_INTERNAL_FOLDER, _PYTHON_DATA_FOLDER)
+_BASHRC_FILE_PATH = os.path.abspath(os.path.join(paths.INSTALL_PATH, "xmipp.bashrc"))
+_TESTS_SEPARATOR = " "
+_PARAM_MAPPER = {
+  params.PARAM_SHOW_TESTS: "--show",
+  params.PARAM_ALL_FUNCTIONS: "--allFuncs",
+  params.PARAM_ALL_PROGRAMS: "--allPrograms"
+}
 
 class ModeTestExecutor(ModeSyncExecutor):
-	"""Class to execute Xmipp tests."""
-
-	def __init__(self, context: Dict):
-		"""
-		### Constructor.
-		
-		#### Params:
-		- context (dict): Dictionary containing the installation context variables.
-		"""
-		super().__init__(context)
-		self.test_names = context.pop(params.PARAM_TEST_NAMES)
-		self.cuda = context.pop(variables.CUDA)
-		self.show = context.pop(params.PARAM_SHOW_TESTS)
-		python_home = context.pop(variables.PYTHON_HOME, None)
-		self.python_home = python_home if python_home else _DEFAULT_PYTHON_HOME
-	
-	def run(self) -> Tuple[int, str]:
-		"""
-		### Runs the provided tests.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		ret_code, output = super().run()
-		if ret_code:
-			return ret_code, output
-		return self.__run_tests()
-
-	def _sync_operation(self) -> Tuple[int, str]:
-		"""
-		### Executes the test operation.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		if os.path.isdir(_DATASET_PATH):
-			task_message = "Updating"
-			task = "update"
-			show_output = False
-		else:
-			task_message = "Downloading"
-			task = "download"
-			show_output = True
-		logger(logger.blue(f"{task_message} the test files"))
-
-		args = f"{_DATASET_PATH} {urls.SCIPION_TESTS_URL} {_DATASET_NAME}"
-		sync_program_relative_call = os.path.join(
-			".",
-			os.path.basename(self.sync_program_path)
-		)
-		return shell_handler.run_shell_command(
-			f"{sync_program_relative_call} {task} {args}",
-			cwd=os.path.dirname(self.sync_program_path),
-			show_output=show_output
-		)
-
-	def __run_tests(self) -> Tuple[int, str]:
-		"""
-		### Runs the specified tests.
-
-		#### Returns:
-		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
-		"""
-		no_cuda_str = "--noCuda" if not self.cuda else ""
-		show_str = "--show" if self.show else ""
-		logger(f" Tests to run: {', '.join(self.test_names)}")
-		
-		return shell_handler.run_shell_command(
-			f"{self.python_home} {_PYTHON_TEST_SCRIPT_NAME} {' '.join(self.test_names)} {no_cuda_str}{show_str}",
-			cwd=_PYTHON_TEST_SCRIPT_PATH,
-			show_output=True,
-			show_error=True
-		)
+  """Class to execute Xmipp tests."""
+
+  def __init__(self, context: Dict):
+    """
+    ### Constructor.
+    
+    #### Params:
+    - context (dict): Dictionary containing the installation context variables.
+    """
+    super().__init__(context)
+    self.param_value = self.__get_selected_param_value(context)
+    self.cuda = context.pop(variables.CUDA)
+    python_home = context.pop(variables.PYTHON_HOME, None)
+    self.python_home = python_home if python_home else _DEFAULT_PYTHON_HOME
+  
+  def run(self) -> Tuple[int, str]:
+    """
+    ### Runs the provided tests.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    ret_code, output = self.__load_bashrc()
+    if ret_code:
+      return ret_code, output
+    
+    ret_code, output = super().run()
+    if ret_code:
+      return ret_code, output
+    return self.__run_tests()
+
+  def _sync_operation(self) -> Tuple[int, str]:
+    """
+    ### Executes the test operation.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    if os.path.isdir(_DATASET_PATH):
+      task_message = "Updating"
+      task = "update"
+    else:
+      task_message = "Downloading"
+      task = "download"
+    logger(logger.blue(f"{task_message} the test files"))
+
+    args = f"{_TEST_DATA} {urls.SCIPION_TESTS_URL} {_DATASET_NAME}"
+    ret_code = shell_handler.run_shell_command_in_streaming(
+      f"{self.sync_program_name} {task} {args}",
+      cwd=_PYTHON_TEST_SCRIPT_PATH,
+      show_output=True,
+      show_error=True,
+      substitute=True
+    )
+    ret_code = 1 if ret_code else ret_code
+    return ret_code, ""
+
+  @staticmethod
+  def __load_bashrc() -> Tuple[int, str]:
+    """
+    ### Loads the bashrc file.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    if not os.path.exists(_BASHRC_FILE_PATH):
+      return errors.IO_ERROR, f"File {_BASHRC_FILE_PATH} does not exist."
+
+    ret_code, output = shell_handler.run_shell_command(
+      f"bash -c 'source {_BASHRC_FILE_PATH} && env'"
+    )
+    if ret_code:
+      return 1, output
+
+    for line in output.splitlines():
+      key, _, value = line.partition("=")
+      os.environ[key] = value
+    return 0, ""
+
+  def __run_tests(self) -> Tuple[int, str]:
+    """
+    ### Runs the specified tests.
+
+    #### Returns:
+    - (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+    """
+    no_cuda_str = "--noCuda" if not self.cuda else ""
+    if self.param_value not in {
+      _PARAM_MAPPER[params.PARAM_SHOW_TESTS],
+      _PARAM_MAPPER[params.PARAM_ALL_FUNCTIONS],
+      _PARAM_MAPPER[params.PARAM_ALL_PROGRAMS]
+    }:
+      test_names = self.param_value.replace(_TESTS_SEPARATOR, ", ")
+      logger(f" Tests to run: {test_names}")
+    ret_code, output = shell_handler.run_shell_command(
+      f"{self.python_home} {_PYTHON_TEST_SCRIPT_INTERNAL_PATH} {self.param_value} {no_cuda_str}",
+      cwd=_PYTHON_TEST_SCRIPT_PATH,
+      show_output=True
+    )
+    ret_code = 1 if ret_code else 0
+    return ret_code, output
+
+  @staticmethod
+  def __get_selected_param_value(context: Dict[str, Any]) -> str:
+    """
+    ### Returns the value of the param selected to run the test execution.
+
+    #### Params:
+    - context (dict(str, any)): Dictionary containing the installation context variables.
+
+    #### Returns:
+    - (str): The value of the selected variable to use
+    """
+    for variable in {
+      params.PARAM_SHOW_TESTS,
+      params.PARAM_ALL_FUNCTIONS,
+      params.PARAM_ALL_PROGRAMS
+    }:
+      if context[variable]:
+        return _PARAM_MAPPER[variable]
+    return _TESTS_SEPARATOR.join(context.pop(params.PARAM_TEST_NAMES))