xmipp3-installer 1.0.0__py3-none-any.whl

xmipp3_installer/installer/modes/mode_sync/mode_add_model_executor.py

@@ -0,0 +1,104 @@
+import os
+import tarfile
+from typing import Dict, Tuple
+
+from xmipp3_installer.application import user_interactions
+from xmipp3_installer.application.logger import errors
+from xmipp3_installer.application.cli.arguments import params
+from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.handlers import shell_handler
+from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSyncExecutor
+
+class ModeAddModelExecutor(ModeSyncExecutor):
+	def __init__(self, context: Dict):
+		"""
+		### Constructor.
+		
+		#### Params:
+		- context (dict): Dictionary containing the installation context variables.
+		"""
+		super().__init__(context)
+		self.login = context.pop(params.PARAM_LOGIN)
+		self.model_path = context.pop(params.PARAM_MODEL_PATH)
+		self.update = context.pop(params.PARAM_UPDATE)
+		self.model_dir = os.path.dirname(self.model_path)
+		self.model_name = os.path.basename(self.model_path)
+		self.tar_file_name = f"xmipp_model_{self.model_name}.tgz"
+		self.tar_file_path = os.path.join(self.model_dir, self.tar_file_name)
+
+	def _sync_operation(self) -> Tuple[int, str]:
+		"""
+		### Uploads the model to the remote server.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		if not os.path.isdir(self.model_path):
+			logger('\n'.join([
+				logger.red(f"{self.model_path} is not a directory. Please, check the path."),
+				logger.red("The name of the model will be the name of that folder.")
+			]))
+			return errors.IO_ERROR, ""
+		
+		ret_code, output = self.__generate_compressed_file()
+		if ret_code:
+			return ret_code, output
+		
+		if not self.__get_confirmation():
+			return errors.INTERRUPTED_ERROR, ""
+		
+		return self.__upload_model()
+
+	def __generate_compressed_file(self) -> Tuple[int, str]:
+		"""
+		### Generates the model's compressed file.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		logger(f"Creating the {self.tar_file_name} model.")
+		try:
+			with tarfile.open(self.tar_file_path, "w:gz") as tar:
+				tar.add(self.model_path, arcname=self.model_name)
+		except (tarfile.ReadError, tarfile.CompressionError) as ex:
+			return errors.IO_ERROR, str(ex)
+		return 0, ""
+
+	def __get_confirmation(self) -> bool:
+		"""
+		### Asks the user for confirmation.
+
+		#### Returns:
+		- (bool): True if the user confirms, False otherwise.
+		"""
+		logger('\n'.join([
+			logger.yellow("Warning: Uploading, please BE CAREFUL! This can be dangerous."),
+			f"You are going to be connected to {self.login} to write in folder {paths.SCIPION_SOFTWARE_EM}.",
+			"Continue? YES/no (case sensitive)"
+		]))
+		return user_interactions.get_user_confirmation("YES")
+
+	def __upload_model(self) -> Tuple[int, str]:
+		"""
+		### Uploads the model to the remote server.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		update = '--update' if self.update else ''
+		args = f"{self.login}, {os.path.abspath(self.tar_file_path)}, {paths.SCIPION_SOFTWARE_EM}, {update}"    
+		logger(f"Trying to upload the model using {self.login} as login")
+		sync_program_relative_call = os.path.join(
+			".",
+			os.path.basename(self.sync_program_path)
+		)
+		ret_code, output = shell_handler.run_shell_command(
+			f"{sync_program_relative_call} upload {args}",
+			cwd=os.path.dirname(self.sync_program_path)
+		)
+		if not ret_code:
+			output = ""
+			logger(logger.green(f"{self.model_name} model successfully uploaded! Removing the local .tgz"))
+			os.remove(self.tar_file_path)
+		return ret_code, output

xmipp3_installer/installer/modes/mode_sync/mode_get_models_executor.py

@@ -0,0 +1,51 @@
+import os
+from typing import Dict, Tuple
+
+from xmipp3_installer.application.cli.arguments import params
+from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.installer import constants, urls
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.handlers import shell_handler
+from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSyncExecutor
+
+class ModeGetModelsExecutor(ModeSyncExecutor):
+	def __init__(self, context: Dict):
+		"""
+		### Constructor.
+		
+		#### Params:
+		- context (dict): Dictionary containing the installation context variables.
+		"""
+		super().__init__(context)
+		self.models_directory = context.pop(params.PARAM_MODELS_DIRECTORY)
+		self.dist_path = paths.get_source_path(constants.XMIPP)
+		if self.models_directory == self.dist_path:
+			self.models_directory = os.path.join(self.models_directory, 'models')
+
+	def _sync_operation(self) -> Tuple[int, str]:
+		"""
+		### Downloads deep learning models.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		if os.path.isdir(self.models_directory):
+			task = "update"
+			in_progress_task = "Updating"
+			completed_task = "updated"
+		else:
+			task = "download"
+			in_progress_task = "Downloading"
+			completed_task = "downloaded"
+		
+		logger(f"{in_progress_task} Deep Learning models (in background)")
+		ret_code, output = shell_handler.run_shell_command(
+			f"{self.sync_program_path} {task} {self.models_directory} {urls.MODELS_URL} DLmodels",
+			show_command=True,
+			show_output=True,
+			show_error=True
+		)
+		if not ret_code:
+			logger(logger.green(f"Models successfully {completed_task}!"))
+
+		return ret_code, output

xmipp3_installer/installer/modes/mode_sync/mode_sync_executor.py

@@ -0,0 +1,48 @@
+import os
+from abc import abstractmethod
+from typing import Dict, Tuple
+
+from xmipp3_installer.application.logger import errors
+from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.modes import mode_executor
+
+_SYNC_PROGRAM_PATH = os.path.join(".", paths.BINARIES_PATH)
+_SYNC_PROGRAM_NAME = "xmipp_sync_data"
+
+class ModeSyncExecutor(mode_executor.ModeExecutor):
+	"""Base class for mode executors with sync operations."""
+
+	def __init__(self, context: Dict):
+		"""
+		### Constructor.
+		
+		#### Params:
+		- context (dict): Dictionary containing the installation context variables.
+		"""
+		super().__init__(context)
+		self.sync_program_path = os.path.join(_SYNC_PROGRAM_PATH, _SYNC_PROGRAM_NAME)
+
+	def run(self) -> Tuple[int, str]:
+		"""
+		### Executes the sync operation.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		if not os.path.exists(self.sync_program_path):
+			logger('\n'.join([
+				logger.red(f"{self.sync_program_path} does not exist."),
+				logger.red("Xmipp needs to be compiled successfully before running this command!")
+			]))
+			return errors.IO_ERROR, ""
+		return self._sync_operation()
+
+	@abstractmethod
+	def _sync_operation(self) -> Tuple[int, str]:
+		"""
+		### Executes the specific sync operation.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""

xmipp3_installer/installer/modes/mode_sync/mode_test_executor.py

@@ -0,0 +1,91 @@
+import os
+from typing import Dict, Tuple
+
+from xmipp3_installer.application.cli.arguments import params
+from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.installer import urls
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.handlers import shell_handler
+from xmipp3_installer.installer.modes.mode_sync.mode_sync_executor import ModeSyncExecutor
+from xmipp3_installer.repository.config_vars import variables
+
+_DATASET_NAME = "xmipp_programs"
+_PYTHON_TEST_SCRIPT_NAME = "test.py"
+_PYTHON_TEST_SCRIPT_PATH = os.path.join(paths.BINARIES_PATH, "tests")
+_DEFAULT_PYTHON_HOME = "python3"
+_DATASET_PATH = os.path.join(_PYTHON_TEST_SCRIPT_PATH, 'data')
+
+class ModeTestExecutor(ModeSyncExecutor):
+	"""Class to execute Xmipp tests."""
+
+	def __init__(self, context: Dict):
+		"""
+		### Constructor.
+		
+		#### Params:
+		- context (dict): Dictionary containing the installation context variables.
+		"""
+		super().__init__(context)
+		self.test_names = context.pop(params.PARAM_TEST_NAMES)
+		self.cuda = context.pop(variables.CUDA)
+		self.show = context.pop(params.PARAM_SHOW_TESTS)
+		python_home = context.pop(variables.PYTHON_HOME, None)
+		self.python_home = python_home if python_home else _DEFAULT_PYTHON_HOME
+	
+	def run(self) -> Tuple[int, str]:
+		"""
+		### Runs the provided tests.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		ret_code, output = super().run()
+		if ret_code:
+			return ret_code, output
+		return self.__run_tests()
+
+	def _sync_operation(self) -> Tuple[int, str]:
+		"""
+		### Executes the test operation.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		if os.path.isdir(_DATASET_PATH):
+			task_message = "Updating"
+			task = "update"
+			show_output = False
+		else:
+			task_message = "Downloading"
+			task = "download"
+			show_output = True
+		logger(logger.blue(f"{task_message} the test files"))
+
+		args = f"{_DATASET_PATH} {urls.SCIPION_TESTS_URL} {_DATASET_NAME}"
+		sync_program_relative_call = os.path.join(
+			".",
+			os.path.basename(self.sync_program_path)
+		)
+		return shell_handler.run_shell_command(
+			f"{sync_program_relative_call} {task} {args}",
+			cwd=os.path.dirname(self.sync_program_path),
+			show_output=show_output
+		)
+
+	def __run_tests(self) -> Tuple[int, str]:
+		"""
+		### Runs the specified tests.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the return code and an error message if there was an error.
+		"""
+		no_cuda_str = "--noCuda" if not self.cuda else ""
+		show_str = "--show" if self.show else ""
+		logger(f" Tests to run: {', '.join(self.test_names)}")
+		
+		return shell_handler.run_shell_command(
+			f"{self.python_home} {_PYTHON_TEST_SCRIPT_NAME} {' '.join(self.test_names)} {no_cuda_str}{show_str}",
+			cwd=_PYTHON_TEST_SCRIPT_PATH,
+			show_output=True,
+			show_error=True
+		)

xmipp3_installer/installer/modes/mode_version_executor.py

@@ -0,0 +1,164 @@
+import os
+from typing import Dict, Tuple
+
+from xmipp3_installer.application.cli.arguments import params
+from xmipp3_installer.application.logger.logger import logger
+from xmipp3_installer.api_client.assembler import installation_info_assembler
+from xmipp3_installer.installer import constants
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.modes import mode_executor
+from xmipp3_installer.installer.handlers import git_handler, versions_manager
+from xmipp3_installer.installer.handlers.cmake import cmake_handler
+from xmipp3_installer.repository.config_vars import variables
+
+class ModeVersionExecutor(mode_executor.ModeExecutor):
+	__LEFT_TEXT_LEN = 25
+
+	def __init__(self, context: Dict):
+		"""
+		### Constructor.
+		
+		#### Params:
+		- context (dict): Dictionary containing the installation context variables.
+		"""
+		super().__init__(context)
+		self.short = context.pop(params.PARAM_SHORT)
+		config_exists = os.path.exists(paths.CONFIG_FILE)
+		self.version_file_exists = os.path.exists(paths.LIBRARY_VERSIONS_FILE)
+		self.is_configured = config_exists and self.version_file_exists
+		versions: versions_manager.VersionsManager = context[constants.VERSIONS_CONTEXT_KEY]
+		self.xmipp_version_number = versions.xmipp_version_number
+		self.xmipp_version_name = versions.xmipp_version_name
+		self.release_date = versions.xmipp_release_date
+	
+	def run(self) -> Tuple[int, str]:
+		"""
+		### Collects all the version information available and displays it.
+
+		#### Returns:
+		- (tuple(int, str)): Tuple containing the error status and an error message if there was an error.
+		"""
+		installation_info =  self.xmipp_version_name if self.short else self.__get_long_version()
+		logger(installation_info)
+		return 0, ""
+
+	def __get_long_version(self) -> str:
+		"""
+		### Returns the long version of the installation info.
+
+		#### Returns:
+		- (str): Long version of the installation info.
+		"""
+		installation_info_lines = []
+		version_type = 'release' if git_handler.is_tag() else git_handler.get_current_branch()
+		title = f"Xmipp {self.xmipp_version_number} ({version_type})"
+		installation_info_lines.append(f"{logger.bold(title)}\n")
+		installation_info_lines.append(self.__get_dates_section())
+		system_version_left_text = self.__add_padding_spaces("System version: ")
+		installation_info_lines.append(f"{system_version_left_text}{installation_info_assembler.get_os_release_name()}")
+		installation_info_lines.append(self.__get_sources_info())
+		if self.version_file_exists:
+			installation_info_lines.append(f"\n{self.__get_library_versions_section()}")
+		if not self.is_configured or not self.__are_all_sources_present():
+			installation_info_lines.append(f"\n{self.__get_configuration_warning_message()}")
+		return '\n'.join(installation_info_lines)
+
+	def __get_dates_section(self) -> str:
+		"""
+		### Returns the message section related to dates.
+
+		#### Returns:
+		- (str): Dates related message section.
+		"""
+		dates_section = f"{self.__add_padding_spaces('Release date: ')}{self.release_date}\n"
+		dates_section += f"{self.__add_padding_spaces('Compilation date: ')}"
+		last_modified = self.context.get(variables.LAST_MODIFIED_KEY)
+		dates_section += last_modified if last_modified else '-'
+		return dates_section
+
+	def __get_sources_info(self) -> str:
+		"""
+		### Returns the message section related to sources.
+
+		#### Returns:
+		- (str): Sources related message section.
+		"""
+		sources_message_lines = []
+		for source_package in constants.XMIPP_SOURCES:
+			sources_message_lines.append(self.__get_source_info(source_package))
+		return '\n'.join(sources_message_lines)
+
+	def __get_source_info(self, source: str) -> str:
+		"""
+		### Returns the info message related to a given source.
+
+		#### Params:
+		- source (str): Source to get the message about.
+
+		#### Returns:
+		- (str): Info message about the given source.
+		"""
+		source_path = paths.get_source_path(source)
+		source_left_text = self.__add_padding_spaces(f"{source} branch: ")
+		if not os.path.exists(source_path):
+			return f"{source_left_text}{logger.yellow('Not found')}"
+		current_commit = git_handler.get_current_commit(dir=source_path)
+		commit_branch = git_handler.get_commit_branch(current_commit, dir=source_path)
+		current_branch = git_handler.get_current_branch(dir=source_path)
+		display_name = commit_branch if git_handler.is_tag(dir=source_path) else current_branch
+		return f"{source_left_text}{display_name} ({current_commit})"
+
+	def __add_padding_spaces(self, left_text: str) -> str:
+		"""
+		### Adds right padding as spaces to the given text until it reaches the desired length.
+
+		#### Params:
+		- left_text (str): Text to add padding to.
+
+		#### Returns:
+		- (str): Padded string.
+		"""
+		text_len = len(left_text)
+		if text_len >= self.__LEFT_TEXT_LEN:
+			return left_text
+		spaces = ''.join([' ' for _ in range(self.__LEFT_TEXT_LEN - text_len)])
+		return f"{left_text}{spaces}"
+
+	def __get_library_versions_section(self) -> str:
+		"""
+		### Retrieves the version of the libraries used in the project.
+		
+		#### Returns:
+		- (str): Libraries with their version.
+		"""
+		version_lines = [] 
+		for library, version in cmake_handler.get_library_versions_from_cmake_file(
+			paths.LIBRARY_VERSIONS_FILE
+		).items():
+			library_left_text = self.__add_padding_spaces(f"{library}: ")
+			version_lines.append(f"{library_left_text}{version}")
+		return '\n'.join(version_lines)
+	
+	def __are_all_sources_present(self) -> bool:
+		"""
+		### Check if all required source packages are present.
+
+		#### Returns:
+		- (bool): True if all source packages are present, False otherwise.
+		"""
+		for source_package in paths.XMIPP_SOURCE_PATHS:
+			if not os.path.exists(source_package):
+				return False
+		return True
+
+	def __get_configuration_warning_message(self) -> str:
+		"""
+		### Returns a message indicating configuration is not complete.
+
+		#### Returns:
+		- (str): 
+		"""
+		return '\n'.join([
+			logger.yellow("This project has not yet been configured, so some detectable dependencies have not been properly detected."),
+			logger.yellow("Run mode 'getSources' and then 'configBuild' to be able to show all detectable ones.")
+		])

xmipp3_installer/installer/orquestrator.py

@@ -0,0 +1,37 @@
+"""### Contains functions that orquestrate other function executions."""
+
+import multiprocessing
+from typing import List, Tuple, Callable, Any
+
+def run_parallel_jobs(
+  funcs: List[Callable],
+  func_args: List[Tuple[Any]],
+  n_jobs: int=multiprocessing.cpu_count()
+) -> List:
+	"""
+	### Runs the given command list in parallel.
+
+	#### Params:
+	- funcs (list(callable)): Functions to run.
+	- func_args (list(tuple(any))): Arguments for each function.
+	- n_jobs (int): Optional. Number of parallel jobs.
+
+	#### Returns:
+	- (list): List containing the return of each function.
+	"""
+	with multiprocessing.Pool(n_jobs) as p:
+		results = p.starmap(__run_lambda, zip(funcs, func_args))
+	return results
+
+def __run_lambda(func: Callable, args: Tuple[Any]) -> Any:
+	"""
+	### Runs the given function with its args.
+	
+	#### Params:
+	- func (callable): Function to run.
+	- args (tuple(any)): Arguments for the function.
+	
+	#### Returns:
+	- (any): Return of the called function.
+	"""
+	return func(*args)

xmipp3_installer/installer/urls.py

@@ -0,0 +1,8 @@
+"""### URLs """
+
+DOCUMENTATION_URL = 'https://i2pc.github.io/docs/'
+CMAKE_INSTALL_DOCS_URL = 'https://i2pc.github.io/docs/Installation/InstallationNotes/index.html#cmake'
+API_URL = 'https://xmipp.i2pc.es/api/attempts/'
+MODELS_URL= "http://campins.cnb.csic.es/deep_cons/"
+SCIPION_TESTS_URL = "http://scipion.cnb.csic.es/downloads/scipion/data/tests"
+I2PC_REPOSITORY_URL = "https://github.com/i2pc/"

xmipp3_installer/repository/__init__.py

@@ -0,0 +1 @@
+"""### Contains functions that interact with the config file."""