xmipp3-installer 1.0.0__py3-none-any.whl

xmipp3_installer/__init__.py

@@ -0,0 +1 @@
+"""### Main package containing the installer."""

xmipp3_installer/__main__.py

@@ -0,0 +1,6 @@
+"""### System's entry point."""
+
+from xmipp3_installer.application.cli.cli import main
+
+if __name__ == "__main__":
+    main()

xmipp3_installer/api_client/api_client.py

@@ -0,0 +1,50 @@
+"""### Contains an API client that registers the installation attempts."""
+
+import http.client
+import json
+from typing import Dict
+from urllib.parse import urlparse, ParseResult
+
+from xmipp3_installer.installer import urls
+from xmipp3_installer.application.logger.logger import logger
+
+def send_installation_attempt(installation_info: Dict):
+	"""
+	### Sends a POST request to Xmipp's metrics's API.
+	
+	#### Params:
+	- installation_info (dict): Dictionary containing all the installation information.
+	"""
+	if installation_info is None:
+		return
+	params = json.dumps(installation_info)
+	headers = {"Content-type": "application/json"}
+	parsed_url = urlparse(urls.API_URL)
+	conn = None
+	try:
+		conn = __get_https_connection(parsed_url, 2)
+		conn.request("POST", parsed_url.path, body=params, headers=headers)
+		conn.getresponse()
+	except TimeoutError:
+		logger(
+			logger.yellow("There was a timeout while sending installation data."),
+			show_in_terminal=False
+		)
+	finally:
+		if conn is not None:
+			conn.close()
+
+
+def __get_https_connection(parsed_url: ParseResult, timeout_seconds: int) -> http.client.HTTPSConnection:
+	"""
+	### Establishes the connection needed to send the API call.
+	Separated to enable integration & E2E tests.
+
+	#### Params:
+	- parsed_url (ParseResult): Object containing all elements of the url.
+	- timeout_seconds (int): Number of seconds to wait until a timeout.
+
+	#### Returns:
+	- (HTTPSConnection): Established connection.
+	"""
+	return http.client.HTTPSConnection(parsed_url.hostname, parsed_url.port, timeout=timeout_seconds)

xmipp3_installer/api_client/assembler/installation_info_assembler.py

@@ -0,0 +1,181 @@
+"""### Contains functions to assemble the data dictionary required by the API client."""
+
+import getpass
+import hashlib
+import platform
+import re
+import os
+from typing import Optional, List, Dict
+
+import distro
+
+from xmipp3_installer.installer import constants, orquestrator
+from xmipp3_installer.installer.constants import paths
+from xmipp3_installer.installer.handlers import shell_handler, git_handler, versions_manager
+from xmipp3_installer.installer.handlers.cmake import cmake_constants, cmake_handler
+
+def get_installation_info(version_manager: versions_manager.VersionsManager, ret_code: int=0) -> Optional[Dict]:
+	"""
+	### Creates a dictionary with the necessary data for the API POST message.
+	
+	#### Params:
+	- version_manager (VersionsManager): Object containing all the version-related info.
+	- ret_code (int): Optional. Return code for the API request.
+	
+	#### Return:
+	- (dict | None): Dictionary with the required info or None if user id could not be produced.
+	"""
+	library_versions = cmake_handler.get_library_versions_from_cmake_file(
+		paths.LIBRARY_VERSIONS_FILE
+	)
+	environment_info = orquestrator.run_parallel_jobs(
+		[
+			__get_cpu_flags,
+			git_handler.get_current_branch,
+			git_handler.is_branch_up_to_date,
+			__is_installed_by_scipion,
+			__get_log_tail
+		],
+		[(), (), (), (), ()]
+	)
+
+	return {
+		"user": {
+			"userId": __get_user_id()
+		},
+		"version": {
+			"os": get_os_release_name(),
+			"cpuFlags": environment_info[0],
+			"cuda": library_versions.get(cmake_constants.CMAKE_CUDA),
+			"cmake": library_versions.get(cmake_constants.CMAKE_CMAKE),
+			"gcc": library_versions.get(cmake_constants.CMAKE_GCC),
+			"gpp": library_versions.get(cmake_constants.CMAKE_GPP),
+			"mpi": library_versions.get(cmake_constants.CMAKE_MPI),
+			"python": library_versions.get(cmake_constants.CMAKE_PYTHON),
+			"sqlite": library_versions.get(cmake_constants.CMAKE_SQLITE),
+			"java": library_versions.get(cmake_constants.CMAKE_JAVA),
+			"hdf5": library_versions.get(cmake_constants.CMAKE_HDF5),
+			"jpeg": library_versions.get(cmake_constants.CMAKE_JPEG)
+		},
+		"xmipp": {
+			"branch": __get_installation_branch_name(
+				environment_info[1], version_manager
+			),
+			"updated": environment_info[2],
+			"installedByScipion": environment_info[3]
+		},
+		"returnCode": ret_code,
+		"logTail": environment_info[4] if ret_code else None # Only needed if something went wrong
+	}
+
+def get_os_release_name() -> str:
+	"""
+	### Returns the name of the current system OS release.
+
+	#### Returns:
+	- (str): OS release name.
+	"""
+	platform_system = platform.system()
+	if platform_system == "Linux":
+		return f"{distro.name()} {distro.version()}"
+	return f"{platform_system} {platform.release()}"
+
+def __get_installation_branch_name(branch_name: str, version_manager: versions_manager.VersionsManager) -> str:
+	"""
+	### Returns the branch or release name of Xmipp.
+
+	#### Params:
+	- branch_name (str): Retrieved branch name.
+	- version_manager (VersionsManager): Object containing all the version-related info.
+
+	#### Return:
+	- (str): Release name if Xmipp is in a release branch, or the branch name otherwise.
+	"""
+	if not branch_name or branch_name == constants.MASTER_BRANCHNAME:
+		return version_manager.xmipp_version_number
+	return branch_name
+
+def __get_user_id() -> str:
+	"""
+	### Returns the unique user id for this machine.
+	
+	#### Returns:
+	- (str): User id, or 'Anoymous' if there were any errors.
+	"""
+	identifier = __get_mac_address()
+	if not identifier:
+		identifier = getpass.getuser()
+		if not identifier:
+			return "Anonymous"
+	
+	sha256 = hashlib.sha256()
+	sha256.update(identifier.encode())
+	return sha256.hexdigest()
+
+def __get_cpu_flags() -> List[str]:
+	"""
+	### This function obtains the list of compilation flags supported by the CPU.
+
+	#### Returns:
+	- (list(str)): List of flags supported by the CPU.  
+	"""
+	flags_header = "Flags:"
+	ret_code, flags_line = shell_handler.run_shell_command(f'lscpu | grep \"{flags_header}\"')
+	if ret_code:
+		return []
+	flags_line = flags_line.replace(flags_header, "").strip()
+	return [flag for flag in flags_line.split(" ") if flag]
+
+def __get_log_tail() -> Optional[str]:
+	"""
+	### Returns the last lines of the installation log.
+	
+	#### Returns:
+	- (str | None): Installation log's last lines, or None if there were any errors.
+	"""
+	ret_code, output = shell_handler.run_shell_command(
+		f"tail -n {constants.TAIL_LOG_NCHARS} {paths.LOG_FILE}"
+	)
+	return output if ret_code == 0 else None
+
+def __get_mac_address() -> Optional[str]:
+	"""
+	### Returns a physical MAC address for this machine. It prioritizes ethernet over wireless.
+	
+	#### Returns:
+	- (str | None): MAC address, or None if there were any errors.
+	"""
+	ret_code, output = shell_handler.run_shell_command("ip addr")
+	return __find_mac_address_in_lines(output.split('\n')) if ret_code == 0 else None
+
+def __find_mac_address_in_lines(lines: List[str]) -> Optional[str]:
+	"""
+	### Returns a physical MAC address within the text lines provided.
+
+	#### Params:
+	- lines (list(str)): Lines of text where MAC address should be looked for.
+	
+	#### Returns:
+	- (str | None): MAC address if found, None otherwise.
+	"""
+	for line_index in range(len(lines) - 1):
+		line = lines[line_index]
+		match = re.match(r"^\d+: (enp|wlp|eth|ens|eno)\w+", line)
+		if not match:
+			continue
+		interface_name = match.group(1)
+		if interface_name.startswith(('enp', 'wlp', 'eth', 'ens', 'eno')):
+			return re.search(
+				r"link/ether ([0-9a-f:]{17})",
+				lines[line_index + 1]
+			).group(1)
+	return None
+
+def __is_installed_by_scipion() -> bool:
+	"""
+	### Checks if the current xmipp installation is being carried out from Scipion.
+
+	#### Returns:
+	- (bool): True if the installation is executed by Scipion, False otherwise.
+	"""
+	return bool(os.getenv("SCIPION_SOFTWARE"))

xmipp3_installer/application/__init__.py

@@ -0,0 +1 @@
+"""### Contains application modules that are used or interact with the main package."""

xmipp3_installer/application/cli/__init__.py

@@ -0,0 +1 @@
+"""### CLI related modules."""

xmipp3_installer/application/cli/arguments/__init__.py

@@ -0,0 +1,8 @@
+"""### Containis all common constants needed for the argument parsing part of Xmipp's installation."""
+
+XMIPP_PROGRAM_NAME = "xmipp"
+
+# Other variables
+COMMON_USAGE_HELP_MESSAGE = 'Run \"./xmipp -h\" for usage help.'
+DEFAULT_BUILD_DIR = 'build'
+DEFAULT_MODELS_DIR = DEFAULT_BUILD_DIR

xmipp3_installer/application/cli/arguments/modes.py

@@ -0,0 +1,130 @@
+"""### Containis all mode constants needed for the argument parsing part of Xmipp's installation."""
+
+from xmipp3_installer.application.cli import arguments
+from xmipp3_installer.application.cli.arguments.params import (
+  PARAM_SHORT, PARAM_JOBS, PARAM_BRANCH, PARAM_KEEP_OUTPUT, PARAM_OVERWRITE,
+  PARAM_MODELS_DIRECTORY, PARAM_TEST_NAMES, PARAM_SHOW_TESTS, PARAM_GIT_COMMAND,
+  PARAM_LOGIN, PARAM_MODEL_PATH, PARAM_UPDATE, PARAMS, SHORT_VERSION, LONG_VERSION
+)
+
+MODE = "mode"
+
+# Mode list (alphabetical order)
+MODE_ADD_MODEL = 'addModel'
+MODE_ALL = 'all'
+MODE_CLEAN_ALL = 'cleanAll'
+MODE_CLEAN_BIN = 'cleanBin'
+MODE_COMPILE_AND_INSTALL = 'compileAndInstall'
+MODE_CONFIG_BUILD = 'configBuild'
+MODE_CONFIG = 'config'
+MODE_GET_MODELS = 'getModels'
+MODE_GET_SOURCES = 'getSources'
+MODE_GIT = 'git'
+MODE_TEST = 'test'
+MODE_VERSION = 'version'
+
+# Modes with help message
+MODES = {
+	'General': {
+		MODE_VERSION: ['Returns the version information. Add \'--short\' to print only the version number.'],
+		MODE_COMPILE_AND_INSTALL: ['Compiles and installs Xmipp based on already obtained sources.'],
+		MODE_ALL: [f'Default param. Runs {MODE_CONFIG}, {MODE_CONFIG_BUILD}, and {MODE_COMPILE_AND_INSTALL}.'],
+		MODE_CONFIG_BUILD: ['Configures the project with CMake.']
+	},
+	'Config': {
+		MODE_CONFIG: ['Generates a config file template with default values.'],
+	},
+	'Downloads': {
+		MODE_GET_MODELS: [f'Download the DeepLearning Models at dir/models ({arguments.DEFAULT_MODELS_DIR} by default).'],
+		MODE_GET_SOURCES: ['Clone all Xmipp\'s sources.']
+	},
+	'Clean': {
+		MODE_CLEAN_BIN: ['Removes all compiled binaries.'],
+		MODE_CLEAN_ALL: ['Removes all compiled binaries and sources, leaves the repository as if freshly cloned (without pulling).']
+	},
+	'Test': {
+		MODE_TEST: ['Runs a given test.']
+	},
+	'Developers': {
+		MODE_GIT: ['Runs the given git action for all source repositories.'],
+		MODE_ADD_MODEL: [
+			"Takes a DeepLearning model from the modelPath, makes a tgz of it and uploads the .tgz according to the <login>.",
+			"This mode is used to upload a model folder to the Scipion/Xmipp server.",
+			"Usually the model folder contains big files used to fed deep learning procedures"
+			"with pretrained data. All the models stored in the server will be downloads"
+			"using the 'get_models' mode or during the compilation/installation time"
+			"or with scipion3 installb deepLearningToolkit modelsPath must be the absolute path.",
+			"",
+			"Usage: -> ./xmipp addModel <usr@server> <modelsPath> [--update]",
+			"Steps:	0. modelName = basename(modelsPath) <- Please, check the folders name!",
+			"        1. Packing in 'xmipp_model_modelName.tgz'",
+			"        2. Check if that model already exists (use --update to override an existing model)",
+			"        3. Upload the model to the server.",
+			"        4. Update the MANIFEST file.",
+			"",
+			"The model name will be the folder name in <modelsPath>",
+			"Must have write permisions to such machine."
+		]
+	}
+}
+
+# Arguments of each mode, sorted by group
+MODE_ARGS = {
+	MODE_VERSION: [PARAM_SHORT],
+	MODE_COMPILE_AND_INSTALL: [PARAM_JOBS, PARAM_KEEP_OUTPUT],
+	MODE_ALL: [PARAM_JOBS, PARAM_BRANCH, PARAM_KEEP_OUTPUT],
+	MODE_CONFIG_BUILD: [PARAM_KEEP_OUTPUT],
+	MODE_CONFIG: [PARAM_OVERWRITE],
+	MODE_GET_MODELS: [PARAM_MODELS_DIRECTORY],
+	MODE_GET_SOURCES: [PARAM_BRANCH, PARAM_KEEP_OUTPUT],
+	MODE_CLEAN_BIN: [],
+	MODE_CLEAN_ALL: [],
+	MODE_TEST: [PARAM_TEST_NAMES, PARAM_SHOW_TESTS],
+	MODE_GIT: [PARAM_GIT_COMMAND],
+	MODE_ADD_MODEL: [PARAM_LOGIN, PARAM_MODEL_PATH, PARAM_UPDATE]
+}
+
+# Examples for the help message of each mode
+MODE_EXAMPLES = {
+	MODE_VERSION: [
+		f'./xmipp {MODE_VERSION}',
+		f'./xmipp {MODE_VERSION} {PARAMS[PARAM_SHORT][LONG_VERSION]}',
+	],
+	MODE_COMPILE_AND_INSTALL: [
+		f'./xmipp {MODE_COMPILE_AND_INSTALL}',
+		f'./xmipp {MODE_COMPILE_AND_INSTALL} {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20',
+	],
+	MODE_ALL: [
+		'./xmipp',
+		f'./xmipp {MODE_ALL}',
+		f'./xmipp {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20',
+		f'./xmipp {PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel',
+		f'./xmipp {MODE_ALL} {PARAMS[PARAM_JOBS][SHORT_VERSION]} 20 '
+		f'{PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel'
+	],
+	MODE_CONFIG_BUILD: [],
+	MODE_CONFIG: [
+		f'./xmipp {MODE_CONFIG} {PARAMS[PARAM_OVERWRITE][LONG_VERSION]}'
+	],
+	MODE_GET_MODELS: [
+		f'./xmipp {MODE_GET_MODELS}',
+		f'./xmipp {MODE_GET_MODELS} {PARAMS[PARAM_MODELS_DIRECTORY][SHORT_VERSION]} /path/to/my/model/directory'
+	],
+	MODE_GET_SOURCES: [
+		f'./xmipp {MODE_GET_SOURCES}'
+		f'./xmipp {MODE_GET_SOURCES} {PARAMS[PARAM_BRANCH][SHORT_VERSION]} devel'
+	],
+	MODE_CLEAN_BIN: [],
+	MODE_CLEAN_ALL: [],
+	MODE_TEST: [
+		f'./xmipp {MODE_TEST} xmipp_sample_test',
+		f'./xmipp {MODE_TEST} {PARAMS[PARAM_SHORT][LONG_VERSION]}',
+	],
+	MODE_GIT: [
+		f'./xmipp {MODE_GIT} pull',
+		f'./xmipp {MODE_GIT} checkout devel'
+	],
+	MODE_ADD_MODEL: [
+		f'./xmipp {MODE_ADD_MODEL} myuser@127.0.0.1 /home/myuser/mymodel'
+	]
+}

xmipp3_installer/application/cli/arguments/params.py

@@ -0,0 +1,76 @@
+"""### Containis all param constants needed for the argument parsing part of Xmipp's installation."""
+
+from xmipp3_installer.application.cli.arguments import DEFAULT_BUILD_DIR
+
+# Definition of all params found in the
+SHORT_VERSION = 'short'
+LONG_VERSION = 'long'
+DESCRIPTION = 'description'
+
+# Possible param list
+PARAM_SHORT = 'short'
+PARAM_JOBS = 'jobs'
+PARAM_BRANCH = 'branch'
+PARAM_MODELS_DIRECTORY = 'directory'
+PARAM_TEST_NAMES = 'testNames'
+PARAM_SHOW_TESTS = 'show'
+PARAM_GIT_COMMAND = 'command'
+PARAM_LOGIN = 'login'
+PARAM_MODEL_PATH = 'modelPath'
+PARAM_UPDATE = 'update'
+PARAM_OVERWRITE = 'overwrite'
+PARAM_KEEP_OUTPUT = "keep_output"
+PARAMS = {
+	PARAM_SHORT: {
+		LONG_VERSION: "--short",
+		DESCRIPTION: "If set, only version number is shown."
+	},
+	PARAM_JOBS: {
+		SHORT_VERSION: "-j",
+		LONG_VERSION: "--jobs",
+		DESCRIPTION: "Number of jobs. Defaults to all available."
+	},
+	PARAM_BRANCH: {
+		SHORT_VERSION: "-b",
+		LONG_VERSION: "--branch",
+		DESCRIPTION: "Branch for the source repositories."
+	},
+	PARAM_MODELS_DIRECTORY: {
+		SHORT_VERSION: "-d",
+		LONG_VERSION: "--directory",
+		DESCRIPTION: f"Directory where models will be saved. Default is \"{DEFAULT_BUILD_DIR}\"."
+	},
+	PARAM_TEST_NAMES: {
+		SHORT_VERSION: PARAM_TEST_NAMES,
+		DESCRIPTION: "Name of the tests to run. If combined with --show, greps the test names from the test list."
+	},
+	PARAM_SHOW_TESTS: {
+		LONG_VERSION: "--show",
+		DESCRIPTION: "Shows the tests available and how to invoke those."
+	},
+	PARAM_GIT_COMMAND: {
+		SHORT_VERSION: PARAM_GIT_COMMAND,
+		DESCRIPTION: "Git command to run on all source repositories."
+	},
+	PARAM_LOGIN: {
+		SHORT_VERSION: "login",
+		DESCRIPTION: "Login (usr@server) for remote host to upload the model with. Must have write permisions to such machine."
+	},
+	PARAM_MODEL_PATH: {
+		SHORT_VERSION: PARAM_MODEL_PATH,
+		DESCRIPTION: "Path to the model to upload to remote host."
+	},
+	PARAM_UPDATE: {
+		LONG_VERSION: "--update",
+		DESCRIPTION: "Flag to update an existing model"
+	},
+	PARAM_OVERWRITE: {
+		SHORT_VERSION: "-o",
+		LONG_VERSION: "--overwrite",
+		DESCRIPTION: "If set, current config file will be overwritten with a new one."
+	},
+	PARAM_KEEP_OUTPUT: {
+		LONG_VERSION: "--keep-output",
+		DESCRIPTION: "If set, output sent through the terminal won't substitute lines, looking more like the log."
+	}
+}