darkskysync 0.4.0__py3-none-any.whl

darkskysync/AbstractRemoteSyncAdapter.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2013 ETH Zurich, Institute of Astronomy
+
+"""
+Created on Sep 24, 2013
+
+@author: J. Akeret
+"""
+
+
+import os
+from abc import ABCMeta, abstractmethod
+
+from darkskysync.Exceptions import FileSystemError
+
+"""
+Created on Sep 30, 2013
+
+@author: J. Akeret
+"""
+
+
+class AbstractRemoteSyncAdapter:
+    """
+    Abstract class for all synchronization adapters
+    """
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def loadFiles(self, files):
+        """
+        Loads files from the remote location
+        """
+        pass
+
+    def _prepareDestination(self, path):
+        if not os.path.isdir(path):
+            # We do not create *all* the parents, but we do create the
+            # directory if we can.
+            try:
+                os.makedirs(path)
+            except OSError as ex:
+                raise FileSystemError(
+                    "Unable to create storage directory: " "%s" % (str(ex))
+                )

darkskysync/DarkSkySync.py

@@ -0,0 +1,251 @@
+# Copyright (C) 2013 ETH Zurich, Institute of Astronomy
+
+"""
+Created on Sep 24, 2013
+
+@author: J. Akeret
+"""
+
+
+import importlib.resources
+import os
+import shutil
+import sys
+
+import darkskysync
+
+from .DataSourceFactory import DataSourceFactory
+from .Exceptions import (
+    ConfigurationError,
+    ConnectionError,
+    FileSystemError,
+    IllegalArgumentException,
+)
+from .LocalSyncAdapter import LocalSyncAdapter
+from .RsyncWrapper import RsyncWrapper
+from .SSHClientAdapter import SSHClientAdapter
+
+
+class DarkSkySync:
+    """
+    Helps to access and synchronize with a remote data storage. Features are:
+        - lists the available file structure on the remote host
+        - lists the file structure in the local cache
+        - synchronizes the local cache with the remote file structure
+        - allows for removing files from the local file system
+
+    It delegates the remote access to the according wrappers (currently either scp over
+    ssh or rsync over ssh) and manages the local file structure
+
+    :param template: [optional] alternate name for the config template to use
+    :param configFile: [optional] alternate config file to use
+    :param verbose: [optional] flag to enable verbose mode
+
+    :raises ConfigurationError: error while reading the config file
+    """
+
+    REMOTE_WRAPPER_TYPE_MAP = {"rsync": SSHClientAdapter, "ssh": SSHClientAdapter}
+
+    REMOTE_SYNC_WRAPPER_TYPE_MAP = {
+        "rsync": RsyncWrapper,
+        "ssh": SSHClientAdapter,
+        "local": LocalSyncAdapter,
+    }
+
+    DEFAULT_TEMPLATE = "master"
+
+    def __init__(self, template=DEFAULT_TEMPLATE, configFile=None, verbose=False):
+        """
+        Constructor
+        """
+        self.configFile = configFile
+
+        if template is None:
+            template = DarkSkySync.DEFAULT_TEMPLATE
+
+        self._verifyConfigFile()
+
+        self.verbose = verbose
+        if self.verbose:
+            darkskysync.logger.info("Using config file: %s" % self.configFile)
+
+        dsFactory = DataSourceFactory.fromConfig(self.configFile)
+        self.dataSourceConfig = dsFactory.createDataSource(template)
+
+    def avail(self, path=None):
+        """
+        Lists the available file structure on the remote host
+
+        :param path: [optional] sub path to directory structure to check
+
+        :return: list of file available on remote host
+
+        :raises ConnectionError: remote host could not be accessed
+        """
+
+        try:
+            RemoteWrapper = DarkSkySync.REMOTE_WRAPPER_TYPE_MAP[
+                self.dataSourceConfig.remote.type
+            ]
+
+            remoteWrapper = RemoteWrapper(self.dataSourceConfig)
+            fileList = remoteWrapper.getRemoteFilesList(path)
+            return self._filterFileList(fileList)
+        except ConnectionError as ex:
+            darkskysync.logger.error(str(ex))
+            raise ex
+
+    def list(self, path=None, recursive=False):
+        """
+        Lists the available files in the local cache
+
+        :param path: [optional] sub path to directory structure to check
+        :param recursive: [optional] flag indicating if the cache should be browsed
+                         recursively
+
+        :return: list of files in the local cache
+
+        """
+
+        filePath = self.dataSourceConfig.local.filePath
+        if path is not None:
+            filePath = os.path.join(filePath, path)
+
+        fileList = []
+
+        if not os.path.exists(filePath):
+            return fileList
+
+        if recursive:
+            for dirpath, dirnames, filenames in os.walk(filePath):
+                for fileName in filenames:
+                    fileList.append(os.path.join(dirpath, fileName))
+        else:
+            pathContent = os.listdir(filePath)
+            fileList = [os.path.join(filePath, file) for file in pathContent]
+
+        return fileList
+
+    def load(self, names, dry_run=False, force=False):
+        """
+        Loads the given file names. Checks first if the requested files are available on
+        the file system.  If not, they will be downloaded from the remote host.
+
+        :param names: a list of names which should be loaded
+        :param dry_run: [optional] flag indicating a dry run. No files will be
+                        downloaded
+        :param force: [optional] flag indicating that the files should be downloaded
+                      even if they already exist on the local file system
+
+        :retun: list of files loaded from the remote host
+
+        :raises ConnectionError: remote host could not be accessed
+        :raises FileSystemError: error while accesing the local file system
+
+        """
+
+        try:
+            RemoteWrapper = DarkSkySync.REMOTE_SYNC_WRAPPER_TYPE_MAP[
+                self.dataSourceConfig.remote.type
+            ]
+
+            remoteWrapper = RemoteWrapper(
+                self.dataSourceConfig,
+                dry_run=dry_run,
+                force=force,
+                verbose=self.verbose,
+            )
+            loadedFiles = remoteWrapper.loadFiles(names)
+            return self._postProcessFileList(loadedFiles)
+        except ConnectionError as ex:
+            darkskysync.logger.error(str(ex))
+            raise ex
+        except FileSystemError as ex:
+            darkskysync.logger.error(str(ex))
+            raise ex
+
+    def remove(self, files=None, allFiles=False):
+        """
+        Removes files and filetrees from the local cache
+
+        :param files: [optional] list of files or directories to delete
+        :param allFiles: [optional] flag indicating that all files shoudl be removed
+                         from the cache
+
+        :raises FileSystemError: error while accessing the local file system
+        :raises IllegalArgumentException: Illegal parameter combination
+
+        """
+        if files is None and not allFiles:
+            darkskysync.logger.warning("At least a file has to be given")
+            raise IllegalArgumentException("At least a file has to be given")
+
+        if files is not None:
+            if isinstance(files, str):
+                files = [files]
+            for file in files:
+                path = os.path.join(self.dataSourceConfig.local.filePath, file)
+                if self.verbose:
+                    darkskysync.logger.info("Removing: %s" % path)
+
+                if os.path.exists(path):
+                    if os.path.isfile(path):
+                        os.remove(path)
+                    else:
+                        shutil.rmtree(path)
+
+        if allFiles:
+            path = self.dataSourceConfig.local.filePath
+            if self.verbose:
+                darkskysync.logger.info("Removing content of: %s" % path)
+            for file in os.listdir(path):
+                filePath = os.path.join(path, file)
+                try:
+                    if os.path.isfile(filePath):
+                        os.unlink(filePath)
+                    else:
+                        shutil.rmtree(filePath)
+                except Exception as e:
+                    darkskysync.logger.warn(e)
+                    raise FileSystemError(e)
+
+    def _verifyConfigFile(self):
+        if self.configFile is None:
+            self.configFile = DataSourceFactory.DEFAULT_CONFIGURATION_FILE
+            if not os.path.isfile(self.configFile):
+                if not os.path.exists(os.path.dirname(self.configFile)):
+                    os.mkdir(os.path.dirname(self.configFile))
+
+                with importlib.resources.as_file(
+                    importlib.resources.files(__package__).joinpath(
+                        "data/config.template"
+                    )
+                ) as template:
+                    darkskysync.logger.warning(
+                        "Deploying default configuration file to %s.", self.configFile
+                    )
+                    shutil.copyfile(template, self.configFile)
+
+        else:
+            # Exit if supplied configuration file does not exists.
+            if not os.path.isfile(self.configFile):
+                sys.stderr.write(
+                    "Unable to read configuration file `%s`.\n" % self.configFile
+                )
+                raise ConfigurationError(
+                    "Unable to read configuration file `%s`.\n" % self.configFile
+                )
+
+    def _filterFileList(self, fileList):
+        fileList = [f for f in fileList if not f.startswith(".")]
+        fileList.sort()
+        return fileList
+
+    def _postProcessFileList(self, fileList):
+        files = []
+        for filePath in fileList:
+            if os.path.exists(filePath):
+                if not os.path.isfile(filePath):
+                    filePath = filePath + "/"
+                files.append(filePath)
+        return files

darkskysync/DarkSkySyncCLI.py

@@ -0,0 +1,108 @@
+#! /usr/bin/env python
+# Copyright (C) 2013 ETH Zurich, Institute of Astronomy
+
+"""
+This is the DarkSkySync command line interface
+
+Created on Sep 23, 2013
+
+@author: J. Akeret
+
+
+Usage:
+  DarkSkySync avail [<path>] [--config=<file>] [--template=<template>] [-v|--verbose]
+  DarkSkySync list [<path>] [-r|--recursive] [--config=<file>] [--template=<template>] [-v|--verbose]
+  DarkSkySync load <name>... [--dry_run] [-f|--force] [--config=<file>] [--template=<template>] [-v|--verbose]
+  DarkSkySync remove (<name>...|--all) [--config=<file>] [--template=<template>] [-v|--verbose]
+  DarkSkySync -h | --help
+  DarkSkySync --version
+
+Options
+  -h --help     Show this screen.
+  --version     Show version.
+  --config=<file>  The configfile to use.
+  --template=<template>  The template to use [default:master]
+  --all         All files
+  -v --verbose  More output
+  -f --force    Force the download
+  -r --recursive  List the directories in a recursive way
+  --dry_run     Dry run - Not loading any files
+"""
+
+import sys
+
+from . import __version__
+from .DarkSkySync import DarkSkySync
+from .docopt import docopt
+from .Exceptions import ConfigurationError
+
+
+class DarkSkySyncCLI:
+    """
+    Command line interface for the darkskysync
+    """
+
+    def __init__(self):
+        pass
+
+    def launch(self):
+        """
+        Starts the darkskysync by parsing the args and the config
+        """
+
+        args = docopt(__doc__, version="DarkSkySync CLI {}".format(__version__))
+
+        #        print args
+        try:
+            dam = DarkSkySync(
+                template=args["--template"],
+                configFile=args["--config"],
+                verbose=args["--verbose"],
+            )
+            self.dispatch(dam, args)
+        except ConfigurationError:
+            sys.exit(1)
+
+    def dispatch(self, dam, args):
+        """
+        Dispatches the sub command to the given darkskysync instance
+
+        :param dam: the instance of the darkskysync to us
+        :param args: the arguments passed by the used
+        """
+        try:
+            if args["avail"]:
+                filesList = dam.avail(path=args["<path>"])
+                self._printFilesList("Available files:", filesList)
+
+            elif args["list"]:
+                filesList = dam.list(path=args["<path>"], recursive=args["--recursive"])
+                self._printFilesList("Available files:", filesList)
+
+            elif args["load"]:
+                filesList = dam.load(args["<name>"], args["--dry_run"], args["--force"])
+                self._printFilesList("Loaded files:", filesList)
+
+            elif args["remove"]:
+                dam.remove(args["<name>"], args["--all"])
+
+        except Exception:
+            sys.exit(1)
+
+    def _printFilesList(self, message, filesList):
+        print(message)
+        if len(filesList) > 0:
+            for fileName in filesList:
+                print(fileName)
+
+        else:
+            print("-")
+
+
+def main():
+    cli = DarkSkySyncCLI()
+    cli.launch()
+
+
+if __name__ == "__main__":
+    main()

darkskysync/DataSourceFactory.py

@@ -0,0 +1,268 @@
+# Copyright (C) 2013 ETH Zurich, Institute of Astronomy
+
+"""
+Created on Sep 23, 2013
+
+@author: J. Akeret
+"""
+
+import os
+import re
+
+from dataclasses import dataclass
+from configobj import ConfigObj
+from voluptuous import All, Invalid, Length, Schema, Url, message
+
+from .Exceptions import ConfigurationError
+
+# from .LocalFileSystem import LocalFileSystem
+from .RemoteFactory import SSHRemoteLocationFactory
+
+
+@dataclass
+class DataSource:
+    name: str
+    local: str
+    remote: str
+
+
+@dataclass
+class LocalFileSystem:
+    name: str
+    filePath: str
+
+
+class DataSourceFactory:
+    """
+    A factory creating DataSource configurations
+    """
+
+    DEFAULT_CONFIGURATION_FILE = os.path.expanduser("~/.darkskysync/config")
+
+    REMOTE_FACTORY_TYPE_MAP = {
+        "rsync": SSHRemoteLocationFactory,
+        "ssh": SSHRemoteLocationFactory,
+        "local": SSHRemoteLocationFactory,
+    }
+
+    def __init__(self, config):
+        """
+        Constructor
+        """
+        self.config = config
+        validator = ConfigValidator(self.config)
+        validator.validate()
+
+    @classmethod
+    def fromConfig(cls, configfile):
+        """
+        Helper method to initialize DataSourceFactory from an ini file.
+        :param configfile: path to the ini file
+        :return: configurator object
+        """
+        config_reader = ConfigReader(configfile)
+        conf = config_reader.read_config()
+        return DataSourceFactory(conf)
+
+    def createDataSource(self, template, name=None):
+        """
+        Creates a data source by inspecting the configuration properties of the
+            given data source template.
+        :param template: name of the data source template
+
+        :param name: name of the data source. If not defined, the data source
+        will be named after the template.
+
+        :return: :py:class:`DataSource` instance
+
+        :raises ConfigurationError: data source template not found in config
+        """
+        if not name:
+            name = template
+
+        if template not in self.config:
+            raise ConfigurationError(
+                "Invalid configuration for data source `%s`: %s" "" % (template, name)
+            )
+
+        conf = self.config[template]
+
+        extra = conf["datasource"].copy()
+        extra.pop("local")
+        extra.pop("remote")
+
+        return DataSource(
+            template, self.createLocal(template), self.createRemote(template)
+        )
+
+    def createLocal(self, template):
+        conf = self.config[template]
+
+        return LocalFileSystem(conf["datasource"]["local"], conf["local"]["path"])
+
+    def createRemote(self, template):
+        conf = self.config[template]
+        remoteType = conf["remote"]["type"]
+
+        RemoteFactory = DataSourceFactory.REMOTE_FACTORY_TYPE_MAP[remoteType]
+
+        factory = RemoteFactory()
+
+        return factory.create(conf["datasource"]["remote"], conf)
+
+
+class ConfigValidator:
+    def __init__(self, config):
+        self.config = config
+
+    def _pre_validate(self):
+        """
+        Handles all pre validation phase functionality, such as:
+        - reading environment variables
+        - interpolating configuraiton options
+        """
+
+    def _post_validate(self):
+        """
+        Handles all post validation phase functionality, such as:
+        - expanding file paths
+        """
+        # expand all paths
+        for dataSource, values in self.config.items():
+            conf = self.config[dataSource]
+
+            privkey = os.path.expanduser(values["login"]["known_hosts"])
+            conf["login"]["known_hosts"] = privkey
+
+            privkey = os.path.expanduser(values["login"]["user_key_private"])
+            conf["login"]["user_key_private"] = privkey
+
+            pubkey = os.path.expanduser(values["login"]["user_key_public"])
+            conf["login"]["user_key_public"] = pubkey
+
+            path = os.path.expanduser(values["local"]["path"])
+            conf["local"]["path"] = path
+
+    def validate(self):
+        """
+        Validates the given configuration :py:attr:`self.config` to comply.
+        As well all types are converted to the expected
+        format if possible.
+        """
+        self._pre_validate()
+
+        # custom validators
+        @message("file could not be found")
+        def check_file(v):
+            f = os.path.expanduser(os.path.expanduser(v))
+            if os.path.exists(f):
+                return f
+            else:
+                raise Invalid("file could not be found `%s`" % v)
+
+        # schema to validate all dataSource properties
+        schema = {
+            "datasource": {
+                "local": All(str, Length(min=1)),
+                "remote": All(str, Length(min=1)),
+            },
+            "local": {
+                "path": All(str, Length(min=1)),  # check_file(),
+            },
+            "login": {
+                "name": All(str, Length(min=1)),
+                "known_hosts": check_file(),
+                "user_key_private": check_file(),
+                "user_key_public": check_file(),
+            },
+        }
+
+        remote_schema_ssh = {
+            "type": "ssh",
+            "host": Url(str),
+            "port": All(str, Length(min=1)),
+            "path": All(str, Length(min=0)),
+            "login": All(str, Length(min=1)),
+        }
+
+        # validation
+        validator = Schema(schema, required=True, extra=True)
+        ssh_validator = Schema(remote_schema_ssh, required=True, extra=False)
+
+        if not self.config:
+            raise Invalid("No data sources found in configuration.")
+
+        for dataSource, properties in self.config.items():
+            validator(properties)
+
+            if properties["remote"]["type"] == "ssh":
+                ssh_validator(properties["remote"])
+
+        self._post_validate()
+
+
+class ConfigReader:
+    """
+    Reads the configuration properties from a ini file.
+    """
+
+    local_section = "local"
+    remote_section = "remote"
+    login_section = "login"
+    datasource_section = "datasource"
+
+    def __init__(self, configfile):
+        """
+        :param configfile: path to configfile
+        """
+        self.configfile = configfile
+        self.conf = ConfigObj(self.configfile, interpolation=False)
+
+    def read_config(self):
+        """
+        Reads the configuration properties from the ini file and links the
+        section to comply with the datasource config dictionary format.
+        :return: dictionary containing all configuration properties from the
+         ini file in compliance to the datasource config format
+        :raises MultipleInvalid: not all sections present or broken links
+            between secitons
+        """
+        datasources = {
+            key: value
+            for key, value in self.conf.items()
+            if (
+                re.search(ConfigReader.datasource_section + "/(.*)", key)
+                and key.count("/") == 1
+            )
+        }
+
+        conf_values = dict()
+
+        for datasource in datasources:
+            name = re.search(
+                ConfigReader.datasource_section + "/(.*)", datasource
+            ).groups()[0]
+            try:
+                datasource_conf = dict(self.conf[datasource])
+                local_name = ConfigReader.local_section + "/" + datasource_conf["local"]
+                remote_name = (
+                    ConfigReader.remote_section + "/" + datasource_conf["remote"]
+                )
+
+                values = dict()
+                values["datasource"] = datasource_conf
+                values["local"] = dict(self.conf[local_name])
+                values["remote"] = remote_conf = dict(self.conf[remote_name])
+
+                login_name = ConfigReader.login_section + "/" + remote_conf["login"]
+
+                values["login"] = dict(self.conf[login_name])
+
+                conf_values[name] = values
+            except KeyError:
+                raise Exception(
+                    "could not find all sections required `datasource`, "
+                    "`setup`, `login`, `cloud` for datasource `%s`" % name
+                )
+
+        return conf_values

darkskysync/Exceptions.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2013 ETH Zurich, Institute of Astronomy
+
+"""
+Created on Sep 23, 2013
+
+@author: J. Akeret
+"""
+
+
+class ConfigurationError(Exception):
+    pass
+
+
+class ConnectionError(Exception):
+    pass
+
+
+class FileSystemError(Exception):
+    pass
+
+
+class IllegalArgumentException(Exception):
+    pass
+
+
+class FileNotFoundError(Exception):
+    pass