batchfetch 1.0.0__py3-none-any.whl

batchfetch/__init__.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+#
+# Copyright (c) James Cherti
+# URL: https://github.com/jamescherti/batchfetch
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""The entry-point of batchfetch."""
+
+from .batchfetch_cli import command_line_interface
+
+if __name__ == "__main__":
+    command_line_interface()

batchfetch/batchfetch_base.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+#
+# Copyright (c) James Cherti
+# URL: https://github.com/jamescherti/batchfetch
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"Base class used to run tasks in parallel."
+
+import os
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Dict
+
+from schema import Optional, Or, Schema
+
+from .helpers import run_indent_str
+
+
+class BatchFetchError(Exception):
+    """Exception raised by Downloader()."""
+
+
+class GitBranchDoesNotExist(Exception):
+    """Exception raised by Name()."""
+
+
+class BatchFetchBase:
+    """Plugin downloader base class."""
+
+    env = os.environ.copy()
+    env["GIT_TERMINAL_PROMPT"] = "0"
+
+    indent = 4
+
+    def __init__(self, data: Dict[str, Any], options: Dict[str, Any]):
+        # Default
+        self.global_options_schema: Dict[Any, Any] = {
+            Optional("pre_exec"): Or([str], str),
+            Optional("post_exec"): Or([str], str),
+        }
+
+        self.item_schema: Dict[Any, Any] = {
+            Optional("path"): str,
+            Optional("delete"): bool,
+
+            Optional("pre_exec"): Or([str], str),
+            Optional("post_exec"): Or([str], str),
+        }
+
+        self.global_options_values: Dict[str, Any] = {
+            "pre_exec": [],
+            "post_exec": [],
+        }
+
+        self.item_default_values: Dict[str, Any] = {
+            # Optional items
+            "delete": False,
+        }
+
+        self._item_values = data
+        self._item_global_options = options
+
+        # Variables
+        self.values: Dict[str, Any] = {}
+        self.options: Dict[str, Any] = {}
+        self._values_initialized = False
+
+    def _initialize_data(self):
+        if self._values_initialized:
+            return
+
+        # Options
+        self.options = {}
+        self.options.update(deepcopy(self._item_global_options))
+        schema = Schema(self.global_options_schema)
+        schema.validate(self.options)
+
+        # Data
+        self.values = {}
+        self.values.update(deepcopy(self.item_default_values))
+        self.values.update(deepcopy(self._item_values))
+        # self.values.update(deepcopy(self._item_global_options))
+        schema = Schema(self.item_schema)
+        schema.validate(self.values)
+
+        self.values["result"] = {
+            "output": "",
+            "error": False,
+            "changed": False,
+        }
+
+        self._values_initialized = True
+
+        # Strip spaces
+        self.values = {
+            key: value.strip() if isinstance(value, str) else value
+            for key, value in self.values.items()
+        }
+
+    def _run(self, *args, **kwargs):
+        stdout = run_indent_str(*args, **kwargs)
+
+        if not stdout.endswith("\n"):
+            stdout += "\n"
+        self.add_output((" " * self.indent) + stdout)
+
+    def _run_pre_exec(self, cwd: os.PathLike = Path(".")):
+        self._initialize_data()
+        for pre_exec in self["pre_exec"]:
+            if not pre_exec:
+                continue
+
+            self._run(pre_exec, cwd=str(cwd), env=self.env,
+                      spaces=self.indent)
+
+    def _run_post_exec(self, cwd: os.PathLike = Path(".")):
+        self._initialize_data()
+        if self["delete"] or not self.is_changed():
+            return
+
+        for post_exec in self["post_exec"]:
+            if not post_exec:
+                continue
+
+            self._run(post_exec, cwd=str(cwd), env=self.env,
+                      spaces=self.indent)
+
+    def __getitem__(self, key):
+        self._initialize_data()
+
+        if key in self.values:
+            return self.values[key]
+
+        if key in self.options:
+            return self.options[key]
+
+        if key in self.global_options_values:
+            return self.global_options_values[key]
+
+        raise KeyError(f"The item '{key}' was not found in '{self.values}")
+
+    def validate_schema(self):
+        self._initialize_data()
+
+    def is_changed(self) -> bool:
+        self._initialize_data()
+        return bool(self.values["result"]["changed"])
+
+    def set_changed(self, changed: bool):
+        self._initialize_data()
+        self.values["result"]["changed"] = changed
+
+    def get_changed(self) -> bool:
+        try:
+            return bool(self.values["result"]["changed"])
+        except KeyError:
+            return False
+
+    def is_error(self) -> bool:
+        self._initialize_data()
+        return bool(self.values["result"]["error"])
+
+    def set_error(self, error: bool):
+        self._initialize_data()
+        self.values["result"]["error"] = error
+
+    def set_output(self, output: str):
+        self._initialize_data()
+        self.values["result"]["output"] = output
+
+    def add_output(self, output: str):
+        self._initialize_data()
+        self.values["result"]["output"] += output
+
+    def get_output(self) -> str:
+        self._initialize_data()
+        return str(self.values["result"]["output"])
+
+    def update(self):
+        self._initialize_data()
+        return self.values

batchfetch/batchfetch_cli.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+#
+# Copyright (c) James Cherti
+# URL: https://github.com/jamescherti/batchfetch
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""Command line interface."""
+
+import argparse
+import logging
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Set
+
+import colorama
+import yaml  # type: ignore
+from colorama import Fore
+from schema import Optional, Or, Schema, SchemaError
+from setproctitle import setproctitle
+
+from .batchfetch_base import BatchFetchError
+from .batchfetch_git import BatchFetchGit
+
+
+class BatchFetchCli:
+    """Command-line-interface that downloads."""
+
+    main_key = "git"
+
+    # TODO: Make this automatic
+    empty_downloader_git = BatchFetchGit(
+        data={main_key: "http://www.domain.com"},
+        options={},
+    )
+    empty_downloader_git.validate_schema()  # Gen structure
+
+    cfg_schema = {
+        Optional("options"):
+            empty_downloader_git.global_options_schema,  # type: ignore
+        # TODO: Make this added automatically
+        Optional("tasks"): [
+            Or(str, empty_downloader_git.item_schema)
+        ]
+    }
+
+    def __init__(self, max_workers: int, verbose: bool = False):
+        self.cfg: dict = {}
+        self.folder = Path(".")
+        self.managed_filenames: Set[str] = set()
+        self.verbose = verbose
+        self.max_workers = max_workers
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self.dirs_relative_to_batchfetch: Set[str] = set()
+
+    def load(self, path: Path):
+        try:
+            with open(path, "r", encoding="utf-8") as fhandler:
+                yaml_dict = yaml.load(fhandler, Loader=yaml.FullLoader)
+                self.loads(dict(yaml_dict))
+        except OSError as err:
+            raise BatchFetchError(str(err)) from err
+
+    def loads(self, data: dict):
+        schema = Schema(BatchFetchCli.cfg_schema)
+        try:
+            schema.validate(data)
+        except SchemaError as err:
+            print(f"Schema error: {err}.", file=sys.stderr)
+            sys.exit(1)
+
+        self.cfg = {
+            "options": {"clone_args": []},
+            # TODO: Make this added automatically
+            "tasks": [],
+        }
+
+        if "options" in data:
+            self.cfg["options"].update(data["options"])
+
+        # TODO: Make this automatic
+        self._loads_git(data)
+
+    def _loads_git(self, data: dict):
+        if "tasks" not in data:
+            return
+
+        dict_local_dir = {}  # type: ignore
+        for git_repo_raw in data["tasks"]:
+            if isinstance(git_repo_raw, str):
+                git_repo_raw = {BatchFetchCli.main_key: git_repo_raw}
+
+            try:
+                plugin_downloader_git = BatchFetchGit(
+                    data=git_repo_raw,
+                    options=self.cfg["options"],
+                )
+                self.cfg["tasks"].append(plugin_downloader_git)
+            except SchemaError as err:
+                print(f"Schema error: {err}.", file=sys.stderr)
+                sys.exit(1)
+
+            if plugin_downloader_git["path"] in dict_local_dir:
+                err_str = ("more than one repository will be cloned " +
+                           "to the directory '" +
+                           str(plugin_downloader_git["path"]) + "' (" +
+                           str(git_repo_raw[BatchFetchCli.main_key]) + " and " +
+                           str(dict_local_dir[plugin_downloader_git["path"]]) +
+                           ")")
+                raise BatchFetchError(err_str)
+            dict_local_dir[plugin_downloader_git["path"]] = \
+                plugin_downloader_git[BatchFetchCli.main_key]
+
+    def download(self) -> bool:
+        failed = []
+        error = False
+        threads = []
+        num_success = 0
+        self.managed_filenames = set()
+
+        executor_update = ThreadPoolExecutor(max_workers=self.max_workers)
+
+        try:
+            self.dirs_relative_to_batchfetch = set()
+
+            # TODO: Make adding to all downloads automatic
+            all_downloads = self.cfg["tasks"]
+            for download_item in all_downloads:
+                self.dirs_relative_to_batchfetch.add(str(download_item["path"]))
+                if not download_item["delete"]:
+                    self.managed_filenames.add(download_item["path"])
+                threads.append(executor_update.submit(download_item.update))
+
+            for future in as_completed(threads):
+                data = future.result()
+                if data["result"]["error"]:
+                    print(Fore.RED, end="")
+                elif data["result"]["changed"]:
+                    print(Fore.YELLOW, end="")
+                else:
+                    if not self.verbose:
+                        continue
+
+                    print(Fore.GREEN, end="")
+
+                if data["result"]["error"]:
+                    error = True
+                    failed.append(data)
+                else:
+                    num_success += 1
+
+                if data["result"]["output"]:
+                    print(data["result"]["output"].rstrip("\n"))
+
+                print(Fore.RESET, end="")
+                print()
+        except KeyboardInterrupt:
+            error = True
+            executor_update.shutdown(cancel_futures=True)  # type: ignore
+
+        if error:
+            if failed:
+                print("Failed:")
+                for git_update_result in failed:
+                    print("  - url:", git_update_result[BatchFetchCli.main_key])
+                    print("    dir:", git_update_result["path"])
+            else:
+                print("Failed.")
+
+            return False
+        else:
+            if num_success == 0:
+                print("Already up to date.")
+            else:
+                print("Success.")
+
+        return True
+
+    # def check_managed_directories(self):
+    #     # managed_downloads = {Path(item).resolve()
+    #     #                      for item in self.managed_filenames}
+    #     # managed_directories = {item.parent
+    #     #                        for item in managed_downloads}
+    #     # TODO Implement checker
+    #     pass
+
+
+def parse_args():
+    """Parse the command line arguments."""
+    desc = __doc__
+    usage = "%(prog)s [--option] [args]"
+    parser = argparse.ArgumentParser(description=desc,
+                                     usage=usage)
+    parser.add_argument(
+        "-p", "--max-procs", default="3", required=False,
+        help="Run up to N Number of parallel git fetch processes.",
+    )
+
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", default=False,
+        help="Verbose mode.",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def command_line_interface():
+    """Command line interface."""
+    errno = 0
+    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
+                        format="%(asctime)s %(name)s: %(message)s")
+
+    colorama.init()
+    setproctitle(subprocess.list2cmdline(
+        [Path(sys.argv[0]).name] + sys.argv[1:]
+    ))
+
+    args = parse_args()
+
+    path_batchfetch_yaml = "batchfetch.yaml"
+    downloader_cli = BatchFetchCli(verbose=args.verbose,
+                                   max_workers=int(args.max_procs))
+    downloader_cli.load(path_batchfetch_yaml)
+    try:
+        if not downloader_cli.download():
+            errno = 1
+    except KeyboardInterrupt:
+        print("Interrupted.", file=sys.stderr)
+        errno = 1
+    except BatchFetchError as err:
+        print(f"Error: {err}.", file=sys.stderr)
+        errno = 1
+    # else:
+    #     downloader_cli.check_managed_directories()
+
+    sys.exit(errno)