gOdoo-cli 0.7.0__py3-none-any.whl

godoo_cli/git/git_odoo_addons.py

@@ -0,0 +1,95 @@
+import concurrent.futures
+import logging
+from pathlib import Path
+from typing import Dict
+
+from git import Commit
+
+from ..helpers.odoo_manifest import update_yml, yaml_roundtrip_loader
+from .git_repo import git_ensure_repo
+from .git_url import GitUrl
+
+LOGGER = logging.getLogger(__name__)
+
+
+def git_ensure_addon_repos(
+    root_folder: Path,
+    git_yml_path: Path,
+    generate_yml_compare_comments: bool = False,
+    download_archive: bool = False,
+):
+    """
+    Clone repos specified in Yml.
+
+    Parameters
+    ----------
+    root_folder : Path
+        clone target folder
+    git_yml_path : Path
+        yml describing what to clone
+    generate_yml_compare_comments : bool, optional
+        wether to add three dot compare on remote to repo urls
+    download_archive : bool, optional
+        wether to download as .zip (fast but no history), by default False
+    """
+    yaml = yaml_roundtrip_loader()
+    git_repos = yaml.load(git_yml_path.resolve())
+    if _git_clone_addon_repos(root_folder=root_folder, git_repos=git_repos, download_archive=download_archive):
+        update_yml(git_repos, generate_yml_compare_comments)
+        LOGGER.info("Updating Git Thirdparty Repo Commit hashes")
+        yaml.dump(git_repos, git_yml_path)
+
+    return git_repos
+
+
+def _git_clone_addon_repos(
+    root_folder: Path,
+    git_repos: Dict[str, Dict[str, str]],
+    download_archive: bool = False,
+) -> Dict[str, Commit]:
+    """
+    Clones Git repos specified in dict into Root folder.
+    Ensures repo names are prefixed and uses 8 threads to clone.
+
+    Parameters
+    ----------
+    root_folder : Path
+        clone target folder
+    git_repos : Dict[str, Dict[str, str]]
+        dict of {parent_folder_name:{"url":clone_url,"branch":clone_branch,"commit":specific_commit_to_clone}}
+        branch and commit are optional
+        branch defaults to odoo branch from spec file
+    download_archive : bool, optional
+        wether to download as .zip (fast but no history), by default False
+    Returns
+    -------
+    Dict[str:Commit]
+        dict of {git_src_url:HeadCommit}
+    """
+    default_branch = git_repos["odoo"].get("branch")
+    LOGGER.info("Cloning Thirdparty Addons source.")
+    with concurrent.futures.ThreadPoolExecutor(8) as executor:
+        futures = []
+        thirdparty_repos = git_repos.get("thirdparty")
+        if not thirdparty_repos:
+            LOGGER.info("No Thirdparty Key in manifest. Skipping...")
+            return
+        for prefix, repos in thirdparty_repos.items():
+            for repo in repos:
+                repo_url = GitUrl(repo["url"])
+                name = f"{prefix}_{repo_url.name}"
+                futures.append(
+                    executor.submit(
+                        git_ensure_repo,
+                        target_folder=Path(root_folder / name),
+                        repo_src=repo_url.url,
+                        branch=repo.get("branch", default_branch),
+                        commit=repo.get("commit"),
+                        zip_mode=download_archive,
+                        filter="blob:none",
+                        single_branch=True,
+                    )
+                )
+        clone_results = [f.result() for f in futures]
+        clone_results = {r[0]: r[1] for r in clone_results if r}
+    return clone_results

godoo_cli/git/git_repo.py

@@ -0,0 +1,207 @@
+""" Module to provide GIT interaction."""
+import logging
+import shutil
+from pathlib import Path
+
+from git import GitCommandError, InvalidGitRepositoryError, Repo
+
+from .git_url import GitUrl
+from .zip_download import git_download_zip
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _git_clean_clone(repo_src: str, target_folder: Path, **kwargs):
+    """Clears targetfolder and does a clean clone_from
+
+    Parameters
+    ----------
+    repo_src : str
+        Clone url
+    target_folder : Path
+        Download folder
+    branch : str
+        branch to clone
+
+    Returns
+    -------
+    _type_re
+        _description_
+    """
+    LOGGER.debug("Cloning Repo: %s, to '%s', Kwargs: '%s'", repo_src, target_folder, kwargs)
+    if not isinstance(target_folder, Path):
+        target_folder = Path(target_folder)
+    if target_folder.exists():
+        LOGGER.debug("Clearing Repo folder: %s", target_folder)
+        shutil.rmtree(target_folder)
+    return Repo.clone_from(repo_src, target_folder, **kwargs)
+
+
+def git_pull_checkout_reset(
+    repo: Repo, branch: str = "master", commit: str = "", pull: str = "", reset_hard: bool = True
+):
+    """
+    Ensure git Repo is on specific branch and commit.
+    Will unshallow if neccessary
+
+    Parameters
+    ----------
+    repo : Repo
+        Git Repo
+    branch : str, optional
+        branch to checkout, by default "master"
+    commit : str, optional
+        specific commitSHA to checkout, by default ""
+    pull : str, optional
+        specific target to pull. Usually used in conjunction with branch.
+    reset_hard : bool, optional
+        will call reset --hard before checkout to ensure clean git repo, by default True
+    """
+
+    if reset_hard:
+        repo.git.reset("--hard", "HEAD")
+
+    if pull:
+        LOGGER.debug("Pulling Repo: %s, %s", repo.working_dir, pull)
+        try:
+            repo.remotes[0].pull(pull)
+        except GitCommandError as e:
+            if "fatal: refusing to merge unrelated histories" in e.stderr:
+                clone_kwargs = {}
+                if branch:
+                    clone_kwargs["branch"] = branch
+                repo = _git_clean_clone(repo.remotes[0].url, repo.working_dir, **clone_kwargs)
+            else:
+                raise e
+    if commit:
+        if str(repo.head.commit) != str(commit):
+            LOGGER.debug("Checking out %s to Commit: %s", repo.git_dir, commit)
+            repo.git.checkout(commit)
+        return
+    if branch:
+        LOGGER.debug("Checking Out repo %s to Branch: %s", repo.git_dir, branch)
+        repo.git.checkout(branch)
+
+
+def git_ensure_ref(
+    target_folder: Path,
+    repo_src: str,
+    branch: str = "master",
+    commit: str = "",
+    pull: str = "",
+    **kwargs,
+):
+    """
+    Clone a git Repo and ensure its HEAD is set to Branch and commit.
+
+
+    Parameters
+    ----------
+    target_folder : Path
+        "target folder name"
+    repo_src : str
+        "repo source url"
+    commit : str, optional
+        commit on which to set the head
+    pull : str, optional
+        specific target to pull. Usually used in conjunction with branch.
+    branch : str, optional
+        branch on which to set head, by default 'master'
+    **kwargs
+        get passed to git clone
+    """
+    LOGGER.info("Ensuring Repo '%s' --> '%s'", repo_src, target_folder)
+    target_folder.mkdir(exist_ok=True, parents=True)
+    try:
+        repo = Repo(target_folder)
+        current = str(repo.head.commit)
+
+        if not pull and commit and str(commit) != str(current):
+            pull = str(commit)
+
+        if not pull and branch and not commit:
+            remote_name = str(repo.remotes[0].name)
+            try:
+                remote_commit = repo.git.rev_parse(remote_name + "/" + str(branch))
+            except Exception:
+                remote_commit = "unknown"
+            LOGGER.debug(
+                "Repo: %s comparing local head '%s' with remote head '%s'", repo.working_dir, current, remote_commit
+            )
+            if str(remote_commit) != current:
+                pull = str(branch)
+
+        git_pull_checkout_reset(repo=repo, branch=branch, commit=commit, pull=pull)
+
+        if current == str(repo.head.commit):
+            LOGGER.info("Repo Commit matches. Skipping: '%s' --> '%s'", repo_src, target_folder)
+        else:
+            LOGGER.info("Pulled Repo from: '%s'. Head is now at: %s", repo_src, repo.head.commit)
+    except InvalidGitRepositoryError:
+        repo = _git_clean_clone(repo_src, target_folder, branch=branch, **kwargs)
+        git_pull_checkout_reset(repo=repo, branch=branch, commit=commit, pull=False)
+
+        LOGGER.info(
+            "Cloned Repo: '%s'. Branch='%s' Head='%s'",
+            repo_src,
+            repo.active_branch.name if not repo.head.is_detached else "Detached",
+            repo.head.commit,
+        )
+    return repo_src, repo.head.commit
+
+
+def git_ensure_repo(
+    target_folder: Path,
+    repo_src: str,
+    branch: str = "master",
+    commit: str = "",
+    pull: str = "",
+    zip_mode: bool = False,
+    **kwargs,
+):
+    """Ensures git remote contents are in folder.
+
+    Parameters
+    ----------
+    target_folder : Path
+        Folder to clone/download into
+    repo_src : str
+        Git Remote Url
+    branch : str, optional
+        target branch, by default "master"
+    commit : str, optional
+        target ref, by default ""
+    pull : str, optional
+        ref to pull, by default ""
+    zip_mode : bool, optional
+        wether to download a zip (fast but  no commit history) or clone and fetch (fully working git clone), by default False
+    **kwargs
+        get passed to git clone
+    """
+
+    if isinstance(target_folder, str):
+        target_folder = Path(target_folder)
+
+    if target_folder.exists() and any(target_folder.iterdir()) and not target_folder.glob(".git"):
+        LOGGER.info("Assuming '%s' got pulled in .Zip mode. Skipping Clone and commit check.", target_folder)
+        zip_mode = True
+
+    if zip_mode and GitUrl(repo_src).url_type == "ssh":
+        LOGGER.info("Zip downloading currently not supported for SSH type Urls")
+        zip_mode = False
+
+    if zip_mode:
+        return git_download_zip(
+            target_folder=target_folder,
+            repo_url=repo_src,
+            branch=branch,
+            commit=commit,
+        )
+    return git_ensure_ref(
+        target_folder=target_folder,
+        repo_src=repo_src,
+        branch=branch,
+        commit=commit,
+        pull=pull,
+        **kwargs,
+    )

godoo_cli/git/git_url.py

@@ -0,0 +1,152 @@
+import re
+from enum import Enum
+from typing import Literal
+
+
+class GitRemoteType(Enum):
+    gitlab = "gitlab"
+    github = "github"
+
+
+class GitUrl:
+    """Class to Structurize Git URLs.
+    Works with SSH and HTTP(s) URLs
+    Can generate Compare Urls
+    """
+
+    url: str
+    url_type: Literal["http", "ssh"]
+    domain: str
+    path: str
+    user: str
+    port: int
+    name: str
+
+    def __init__(self, url: str) -> None:
+        self.url = url
+        if "http" in url:
+            http_regex = r"(?P<schema>https?):\/\/(?P<domain>[^\/]+)(?P<path>.*)"
+            http_match = re.search(http_regex, url)
+            self.url_type = http_match.group("schema")
+            self.domain = http_match.group("domain")
+            self.path = http_match.group("path")
+        else:
+            ssh_regex = r"(?P<user>\w+)@(?P<domain>[^:]+):(?:(?P<port>\d+)]?:)?(?P<path>.*)"
+            ssh_match = re.search(ssh_regex, url)
+            self.url_type = "ssh"
+            self.domain = ssh_match.group("domain")
+            self.path = ssh_match.group("path")
+            self.user = ssh_match.group("user")
+            self.port = ssh_match.group("port")
+
+        if self.path.endswith(".git"):  # Todo Python 3.9 Use Removesuffix and RemovePrefix
+            self.path = self.path[:-4]
+        if self.path.endswith("/"):
+            self.path = self.path[:-1]
+        if self.path.startswith("/"):
+            self.path = self.path[1:]
+        self.name = self.path.split("/")[-1]
+
+    def _clean_http_url(self) -> str:
+        """Return HTTPs URL without .git suffix.
+
+        Returns
+        -------
+        str
+            Https:// url
+        """
+        return f"https://{self.domain}/{self.path}"
+
+    def _git_type(self) -> GitRemoteType:
+        """Get git Remote type.
+
+        Returns
+        -------
+        Literal
+            "github", "gitlab"
+
+        Raises
+        ------
+        ValueError
+            If Type cannot be determined.
+        """
+        if "gitlab" in self.domain:
+            return GitRemoteType.gitlab
+        elif "github" in self.domain:
+            return GitRemoteType.github
+        else:
+            raise ValueError(f"Cant get Git Service type from {self.domain}")
+
+    def get_compare_url(self, from_compare: str, to_compare: str) -> str:
+        """Get Compare url between two Refs.
+
+        Parameters
+        ----------
+        from_compare : str
+            from compare ref
+        to_compare : str
+            to compare ref
+
+        Returns
+        -------
+        str
+            Compare Url Like: https://github.com/odoo/odoo/compare/<commitSHA>...<branchName>
+        """
+        remote_type = self._git_type()
+        if from_compare == to_compare:
+            return  # Nothing to Compare here
+        http_url = self._clean_http_url()
+        if remote_type in [GitRemoteType.github, GitRemoteType.gitlab]:
+            return f"{http_url}/compare/{from_compare}...{to_compare}"
+
+    def get_archive_url(self, ref: str) -> str:
+        """Get Download Url for Zip file.
+
+        Parameters
+        ----------
+        branch : str, optional
+            Repo Branch, by default ""
+        commit : str, optional
+            Repo download ref, by default ""
+
+        Returns
+        -------
+        str
+            Url from which to download a zip file
+
+        Raises
+        ------
+        ValueError
+            if ref is not specified
+        """
+        if not ref:
+            raise ValueError("Missing either download ref (e.g. branch or commit) to generate Archive URL.")
+        http_url = self._clean_http_url()
+        remote_type = self._git_type()
+        if remote_type == GitRemoteType.github:
+            return f"{http_url}/archive/{ref }.zip"
+        if remote_type == GitRemoteType.gitlab:
+            return f"{http_url}/-/archive/{ref}/{self.name}.zip"
+
+    def get_file_raw_url(self, ref: str, file_path: str) -> str:
+        """Gets the URL Pointing to the Raw file contents on the Remote.
+
+        Parameters
+        ----------
+        ref : str
+            Branch, Commit, Tag ,...
+        file_path : str
+            Relative file path in Repository
+
+        Returns
+        -------
+        str
+            URL Pointing to the Raw file contents on the Remote
+        """
+
+        http_url = self._clean_http_url()
+        remote_type = self._git_type()
+        if remote_type == GitRemoteType.github:
+            return f"{http_url.replace(self.domain,'raw.githubusercontent.com')}/{ref}/{file_path}"
+        if remote_type == GitRemoteType.gitlab:
+            return f"{http_url}/-/raw/{ref}/{file_path}"

godoo_cli/git/zip_download.py

@@ -0,0 +1,49 @@
+import logging
+import shutil
+import tempfile
+import zipfile
+from pathlib import Path
+
+from ..helpers.system import download_file
+from .git_url import GitUrl
+
+LOGGER = logging.getLogger(__name__)
+
+
+def git_download_zip(repo_url: str, target_folder: Path, branch: str, commit: str = ""):
+    """Download Repo Zip from Github.
+
+    Parameters
+    ----------
+    repo_url : str
+        Github Repo Url
+    target_folder : Path
+        Download Target
+    branch : str
+        BRanch to download
+    commit : str, optional
+        Specific Commit to download, by default ""
+
+    Raises
+    ------
+    FileNotFoundError
+        If Download failed
+    """
+    git_url = GitUrl(repo_url)
+    download_url = git_url.get_archive_url(ref=commit or branch)
+    with tempfile.TemporaryDirectory() as tdir:
+        zip_path = Path(tdir) / f"{git_url.name}.zip"
+        LOGGER.info("Downloading GitRepo Zip: '%s'", download_url)
+        LOGGER.debug("Target Path: '%s' ", zip_path)
+        download_file(download_url, zip_path)
+        if not zip_path.exists():
+            raise FileNotFoundError(f"Could not download Repo Zip from: {download_url}")
+        LOGGER.info("Extracting GitRepo Zip: %s", git_url.name)
+        ex_location = Path(tdir) / "extract"
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(ex_location)
+        for path in ex_location.glob("*"):
+            LOGGER.info("Moving %s to %s", path.stem, target_folder)
+            shutil.rmtree(target_folder, ignore_errors=True)
+            path.rename(target_folder)
+            break

godoo_cli/helpers/__init__.py

@@ -0,0 +1 @@
+"""Provides Various Helper Functions for Odoo"""

godoo_cli/helpers/bootstrap.py

@@ -0,0 +1,61 @@
+import logging
+import re
+import subprocess
+from ast import literal_eval
+from pathlib import Path
+from typing import List
+
+from .odoo_files import _get_python_requirements_of_modules
+from .system import run_cmd
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _install_py_reqs_for_modules(addon_paths: List[Path], module_names: List[str] = None):
+    """Install Python Requirements mentioned in odoo module manifests of given modules
+
+    Parameters
+    ----------
+    addon_paths : List[Path]
+        odoo-bin addons path
+    module_names : List[str], Optional
+        List of modules to filter for, by default "all available modules"
+
+    Returns
+    -------
+    CompletedProcess
+    """
+    py_reqs = _get_python_requirements_of_modules(addon_paths=addon_paths, filter_module_names=module_names)
+    if py_reqs:
+        installed_packages = run_cmd(
+            "pip list --format json --disable-pip-version-check", check=True, shell=True, stdout=subprocess.PIPE
+        ).stdout.decode("utf-8")
+        installed_packages = [m.get("name") for m in literal_eval(installed_packages)]
+        if missing_packages := [p for p in py_reqs if p not in installed_packages]:
+            LOGGER.info("Installing Python requirements: %s", missing_packages)
+            res = run_cmd(f"pip install {' '.join(missing_packages)} --disable-pip-version-check", shell=True)
+            if res.returncode != 0:
+                raise FileNotFoundError("Pip installation error at: %s" % missing_packages)
+            return res
+    LOGGER.debug("No py Requirements to be installed")
+
+
+def _install_py_reqs_by_odoo_cmd(addon_paths: List[Path], odoo_bin_cmd: str):
+    """Install Python reqs for modules mentioned in odoo-bin commandline --init or -i directives.
+
+    Parameters
+    ----------
+    addon_paths : List[Path]
+        odoo-bin addons-path
+    odoo_bin_cmd : str
+        odoo-bin commandline
+
+    Returns
+    -------
+    CompletedProcess
+    """
+    install_modules = []
+    for m in re.finditer(r'(--init|-i) "?([^ \n]+)"?', odoo_bin_cmd):
+        install_modules += m.group(2).split(",")
+    if install_modules:
+        return _install_py_reqs_for_modules(addon_paths=addon_paths, module_names=install_modules)

godoo_cli/helpers/cli.py

@@ -0,0 +1,15 @@
+import logging
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+
+def check_dangerous_command():
+    isdev = str(os.getenv("WORKSPACE_IS_DEV"))
+    if not isdev.lower() == "true":
+        LOGGER.warning(
+            """This function is dangerous in Production environments.
+Please set 'WORKSPACE_IS_DEV=true' as environment Variable to continue
+            """
+        )
+        exit(1)