travis2docker 7.0.0__py3-none-any.whl

travis2docker/__init__.py

@@ -0,0 +1 @@
+__version__ = "7.0.0"

travis2docker/__main__.py

@@ -0,0 +1,14 @@
+"""Entrypoint module, in case you use `python -mtravis2docker`.
+
+
+Why does this file exist, and why __main__? For more info, read:
+
+- https://www.python.org/dev/peps/pep-0338/
+- https://docs.python.org/2/using/cmdline.html#cmdoption-m
+- https://docs.python.org/3/using/cmdline.html#cmdoption-m
+"""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

travis2docker/cli.py

@@ -0,0 +1,281 @@
+"""Module that contains the command line app.
+
+Why does this file exist, and why not put this in __main__?
+
+  You might be tempted to import things from __main__ later, but that will cause
+  problems: the code will get executed twice:
+
+  - When you run `python -mtravis2docker` python will execute
+    ``__main__.py`` as a script. That means there won't be any
+    ``travis2docker.__main__`` in ``sys.modules``.
+  - When you import __main__ it will get executed again (as a module) because
+    there's no ``travis2docker.__main__`` in ``sys.modules``.
+
+"""
+
+import argparse
+import os
+import pathlib
+from sys import stdout
+
+from . import __version__
+from .exceptions import InvalidRepoBranchError
+from .git_run import GitRun
+from .travis2docker import Travis2Docker
+
+
+def variables_sh_read(variables_sh_path):
+    variables_sh_path = pathlib.Path(variables_sh_path).expanduser()
+    if variables_sh_path.is_dir():
+        variables_sh_path /= "variables.sh"
+    if not variables_sh_path.is_file():
+        return None
+    return variables_sh_path.read_text()
+
+
+def get_git_data(project, path, revision):
+    git_obj = GitRun(project, path, path_prefix_repo=True)
+    git_obj.update()
+    data = {
+        "sha": git_obj.get_sha(revision),
+        "variables_sh": git_obj.show_file("variables.sh", revision),
+        "repo_owner": git_obj.owner,
+        "repo_project": git_obj.repo,
+        "git_email": git_obj.get_config_data("user.email"),
+        "git_user": git_obj.get_config_data("user.name"),
+        "revision": revision,
+        "project": project,
+    }
+    return data
+
+
+def main(return_result=False):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "git_repo_url",
+        help="Specify repository git of work."
+        "\nThis is used to clone it "
+        "and get the variables.sh file of the deployv image"
+        "\nIf your repository is private, "
+        "don't use https url, "
+        "use ssh url",
+    )
+    parser.add_argument(
+        "git_revision",
+        help="Revision git of work."
+        "\nYou can use "
+        "branch name e.g. master or 8.0 "
+        "or pull number with 'pull/#' e.g. pull/1 "
+        "NOTE: A sha e.g. b48228 NOT IMPLEMENTED YET",
+    )
+    parser.add_argument(
+        "--docker-user",
+        dest="docker_user",
+        help="User of work into Dockerfile.\nBased on your docker image.\nDefault: odoo",
+    )
+    parser.add_argument(
+        "--docker-image",
+        dest="default_docker_image",
+        help="Docker image to use by default in Dockerfile."
+        "\nDefault: built from variables.sh as "
+        "'DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT'",
+    )
+    default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH")
+    if not default_root_path:
+        default_root_path = pathlib.Path("~").expanduser()
+    default_root_path = str(pathlib.Path(default_root_path) / ".t2d")
+    parser.add_argument(
+        "--root-path",
+        dest="root_path",
+        help=f"Root path to save scripts generated.\nDefault: {default_root_path}",
+        default=default_root_path,
+    )
+    parser.add_argument(
+        "--add-remote",
+        dest="remotes",
+        help="Add git remote to git of build path, separated by a comma.\nUse remote name. E.g. 'Vauxoo,moylop260'",
+    )
+    parser.add_argument(
+        "--exclude-after-success",
+        dest="exclude_after_success",
+        action="store_true",
+        default=False,
+        help="Deprecated. Ignored: the travis_after_success section does not exist anymore",
+    )
+    parser.add_argument(
+        "--run-extra-args",
+        dest="run_extra_args",
+        help="Extra arguments to `docker run RUN_EXTRA_ARGS` command",
+        default="-itP -e LANG=C.UTF-8",
+    )
+    parser.add_argument(
+        "--run-extra-cmds",
+        dest="run_extra_cmds",
+        nargs="*",
+        default="",
+        help='Extra commands to run after "run" script. '
+        "Note: You can use \\$IMAGE escaped environment variable."
+        'E.g. "docker rmi -f \\$IMAGE"',
+    )
+    parser.add_argument(
+        "--build-extra-args",
+        dest="build_extra_args",
+        help="Extra arguments to `docker build BUILD_EXTRA_ARGS` command",
+        default="--rm",
+    )
+    parser.add_argument(
+        "--build-extra-cmds",
+        dest="build_extra_cmds",
+        nargs="*",
+        default="",
+        help='Extra commands to run after "build" script. Note: You can use \\$IMAGE escaped environment variable.',
+    )
+    parser.add_argument(
+        "--travis-yml-path",
+        dest="travis_yml_path",
+        default=None,
+        help="Deprecated. Ignored: the .travis.yml file is not used anymore",
+    )
+    parser.add_argument(
+        "--variables-sh-path",
+        dest="variables_sh_path",
+        default=None,
+        help="Optional path of the variables.sh file (or the directory containing it) to use.\n"
+        "Default: Extracted from git repo and git revision.",
+    )
+    parser.add_argument(
+        "--no-clone",
+        dest="no_clone",
+        action="store_true",
+        default=False,
+        help="Avoid cloning the repository. It requires --variables-sh-path",
+    )
+    parser.add_argument(
+        "--add-rcfile",
+        dest="add_rcfile",
+        default="",
+        help="Optional paths of configuration files to "
+        "copy for user's HOME path into container, separated by a comma.",
+    )
+    parser.add_argument("-v", "--version", action="version", version="%(prog)s " + __version__)
+    parser.add_argument(
+        "--runs-at-the-end-script",
+        dest="runs_at_the_end_script",
+        nargs="*",
+        default="",
+        help="Deprecated. Ignored: the script section of .travis.yml does not exist anymore",
+    )
+    parser.add_argument(
+        "--build-env-args",
+        dest="build_env_args",
+        nargs="*",
+        action="append",
+        default=[],
+        help="Args used as environment variables "
+        "More info about: https://vsupalov.com/docker-build-time-env-values\n"
+        "E.g. --build-env-args ENVAR1\n"
+        "It generates the following line for Dockerfile:\n"
+        "ARG ENVVAR1\nENV ENVVAR1=$ENVVAR1",
+    )
+    parser.add_argument(
+        "--deployv",
+        dest="deployv",
+        action="store_true",
+        default=True,
+        help="Deprecated. The deployv image is now the only supported mode",
+    )
+    parser.add_argument(
+        "--build-extra-steps",
+        nargs="*",
+        default="",
+        dest="build_extra_steps",
+        help="Append these extra steps at the end of the Dockerfile",
+    )
+
+    args = parser.parse_args()
+    deprecated_args = {
+        "--exclude-after-success": args.exclude_after_success,
+        "--travis-yml-path": args.travis_yml_path,
+        "--runs-at-the-end-script": args.runs_at_the_end_script,
+    }
+    for deprecated_arg, value in deprecated_args.items():
+        if value:
+            stdout.write("WARNING: %s is deprecated and its value will be ignored\n" % deprecated_arg)
+    revision = args.git_revision
+    git_repo = args.git_repo_url
+    git_base = GitRun.get_data_url(git_repo, False)[0]
+    docker_user = args.docker_user
+    root_path = args.root_path
+    default_docker_image = args.default_docker_image
+    remotes = args.remotes and args.remotes.split(",")
+    run_extra_args = args.run_extra_args
+    build_extra_args = args.build_extra_args
+    build_extra_cmds = "\n".join(args.build_extra_cmds)
+    run_extra_cmds = "\n".join(args.run_extra_cmds)
+    rcfiles_args = args.add_rcfile and args.add_rcfile.split(",")
+    build_env_args = [build_env_args[0] for build_env_args in args.build_env_args]
+    rcfiles = [
+        (pathlib.Path(rc_file).expanduser(), "$HOME/%s" % pathlib.Path(rc_file).name) for rc_file in rcfiles_args
+    ]
+    if args.no_clone:
+        os_kwargs = {
+            "repo_owner": "local_file",
+            "repo_project": "local_file",
+            "revision": revision,
+            "sha": "local_file",
+            "project": git_repo,
+        }
+    else:
+        os_kwargs = get_git_data(git_repo, pathlib.Path(root_path) / "repo", revision)
+
+    if args.variables_sh_path:
+        os_kwargs["variables_sh"] = variables_sh_read(args.variables_sh_path)
+
+    if not os_kwargs.get("variables_sh"):
+        msg = (
+            "The file %s is empty or does not exist." % args.variables_sh_path
+            if args.variables_sh_path
+            else "The repo or the branch is incorrect value, because "
+            + "It can not got the variables.sh content from %s %s. " % (git_repo, revision)
+            + "\nPlease, verify access repository,"
+            + "\nverify exists url and revision, "
+            + "\nverify exists variables.sh"
+        )
+        raise InvalidRepoBranchError(msg)
+    os_kwargs.update({"remotes": remotes, "git_base": git_base})
+    if docker_user:
+        os_kwargs.update({"user": docker_user})
+    t2d = Travis2Docker(
+        work_path=pathlib.Path(root_path) / "script" / GitRun.url2dirname(git_repo) / revision,
+        image=default_docker_image,
+        os_kwargs=os_kwargs,
+        copy_paths=[(pathlib.Path("~/.ssh").expanduser(), "$HOME/.ssh")] + rcfiles,
+        build_env_args=build_env_args,
+        build_extra_steps=args.build_extra_steps,
+    )
+    t2d.build_extra_params = {
+        "extra_params": build_extra_args,
+        "extra_cmds": build_extra_cmds,
+    }
+    t2d.run_extra_params = {
+        "extra_params": run_extra_args,
+        "extra_cmds": run_extra_cmds,
+    }
+    fname_scripts = t2d.compute_dockerfile()
+    if fname_scripts:
+        fname_list = "- " + "\n- ".join(fname_scripts)
+        stdout.write("\nGenerated scripts:\n%s\n" % fname_list)
+        if not default_docker_image:
+            stdout.write("=" * 80)
+            # TODO: Add the URL to open the pipelines
+            stdout.write(
+                '\nTIP: Use the parameter "--docker-image=quay.io/vauxoo/PROJECT:TAG" '
+                'get the PROJECT:TAG info in your "build_docker" pipeline similar to '
+                '\n"... INFO  - deployv.deployv_addon_gitlab_tools.common.common.push_image - '
+                'Pushing image ... to quay.io/vauxoo/PROJECT:TAG"\n'
+            )
+            stdout.write("=" * 80)
+    else:
+        stdout.write("\nNo scripts were generated.")
+    if return_result:
+        return fname_scripts

travis2docker/docker_helper/build.py

@@ -0,0 +1,88 @@
+# No plan to use logging here
+# pylint: disable=print-used
+
+import glob
+import pathlib
+import re
+import subprocess
+import sys
+
+
+def ssh_keyscan2known_hosts(url, known_hosts_path=None):
+    # python3 -c "import build;build.ssh_keyscan2known_hosts('url')"
+    if known_hosts_path is None:
+        known_hosts_path = pathlib.Path("~/.ssh/known_hosts").expanduser()
+
+    # Clear current known hosts
+    cmd = ["ssh-keygen", "-R", url]
+    print(" ".join(cmd))
+    subprocess.call(cmd)
+
+    # Scan new key of host and store
+    cmd = ["ssh-keyscan", "-p", "22", url]
+    print(" ".join(cmd))
+    keys_scanned = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip()
+    with pathlib.Path(known_hosts_path).open("r+") as known_hosts_f:
+        known_hosts_f.write("\n" + keys_scanned)
+
+
+def git_set_remote(path=None):
+    # python3 -c "import build;build.git_set_remote('')"
+    if path is None:
+        # /home/odoo/instance/odoo spends a lot of time
+        path = "/home/odoo/instance/extra_addons"
+    git_re = re.compile(r"([^/|@]+)/([^/]+)/([^/.]+(.git)?)")
+    # TODO: Support ssh url
+    path = str(pathlib.Path(path) / "*" / "**" / ".git")
+    hosts_scanned = set()
+    for git_dir in glob.glob(path, recursive=True):
+        git_cmd = [
+            "git",
+            "--work-tree=%s" % pathlib.Path(git_dir).parent,
+            "--git-dir=%s" % git_dir,
+        ]
+        cmd = git_cmd + ["remote", "get-url", "--push", "origin"]
+        remote = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip().splitlines()[0]
+        git_re_match = git_re.search(remote)
+
+        # Configure to fetch all branches (not only the single-branch)
+        cmd = git_cmd + [
+            "config",
+            "remote.origin.fetch",
+            "+refs/heads/*:refs/remotes/origin/*",
+        ]
+        subprocess.call(cmd)
+
+        if not git_re_match:
+            print("Remote not matched %s" % remote)
+            continue
+
+        git_re_groups = git_re_match.groups()
+        host = git_re_groups[0]
+        org = git_re_groups[1]
+        repo = git_re_groups[2]
+
+        # Transform https url to ssh format
+        ssh_url_stb = "git@%s:%s/%s" % (host, org, repo)
+        cmd = git_cmd + ["remote", "set-url", "origin", ssh_url_stb]
+        print(" ".join(cmd))
+        subprocess.call(cmd)
+
+        # Unshallow repository
+        cmd = git_cmd + ["rev-parse", "--is-shallow-repository"]
+        is_shallow = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip().splitlines()[0]
+        if is_shallow == "true":
+            if host not in hosts_scanned:
+                ssh_keyscan2known_hosts(host)
+                hosts_scanned.add(host)
+            cmd = git_cmd + ["fetch", "--unshallow"]
+            print(" ".join(cmd))
+            subprocess.call(cmd)
+
+        # Add extra remote if "stb" so add "dev" if "dev" so add "stb"
+        new_org = org.replace("-dev", "") if "-dev" in org else "%s-dev" % org
+        new_remote = "dev" if "dev" in new_org else "stb"
+        ssh_url_dev = "git@%s:%s/%s" % (host, new_org, repo)
+        cmd = git_cmd + ["remote", "add", new_remote, ssh_url_dev]
+        print(" ".join(cmd))
+        subprocess.call(cmd)

travis2docker/exceptions.py

@@ -0,0 +1,5 @@
+"""Exception classes raised by various operations within travis2docker."""
+
+
+class InvalidRepoBranchError(Exception):
+    """Raised when a repo branch is wrong."""

travis2docker/git_run.py

@@ -0,0 +1,114 @@
+# pylint: disable=useless-object-inheritance,print-used,except-pass
+
+import contextlib
+import pathlib
+import re
+import subprocess
+
+
+def decode_utf(field):
+    try:
+        return field.decode("utf-8")
+    except UnicodeDecodeError:
+        return ""
+
+
+class GitRun:
+    def __init__(self, repo_git, path, path_prefix_repo=False):
+        self.repo_git = repo_git
+        path = pathlib.Path(path)
+        if path_prefix_repo:
+            path /= self.url2dirname(repo_git)
+        self.path = path
+        self.host, self.owner, self.repo = self.get_data_url(repo_git)
+
+    @staticmethod
+    def get_data_url(repo_git, no_user=True):
+        host, owner, repo = False, False, False
+        repo_git_sub = repo_git.replace(":", "/")
+        if no_user:
+            repo_git_sub = re.sub(r".+@", "", repo_git_sub)
+        repo_git_sub = re.sub(r".git$", "", repo_git_sub)
+        match_object = re.search(r"(?P<host>[^/]+)/(?P<owner>[^/]+)/(?P<repo>[^/]+)", repo_git_sub)
+        if match_object:
+            host = match_object.group("host")
+            owner = match_object.group("owner")
+            repo = match_object.group("repo")
+        elif pathlib.Path(repo_git).is_dir():
+            host = "local"
+            owner = pathlib.Path(repo_git).name
+            repo = pathlib.Path(repo_git).parent.name
+        return host, owner, repo
+
+    @staticmethod
+    def url2dirname(url):
+        for invalid_char in "@:/#\\":
+            url = url.replace(invalid_char, "_")
+        return url
+
+    def checkout_bare(self, branch):
+        return self.run(["symbolic-ref", "HEAD", branch])
+
+    def get_config_data(self, field=None):
+        if field is None:
+            field = "-l"
+        res = self.run(["config", field])
+        if res:
+            res = res.strip("\n ")
+        return res
+
+    def run(self, cmd):
+        """Execute git command in bash"""
+        cmd = ["git", "--git-dir=%s" % self.path] + cmd
+        print("cmd list", cmd)
+        print("cmd", " ".join(cmd))
+        res = None
+        with contextlib.suppress(BaseException):
+            res = subprocess.check_output(cmd)
+        if res:
+            try:
+                res = res.decode()
+            except UnicodeDecodeError:
+                res = res.decode("utf-8")
+        return res
+
+    def get_ref_data(self, refs=None, fields=None):
+        if refs is None:
+            refs = ["refs/heads"]
+        if fields is None:
+            fields = []
+        if "refname" not in fields:
+            fields.append("refname")
+        # fields = ['refname', 'objectname', 'committerdate:iso8601',
+        #           'authorname', 'authoremail','subject','committername',
+        #           'committeremail']
+        fmt = "%00".join(["%(" + field + ")" for field in fields])
+        git_refs = self.run(["for-each-ref", "--format", fmt, "--sort=refname"] + refs)
+        git_refs = git_refs.strip()
+        refs = [[decode_utf(field) for field in line.split("\x00")] for line in git_refs.split("\n")]
+        res = {}
+        for data_field in refs:
+            subres = dict(zip(fields, data_field))
+            res[subres.pop("refname")] = subres
+        return res
+
+    def update(self):
+        """Get a repository git or update it"""
+        if not self.path.is_dir():
+            self.path.mkdir(parents=True)
+        if not (self.path / "refs").is_dir():
+            subprocess.check_output(["git", "clone", "--bare", self.repo_git, self.path])
+        self.run(["gc", "--auto", "--prune=all"])
+        self.run(["fetch", "-p", "origin", "+refs/heads/*:refs/heads/*"])
+        # github support
+        self.run(["fetch", "origin", "+refs/pull/*/head:refs/pull/*"])
+        # gitlab support
+        self.run(["fetch", "origin", "+refs/merge-requests/*/head:refs/pull/*"])
+
+    def show_file(self, git_file, sha):
+        result = self.run(["show", "%s:%s" % (sha, git_file)])
+        return result
+
+    def get_sha(self, revision):
+        result = self.run(["rev-parse", revision])
+        return result if isinstance(result, list) else (result or "").strip(" \n")

travis2docker/templates/.coveragerc

@@ -0,0 +1,39 @@
+[run]
+branch = True
+command_line =
+    ${HOME}/instance/odoo/odoo-bin -c ${ODOO_CONFIG_FILE} -d odoocov -i ${MODULES2INSTALL}
+    -u ${MODULES2INSTALL} --test-tags ${MODULES2TEST}  --workers 0 --stop-after-init
+relative_files = True
+source = ${COVERAGE_HOME}
+data_file = ${COVERAGE_HOME}/.coverage
+dynamic_context = ${COVERAGE_DYNAMIC_CONTEXT}
+omit =
+    */scenario/*
+    */scenarios/*
+    */test/*
+    */tests/*
+    */migrations/*
+    *migration*
+    *_example/*
+    *__init__.py
+    *__openerp__.py
+    *__manifest__.py
+    ${EXCLUDE_COVERAGE}
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    # Don't complain about null context checking
+    if context is None:
+    # Don't check obviously not implemented
+    raise NotImplementedError
+    # We don't really care what happens if fail
+    except ImportError:
+
+fail_under = ${COVERAGE_MIN-0}
+precision = ${COVERAGE_PRECISION-0}
+
+[html]
+show_contexts=True

travis2docker/templates/.vscode/extensions.json

@@ -0,0 +1,14 @@
+{
+    "recommendations": [
+        "eamodio.gitlens",
+        "ms-python.pylint",
+        "dotjoshjohnson.xml",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-python.flake8",
+        "dbaeumer.vscode-eslint",
+        "redhat.vscode-xml",
+        "mstuttgart.odoo-snippets",
+        "shardulm94.trailing-spaces"
+    ]
+}

travis2docker/templates/.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Odoo",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/instance/odoo/odoo-bin",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "gevent": true,
+            "args": [
+                "-c", "/home/odoo/.openerp_serverrc"
+            ]
+        },
+        {
+            "name": "Odoo Tests",
+            "type": "python",
+            "request": "launch",
+            "program": "${env:HOME}/instance/odoo/odoo-bin",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "gevent": true,
+            "args": [
+                "-c", "/home/odoo/.openerp_serverrc",
+                "--test-tags", "standard"
+            ]
+        }
+    ]
+}

travis2docker/templates/.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+  "editor.rulers": [120],
+  "files.autoSave": "afterDelay",
+  "files.autoSaveDelay": 500,
+  "python.analysis.extraPaths": ["${env:HOME}/instance/odoo"],
+  "python.linting.enabled": true,
+  "pylint.args": ["--rcfile=${env:HOME}/.pylintrc"],
+  "flake8.args": ["--config", "${env:HOME}/.flake8"],
+  "isort.args": ["--settings-path", "${env:HOME}/.isort.cfg"],
+  "eslint.options": { "overrideConfigFile": "${env:HOME}/.eslintrc.json" }
+}

travis2docker/templates/10-build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+export IMAGE={{ image }}
+docker build --pull {{ extra_params }} $1 -t $IMAGE {{ dirname_dockerfile }}
+{{ extra_cmds }}

travis2docker/templates/20-run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+export IMAGE={{ image }}
+CONTAINER=$(docker run {{ extra_params }} $1 -ditP $IMAGE $2)
+
+# Some files under /home/odoo end up owned by root or other users (declared VOLUMEs and
+# files shared into the container); fix their ownership in the background so we can attach ASAP
+docker exec --user=root "$CONTAINER" find /home/odoo -maxdepth 2 -not -user odoo -exec chown -R odoo:odoo {} + &
+
+# If asked to detach, don't attach but print the ID, to mimic docker run behavior
+detach_re='[[:space:]](-[a-zA-Z]*d[a-zA-Z]*|--detach)[[:space:]]'
+if [[ " {{ extra_params }} $1 " =~ $detach_re ]]; then
+    wait
+    echo "$CONTAINER"
+else
+    docker attach "$CONTAINER"
+fi
+{{ extra_cmds }}