xflow.framework 0.1.0__tar.gz → 0.1.1__tar.gz

xflow_framework-0.1.1/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.4
+Name: xflow.framework
+Version: 0.1.1
+Summary: 可以用 Python 写 Pipeline 的 CICD 框架。
+Author-email: zhaowcheng <zhaowcheng@163.com>
+License-Expression: BSD-2-Clause
+Project-URL: Homepage, https://github.com/zhaowcheng/xflow.framework
+Project-URL: Issues, https://github.com/zhaowcheng/xflow.framework/issues
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ruamel.yaml
+Requires-Dist: paramiko
+Requires-Dist: decorator
+Requires-Dist: filelock
+Requires-Dist: docker
+Requires-Dist: typed-settings[click,option-groups,pydantic]
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license-file
+
+# xflow.framework
+
+可以用 Python 写 Pipeline 的 CICD 框架。

xflow_framework-0.1.1/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "xflow.framework"
+description = "可以用 Python 写 Pipeline 的 CICD 框架。"
+authors = [
+    { name = "zhaowcheng", email = "zhaowcheng@163.com" },
+]
+license = "BSD-2-Clause"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+dependencies = [
+    "ruamel.yaml",
+    "paramiko",
+    "decorator",
+    "filelock",
+    "docker",
+    "typed-settings[pydantic,click,option-groups]",
+]
+classifiers = [
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dynamic = ["version", "readme"]
+
+[project.urls]
+Homepage = "https://github.com/zhaowcheng/xflow.framework"
+Issues = "https://github.com/zhaowcheng/xflow.framework/issues"
+
+[project.scripts]
+xflow = "xflow.framework.main:main"
+
+[tool.setuptools.dynamic]
+
+[tool.setuptools.packages.find]
+include = ["xflow*"]
+exclude = ["tests", "tests.*"]
+
+[tool.setuptools.package-data]
+"xflow.framework" = ["statics/initdir/**"]

xflow_framework-0.1.1/setup.py

@@ -0,0 +1,32 @@
+# Build hook: read version from xflow/framework/version.py,
+# generate long_description from README files.
+# All other metadata lives in pyproject.toml.
+import re
+
+from setuptools import setup
+
+
+_VERSION_FILE = "xflow/framework/version.py"
+
+
+def _find_version() -> str:
+    with open(_VERSION_FILE, encoding="utf8") as f:
+        m = re.search(r'''__version__\s*=\s*['"]([^'"]+)''', f.read())
+    return m.group(1) if m else "0.0.0"
+
+
+def _build_long_description(version: str) -> str:
+    with open("README.md", encoding="utf8") as f:
+        desc = f.read()
+    desc = desc.replace("/blob/master/", f"/blob/v{version}/")
+    desc = desc.replace("/tree/master/", f"/tree/v{version}/")
+    return desc
+
+
+_VERSION = _find_version()
+
+setup(
+    version=_VERSION,
+    long_description=_build_long_description(_VERSION),
+    long_description_content_type="text/markdown",
+)

xflow_framework-0.1.1/tests/test_cli.py

@@ -0,0 +1,137 @@
+import os
+import sys
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import click
+from click.testing import CliRunner
+
+from xflow.framework.main import RunGroup, main
+
+
+class FakeNode:
+    is_container = False
+    existed = True
+
+    def mkcwd(self):
+        pass
+
+    def rmcwd(self):
+        pass
+
+    def remove(self, force=False):
+        pass
+
+
+class FakeEnv:
+    def __init__(self, envfile):
+        self.envfile = envfile
+
+    def get_node(self, name):
+        return FakeNode()
+
+
+class CliTest(unittest.TestCase):
+    def setUp(self):
+        self.old_cwd = os.getcwd()
+        self.drop_pipeline_modules()
+
+    def tearDown(self):
+        os.chdir(self.old_cwd)
+        self.drop_pipeline_modules()
+
+    def drop_pipeline_modules(self):
+        for name in list(sys.modules):
+            if name == "pipelines" or name.startswith("pipelines."):
+                sys.modules.pop(name)
+
+    def test_init_copies_template_into_empty_project(self):
+        runner = CliRunner()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            projdir = Path(tmpdir).joinpath("project")
+
+            result = runner.invoke(main, ["-p", str(projdir), "init"])
+
+            self.assertEqual(result.exit_code, 0, result.output)
+            self.assertTrue(projdir.joinpath("env.yml").is_file())
+            self.assertTrue(projdir.joinpath("pipelines", "example.py").is_file())
+            self.assertIn(f"Initialized {projdir}", result.output)
+
+    def test_init_rejects_non_empty_project_directory(self):
+        runner = CliRunner()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            projdir = Path(tmpdir).joinpath("project")
+            projdir.mkdir()
+            projdir.joinpath("keep.txt").write_text("data")
+
+            result = runner.invoke(main, ["-p", str(projdir), "init"])
+
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn("exists and is not empty", result.output)
+
+    def test_run_group_lists_pipeline_modules_only(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            pipelines = Path(tmpdir).joinpath("pipelines")
+            pipelines.mkdir()
+            pipelines.joinpath("__init__.py").write_text("")
+            pipelines.joinpath("template.py").write_text("")
+            pipelines.joinpath("demo.py").write_text("")
+            pipelines.joinpath("build.py").write_text("")
+            pipelines.joinpath("notes.txt").write_text("")
+            group = RunGroup("run")
+            ctx = click.Context(group, obj={"projdir": tmpdir})
+
+            self.assertCountEqual(group.list_commands(ctx), ["demo", "build"])
+
+    def test_run_executes_pipeline_command(self):
+        runner = CliRunner()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            projdir = Path(tmpdir)
+            pipelines = projdir.joinpath("pipelines")
+            pipelines.mkdir()
+            pipelines.joinpath("__init__.py").write_text("")
+            pipelines.joinpath("demo.py").write_text(
+                textwrap.dedent(
+                    """
+                    from pathlib import Path
+                    from xflow.framework.pipeline import Pipeline
+
+                    class demo(Pipeline):
+                        def stage1(self):
+                            Path(self.projdir).joinpath("ran.txt").write_text("ok")
+                    """
+                )
+            )
+
+            with patch("xflow.framework.main.Env", FakeEnv):
+                result = runner.invoke(main, ["-p", str(projdir), "run", "-n", "local", "demo"])
+
+            self.assertEqual(result.exit_code, 0, result.output)
+            self.assertEqual(projdir.joinpath("ran.txt").read_text(), "ok")
+
+    def test_failed_pipeline_command_returns_nonzero(self):
+        runner = CliRunner()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            projdir = Path(tmpdir)
+            pipelines = projdir.joinpath("pipelines")
+            pipelines.mkdir()
+            pipelines.joinpath("__init__.py").write_text("")
+            pipelines.joinpath("broken.py").write_text(
+                textwrap.dedent(
+                    """
+                    from xflow.framework.pipeline import Pipeline
+
+                    class broken(Pipeline):
+                        def stage1(self):
+                            raise RuntimeError("boom")
+                    """
+                )
+            )
+
+            with patch("xflow.framework.main.Env", FakeEnv):
+                result = runner.invoke(main, ["-p", str(projdir), "run", "-n", "local", "broken"])
+
+            self.assertEqual(result.exit_code, 1)

xflow_framework-0.1.1/tests/test_command_result.py

@@ -0,0 +1,39 @@
+import unittest
+
+from xflow.framework.ssh import CommandResult
+
+
+class CommandResultTest(unittest.TestCase):
+    def test_normalizes_output_and_keeps_metadata(self):
+        result = CommandResult("\x1b[31m hello\x1b[0m\xe9\r\nworld\n", rc=7, cmd="echo hello")
+
+        self.assertEqual(str(result), "hello\nworld")
+        self.assertEqual(result.rc, 7)
+        self.assertEqual(result.cmd, "echo hello")
+
+    def test_getfield_finds_last_matching_line(self):
+        result = CommandResult(
+            """\
+UID        PID   CMD
+postgres   45    /opt/pgsql/bin/postgres
+postgres   51    postgres: checkpointer process
+postgres   52    postgres: writer process
+"""
+        )
+
+        self.assertEqual(result.getfield("postgres", 2), "52")
+        self.assertEqual(result.getfield("checkpointer", 1, sep=":"), "postgres   51    postgres")
+        self.assertEqual(result.getfield(1, 2), "PID")
+        self.assertIsNone(result.getfield("missing", 1))
+
+    def test_getcol_returns_available_fields(self):
+        result = CommandResult(
+            """\
+name: value
+other: second
+lonely
+"""
+        )
+
+        self.assertEqual(result.getcol(1, sep=":"), ["name", "other", "lonely"])
+        self.assertEqual(result.getcol(2, sep=":"), [" value", " second"])

xflow_framework-0.1.1/tests/test_container.py

@@ -0,0 +1,41 @@
+import unittest
+
+from xflow.framework.container import ContainerConnection
+
+
+class DummyContainer:
+    name = "created"
+
+
+class ContainerConnectionTest(unittest.TestCase):
+    def test_requires_exactly_one_container_name_or_image(self):
+        with self.assertRaisesRegex(ValueError, "only one"):
+            ContainerConnection("127.0.0.1")
+
+        with self.assertRaisesRegex(ValueError, "only one"):
+            ContainerConnection("127.0.0.1", name="existing", image="busybox")
+
+    def test_client_certificate_and_key_must_be_paired(self):
+        with self.assertRaisesRegex(ValueError, "clientcert"):
+            ContainerConnection("127.0.0.1", name="existing", clientcert="cert.pem")
+
+        with self.assertRaisesRegex(ValueError, "clientcert"):
+            ContainerConnection("127.0.0.1", name="existing", clientkey="key.pem")
+
+    def test_connection_string_and_existed_flag(self):
+        existing = ContainerConnection("127.0.0.1", name="existing")
+        self.assertTrue(existing.existed)
+        self.assertEqual(existing._connstr, "docker://127.0.0.1:2375:existing")
+
+        created = ContainerConnection("127.0.0.1", user="root", image="busybox")
+        self.assertFalse(created.existed)
+        self.assertEqual(created._connstr, "docker://root@127.0.0.1:2375:busybox->...")
+
+        created._container = DummyContainer()
+        self.assertEqual(created._connstr, "docker://root@127.0.0.1:2375:busybox->created")
+
+    def test_default_locale_environment_is_set(self):
+        conn = ContainerConnection("127.0.0.1", name="existing", envs={"LANG": "C.UTF-8"})
+
+        self.assertEqual(conn._envs["LANG"], "C.UTF-8")
+        self.assertEqual(conn._envs["LANGUAGE"], "en_US.UTF-8")

xflow_framework-0.1.1/tests/test_env.py

@@ -0,0 +1,126 @@
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from xflow.framework.env import Env
+from xflow.framework.errors import NoSuchDockerError, NoSuchNodeError
+
+
+class FakeNode:
+    kind = "base"
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.name = kwargs["name"]
+
+
+class FakeNativeNode(FakeNode):
+    kind = "native"
+
+
+class FakeContainerNode(FakeNode):
+    kind = "container"
+
+
+class EnvTest(unittest.TestCase):
+    def write_env(self, content):
+        tmpdir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmpdir.cleanup)
+        envfile = Path(tmpdir.name).joinpath("env.yml")
+        envfile.write_text(textwrap.dedent(content), encoding="utf8")
+        return envfile
+
+    def test_loads_native_and_container_nodes(self):
+        envfile = self.write_env(
+            """
+            nodes:
+              - name: native
+                docker:
+                ip: 192.0.2.10
+                sshport: 22
+                user: builder
+                password: secret
+                workdir: /work/native
+                envs:
+                  A: B
+              - name: container
+                docker: local-docker
+                user: root
+                workdir: /work/container
+                container: buildbox
+                image:
+                runargs: {}
+                envs:
+                  C: D
+            dockers:
+              - name: local-docker
+                ip: 127.0.0.1
+                port: 2375
+                tls:
+                  cacert:
+                  clientcert:
+                  clientkey:
+            """
+        )
+
+        with patch("xflow.framework.env.NativeNode", FakeNativeNode), patch(
+            "xflow.framework.env.ContainerNode", FakeContainerNode
+        ):
+            env = Env(envfile)
+
+        native = env.get_node("native")
+        container = env.get_node("container")
+
+        self.assertIsInstance(native, FakeNativeNode)
+        self.assertEqual(native.kwargs["ip"], "192.0.2.10")
+        self.assertEqual(native.kwargs["envs"], {"A": "B"})
+
+        self.assertIsInstance(container, FakeContainerNode)
+        self.assertEqual(container.kwargs["ip"], "127.0.0.1")
+        self.assertEqual(container.kwargs["container"], "buildbox")
+        self.assertIsNone(container.kwargs["image"])
+        self.assertEqual(container.kwargs["envs"], {"C": "D"})
+
+    def test_missing_docker_reference_raises(self):
+        envfile = self.write_env(
+            """
+            nodes:
+              - name: container
+                docker: missing
+                user: root
+                workdir: /work/container
+                container: buildbox
+                image:
+                runargs: {}
+                envs: {}
+            dockers: []
+            """
+        )
+
+        with patch("xflow.framework.env.ContainerNode", FakeContainerNode):
+            with self.assertRaisesRegex(NoSuchDockerError, "missing"):
+                Env(envfile)
+
+    def test_get_node_raises_for_unknown_name(self):
+        envfile = self.write_env(
+            """
+            nodes:
+              - name: native
+                docker:
+                ip: 192.0.2.10
+                sshport: 22
+                user: builder
+                password: secret
+                workdir: /work/native
+                envs: {}
+            dockers: []
+            """
+        )
+
+        with patch("xflow.framework.env.NativeNode", FakeNativeNode):
+            env = Env(envfile)
+
+        with self.assertRaisesRegex(NoSuchNodeError, "unknown"):
+            env.get_node("unknown")

xflow_framework-0.1.1/tests/test_node.py

@@ -0,0 +1,182 @@
+import tempfile
+import unittest
+from contextlib import contextmanager
+from pathlib import Path, PurePosixPath
+
+from xflow.framework.node import Node
+from xflow.framework.pipeline import Pipeline
+from xflow.framework.ssh import CommandResult
+
+
+class FakeConnection:
+    existed = True
+
+    def __init__(self):
+        self.current_dir = ""
+        self.dir_calls = []
+        self.exec_calls = []
+        self.getfile_calls = []
+        self.putfile_calls = []
+        self.remove_calls = []
+        self.existing = set()
+        self.uploaded_text = {}
+
+    @contextmanager
+    def dir(self, path):
+        self.dir_calls.append(path)
+        previous = self.current_dir
+        self.current_dir = str(path)
+        try:
+            yield
+        finally:
+            self.current_dir = previous
+
+    def exec(self, cmd, envs=None):
+        self.exec_calls.append((cmd, envs, self.current_dir))
+        if cmd.startswith("mkdir -p "):
+            self.existing.add(cmd.removeprefix("mkdir -p "))
+        return CommandResult("ok", rc=0, cmd=cmd)
+
+    def getfile(self, rfile, ldir):
+        self.getfile_calls.append((rfile, ldir))
+
+    def putfile(self, lfile, rdir):
+        lfile = Path(lfile)
+        rdir = PurePosixPath(rdir)
+        rfile = rdir.joinpath(lfile.name)
+        self.putfile_calls.append((lfile, rdir))
+        self.existing.add(str(rfile))
+        try:
+            self.uploaded_text[str(rfile)] = lfile.read_text()
+        except (FileNotFoundError, UnicodeDecodeError):
+            pass
+
+    def exists(self, path):
+        return str(path) in self.existing
+
+    def remove(self, force=False):
+        self.remove_calls.append(force)
+
+
+class DummyPipeline(Pipeline):
+    def stage1(self):
+        pass
+
+
+class FakeEnv:
+    def __init__(self, node):
+        self.node = node
+
+    def get_node(self, name):
+        return self.node
+
+
+class NodeTest(unittest.TestCase):
+    def make_node_and_pipeline(self):
+        conn = FakeConnection()
+        node = Node("local", "builder", "/base", conn)
+        tmpdir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmpdir.cleanup)
+        pipeline = DummyPipeline(tmpdir.name, FakeEnv(node), "local", DummyPipeline.Options())
+        pipeline.buildid = 42
+        return node, conn, pipeline
+
+    def test_cwd_uses_current_pipeline_and_dir_context(self):
+        node, _, pipeline = self.make_node_and_pipeline()
+
+        self.assertEqual(node.cwd, PurePosixPath("/base/DummyPipeline/42"))
+
+        with node.dir("src"):
+            self.assertEqual(node.cwd, PurePosixPath("/base/DummyPipeline/42/src"))
+
+        with node.dir("/absolute"):
+            self.assertEqual(node.cwd, PurePosixPath("/absolute"))
+
+        self.assertEqual(node.cwd, PurePosixPath("/base/DummyPipeline/42"))
+        self.assertIs(pipeline.node, node)
+
+    def test_mkcwd_and_rmcwd_execute_against_pipeline_workdir(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        node.mkcwd()
+        node.rmcwd()
+
+        self.assertEqual(
+            conn.exec_calls,
+            [
+                ("mkdir -p /base/DummyPipeline/42", None, ""),
+                ("rm -rf /base/DummyPipeline/42", None, ""),
+            ],
+        )
+        self.assertIs(pipeline.node, node)
+
+    def test_exec_runs_inside_current_workdir(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        with node.dir("src"):
+            result = node.exec("make test", envs={"A": "B"})
+
+        self.assertEqual(result, "ok")
+        self.assertEqual(conn.dir_calls, [PurePosixPath("/base/DummyPipeline/42/src")])
+        self.assertEqual(conn.exec_calls, [("make test", {"A": "B"}, "/base/DummyPipeline/42/src")])
+        self.assertIs(pipeline.node, node)
+
+    def test_nixenv_wraps_command_and_clears_after_context(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        with node.nixenv("/flake", system="x86_64-linux", name="dev", options="--impure"):
+            node.exec("make")
+        node.exec("pwd")
+
+        self.assertEqual(
+            conn.exec_calls[0][0],
+            "nix develop /flake#devShells.x86_64-linux.dev --impure --log-format raw -c make",
+        )
+        self.assertEqual(conn.exec_calls[1][0], "pwd")
+        self.assertIs(pipeline.node, node)
+
+    def test_exec_script_uploads_missing_script_once(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            script = Path(tmpdir).joinpath("build.sh")
+            script.write_text("#!/bin/sh\necho ok\n")
+
+            node.exec_script(script, "arg1")
+            node.exec_script(script, "arg2")
+
+        remote_script = "/base/DummyPipeline/42/scripts/build.sh"
+        self.assertEqual(conn.exec_calls[0][0], "mkdir -p /base/DummyPipeline/42/scripts")
+        self.assertEqual(conn.exec_calls[1][0], f"chmod +x {remote_script}")
+        self.assertEqual(conn.exec_calls[2][0], f"{remote_script} arg1")
+        self.assertEqual(conn.exec_calls[3][0], f"{remote_script} arg2")
+        self.assertEqual(len(conn.putfile_calls), 1)
+        self.assertIs(pipeline.node, node)
+
+    def test_file_helpers_delegate_to_connection(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        node.getfile("/remote/out.txt", "/local")
+        node.putfile("/local/in.txt", "/remote")
+        self.assertFalse(node.exists("/missing"))
+
+        self.assertEqual(conn.getfile_calls, [("/remote/out.txt", "/local")])
+        self.assertEqual(conn.putfile_calls[-1], (Path("/local/in.txt"), PurePosixPath("/remote")))
+        self.assertIs(pipeline.node, node)
+
+    def test_write_uploads_text_file_to_remote_directory(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        node.write("hello", "/remote/message.txt")
+
+        self.assertEqual(conn.uploaded_text["/remote/message.txt"], "hello")
+        self.assertIs(pipeline.node, node)
+
+    def test_git_clones_and_checks_out_revision(self):
+        node, conn, pipeline = self.make_node_and_pipeline()
+
+        node.git("https://example.invalid/repo.git", "v1.0", options="--depth 1")
+
+        self.assertEqual(conn.exec_calls[0][0], "git clone --depth 1 https://example.invalid/repo.git repo")
+        self.assertEqual(conn.dir_calls[1], PurePosixPath("/base/DummyPipeline/42/repo"))
+        self.assertEqual(conn.exec_calls[1][0], "git checkout v1.0")
+        self.assertIs(pipeline.node, node)