xflow.framework 0.1.0__py3-none-any.whl

xflow/framework/ssh.py

@@ -0,0 +1,372 @@
+# Copyright (c) 2025-2026, zhaowcheng <zhaowcheng@163.com>
+
+"""
+SSH 模块。
+"""
+
+import time
+import socket
+
+from typing import Generator, Optional, Dict, Union, Callable
+from select import select
+from contextlib import contextmanager
+from pathlib import Path, PurePosixPath
+
+from decorator import decorator
+from paramiko import SSHClient, AutoAddPolicy, SFTPClient
+from paramiko.ssh_exception import (AuthenticationException, 
+                                    NoValidConnectionsError, 
+                                    SSHException)
+
+from xflow.framework.errors import SSHConnectError, CommandError
+from xflow.framework.utils import remove_ansi_escape_chars, remove_unprintable_chars
+
+
+@decorator
+def autopen(func, *args, **kwargs):
+    """
+    自动连接器。
+    """
+    conn: SSHConnection = args[0]
+    conn.open()
+    return func(*args, **kwargs)
+
+
+class CommandResult(str):
+    """
+    命令输出结果。
+    """
+    def __new__(cls, out: str, rc: int = 0, cmd: str = '') -> str:
+        """
+        :param out: 输出。
+        :param rc: 返回码。
+        :param cmd: 执行的命令。
+        """
+        out = remove_ansi_escape_chars(out)
+        out = remove_unprintable_chars(out)
+        out = '\n'.join(out.splitlines())
+        o = str.__new__(cls, out.strip())
+        o.__rc = rc
+        o.__cmd = cmd
+        return o
+
+    @property
+    def rc(self) -> int:
+        """
+        返回码。
+        """
+        return self.__rc
+
+    @property
+    def cmd(self) -> str:
+        """
+        执行的命令。
+        """
+        return self.__cmd
+
+    def getfield(
+        self,
+        key: str,
+        col: int,
+        sep: str = None
+    ) -> Optional[str]:
+        """
+        从输出中获取指定字段。
+
+        :param key: 用来筛选行的关键字。
+        :param col: 筛选行中字段所在的列号（从 1 开始）。
+        :param sep: 用来分割行的符号。
+
+        >>> r = CommandResult('''\\
+        ... UID        PID   CMD
+        ... postgres   45    /opt/pgsql/bin/postgres
+        ... postgres   51    postgres: checkpointer process
+        ... postgres   52    postgres: writer process
+        ... postgres   53    postgres: wal writer process''', 0, '')
+        >>> r.getfield('/opt/pgsql', 2)
+        '45'
+        >>> r.getfield('checkpointer', 1, sep=':')
+        'postgres   51    postgres'
+        """
+        matchline = ''
+        lines = self.splitlines()
+        if isinstance(key, str):
+            for line in self.splitlines():
+                if key in line:
+                    matchline = line
+        elif isinstance(key, int):
+            matchline = lines[key-1]
+        if matchline:
+            fields = matchline.split(sep)
+            return fields[col-1].strip()
+
+    def getcol(
+        self,
+        col: int,
+        sep: str = None
+    ) -> list:
+        """
+        从输出中获取指定列。
+
+        :param col: 列号（从 1 开始）。
+        :param sep: 用来分割行的符号。
+
+        >>> r = CommandResult('''\\
+        ... UID        PID   CMD
+        ... postgres   45    /opt/pgsql/bin/postgres
+        ... postgres   51    postgres: checkpointer process
+        ... postgres   52    postgres: writer process
+        ... postgres   53    postgres: wal writer process''', 0, '')
+        >>> r.getcol(2)
+        ['PID', '45', '51', '52', '53']
+        """
+        fields = []
+        for line in self.splitlines():
+            segs = line.split(sep)
+            if col <= len(segs):
+                fields.append(segs[col-1])
+        return fields
+
+
+class SSHConnection(object):
+    """
+    SSH 连接。
+    """
+    def __init__(
+        self, 
+        ip: str,
+        user: str,
+        password: str,
+        port: int = 22,
+        envs: Optional[Dict[str, str]] = None
+    ):
+        """
+        :param ip: IP 地址。
+        :param user: 用户名。
+        :param password: 用户密码。
+        :param port: SSH 端口。
+        :param envs: 连接的默认环境变量，其中：
+            `LANG` 默认值为 `en_US.UTF-8`。
+            `LANGUAGE` 默认值为 `en_US.UTF-8`。
+        """
+        self._ip = ip
+        self._user = user
+        self._password = password
+        self._port = port
+        self._envs = envs or {}
+        self._connstr = f'ssh://{user}@{ip}:{port}'
+        self._cwd: str = ''
+        self._sshclient = SSHClient()
+        self._sshclient.set_missing_host_key_policy(AutoAddPolicy())
+        self._sftpclient: SFTPClient = None
+        for k, v in {'LANG': 'en_US.UTF-8',
+                     'LANGUAGE': 'en_US.UTF-8'}.items():
+            self._envs.setdefault(k, v)
+
+    def _progress_bar_generator(
+        self,
+        method: str,
+        local: Union[str, Path],
+        remote: Union[str, PurePosixPath],
+        interval: int = 1
+    ) -> Callable:
+        """
+        文件传输进度展示函数生成器。
+
+        :param method: 传输方法，`get` 或 `put`。
+        :param local: 本地文件路径。
+        :param remote: 远端文件路径。
+        :param interval: 展示间隔（秒）。
+        :return: 进度展示函数。
+        """
+        local = Path(local)
+        remote = PurePosixPath(remote)
+        premsg = {
+            'get': f'Get {local} <= {remote}',
+            'put': f'Put {local} => {remote}'
+        }[method]
+        anchor = {'start': int(time.time()), 'last': None}
+        def progress_bar(transferred: int, total: int):
+            def getsize(b: int) -> str:
+                kb = b // 1024
+                mb = round(kb / 1024, 1)
+                gb = round(mb / 1024, 2)
+                if gb >= 1:
+                    return f'{gb}GB'
+                elif mb >= 1:
+                    return f'{mb}MB'
+                elif kb >= 1:
+                    return f'{kb}KB'
+                else:
+                    return f'{b}B'
+            totalsize = getsize(total)
+            transsize = getsize(transferred)
+            percent = int(transferred / total * 100)
+            now = int(time.time())
+            if (now - anchor['start']) % interval == 0 \
+                    and now != anchor['last'] or percent == 100:
+                print(f'[{self._connstr}] {premsg} {transsize}/{totalsize} {percent}%')
+                anchor['last'] = now
+        return progress_bar
+
+    def open(self) -> None:
+        """
+        开启连接。
+        """
+        transport = self._sshclient.get_transport()
+        if transport and transport.active:
+            return
+        timeout = 10
+        try:
+            print(f'[{self._connstr}] Connecting...')
+            self._sshclient.connect(self._ip, 
+                                    port=self._port, 
+                                    username=self._user, 
+                                    password=self._password, 
+                                    timeout=10)
+            self._sftpclient = self._sshclient.open_sftp()
+            print(f'[{self._connstr}] Connected')
+        except AuthenticationException:
+            raise SSHConnectError(
+                f'Authentication failed when SSH connect to {self._ip} with user `{self._user}`, '
+                f'please check whether the username and password are correct.'
+            ) from None
+        except socket.timeout:
+            raise SSHConnectError(
+                f'Timed out when SSH connect to {self._ip}({timeout}s), '
+                'please check whether the network is normal.'
+            ) from None
+        except NoValidConnectionsError:
+            raise SSHConnectError(
+                f'Could not connect to port {self._port} on {self._ip}, '
+                'please check whether the port is opened.'
+            ) from None
+        except SSHException as e:
+            msg = str(e)
+            if 'Error reading SSH protocol banner' in msg:
+                raise SSHConnectError(
+                    f'Read SSH protocol banner failed when connect to port {self._port} '
+                    f'on {self._ip}, please check whether the port is correct.'
+                ) from None
+            raise e from None
+
+    def close(self) -> None:
+        """
+        关闭连接。
+        """
+        self._sshclient.close()
+        self._sftpclient.close()
+
+    @autopen()
+    def exec(
+        self,
+        cmd: str,
+        envs: Optional[Dict[str, str]] = None
+    ) -> CommandResult:
+        """
+        执行命令。
+
+        :param cmd: 被执行的命令。
+        :param envs: 环境变量。
+        :return: 命令输出。
+
+        :raises: 
+            `CommandError` -- 命令返回码不为 0。
+
+        >>> exec('ls /home')  # successful                              # doctest: +SKIP
+        >>> exec('ls /errpath')  # CommandError                         # doctest: +SKIP
+        """
+        envs = envs or {}
+        environment = self._envs.copy()
+        environment.update(envs)
+        print(f'[{self._connstr}:{self._cwd or "~"}] {cmd}')
+        stdin, stdout, stderr = self._sshclient.exec_command(
+            f'cd {self._cwd} && {cmd}' if self._cwd else cmd, 
+            get_pty=True, 
+            environment=environment)
+        output = ''
+        encoding = environment['LANG'].split('.')[-1]
+        while True:
+            rlist, _, _ = select([stdout.channel], [], [], 0.1)
+            if stdout.channel in rlist:
+                data = stdout.channel.recv(1024).decode(encoding=encoding, errors='ignore')
+                if data == '':
+                    break
+                output += data
+                print(data, end='')
+        rc = stdout.channel.recv_exit_status()
+        if rc != 0:
+            raise CommandError(f'ExitCode {rc}: `{cmd}`')
+        return CommandResult(output, rc=rc, cmd=cmd)
+
+    @contextmanager
+    def dir(self, path: str | PurePosixPath) -> Generator[None, None, None]:
+        """
+        切换工作目录。
+
+        >>> with dir('/my/workdir'):     # doctest: +SKIP
+        ...     d = exec('pwd')          # doctest: +SKIP
+        ...                              # doctest: +SKIP
+        >>> d                            # doctest: +SKIP
+        '/my/workdir'                    # doctest: +SKIP
+        """
+        try:
+            self._cwd = str(path)
+            yield
+        finally:
+            self._cwd = ''
+
+    @autopen()
+    def getfile(
+        self, 
+        rfile: Union[str, PurePosixPath], 
+        ldir: Union[str, Path]
+    ) -> None:
+        """
+        从远端下载文件 `rfile` 到本地目录 `ldir`。
+        
+        :param rfile: 远端文件。
+        :param ldir: 本地目录。
+
+        >>> getfile('/tmp/myfile', '/home')  # /home/myfile
+        >>> getfile('/tmp/myfile', 'D:\\')  # D:\\myfile
+        """
+        rfile = PurePosixPath(rfile)
+        ldir = Path(ldir)
+        lfile = ldir.joinpath(rfile.name)
+        self._sftpclient.get(str(rfile), str(lfile), 
+            callback=self._progress_bar_generator('get', lfile, rfile))
+
+    @autopen()
+    def putfile(
+        self, 
+        lfile: Union[str, Path], 
+        rdir: Union[str, PurePosixPath]
+    ) -> None:
+        """
+        上传本地文件 `lfile` 到远端目录 `rdir`。
+
+        :param lfile: 本地文件。
+        :param rdir: 远端目录。
+
+        >>> putfile('/home/myfile', '/tmp')  # /tmp/myfile
+        >>> putfile('D:\\myfile', '/tmp')  # /tmp/myfile
+        """
+        lfile = Path(lfile)
+        rdir = PurePosixPath(rdir)
+        rfile = rdir.joinpath(lfile.name)
+        self._sftpclient.put(str(lfile), str(rfile),
+            callback=self._progress_bar_generator('put', lfile, rfile))
+
+    @autopen()
+    def exists(self, path: Union[str, PurePosixPath]) -> bool:
+        """
+        检查远端路径是否存在。
+        """
+        try:
+            self._sftpclient.stat(str(path))
+            return True
+        except FileNotFoundError:
+
+            return False
+

xflow/framework/statics/initdir/.gitignore

@@ -0,0 +1,215 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+
+# IDE config files
+.vscode/
+.idea/
+
+# workdir
+workdir/

xflow/framework/statics/initdir/env.yml

@@ -0,0 +1,41 @@
+# Docker 服务端列表
+dockers:
+  - name: docker1
+    ip: 192.168.10.100
+    port: 2375
+    tls:
+      cacert: /path/to/ca-cert.pem  # CA 证书（可选项）
+      clientcert: /path/to/client-cert.pem # 客户端证书（可选项，但提供 clientkey 时则必须提供该选项）
+      clientkey: /path/to/client-key.pem  # 客户端私钥（可选项，但提供 clientcert 时则必须提供该选项）
+
+# 节点列表
+nodes:
+  - name: el7-x86_64  # 必须唯一
+    workdir: /home/xflow/workdir
+    envs:
+      PATH: /usr/local/pgsql/bin:$PATH
+      LD_LIBRARY_PATH: /usr/local/pgsql/lib:$LD_LIBRARY_PATH
+      PGPORT: 5432
+    # 以下是非 docker 类型节点的必填项。
+    ip: 192.168.10.10
+    sshport: 22
+    user: xflow  # docker 类型节点时可选，不填将使用容器默认用户。
+    password: xflow@123
+    # 以下是 docker 类型节点的必填项。
+    docker:
+    image:
+    container:
+  - name: ubt22-aarch64
+    workdir: /home/xflow/workdir
+    envs:
+    # 以下是非 docker 类型节点的必填项。
+    ip: 
+    sshport:
+    user: xflow  # docker 类型节点时可选，不填将使用容器默认用户。
+    password: xflow@123
+    # 以下是 docker 类型节点的必填项。
+    docker: docker1  # docker 服务端名称。
+    container:  # 容器名，和 image 为互斥项，如果指定则用该容器作为节点。
+    image: ubuntu-22.04 # 镜像名，和 container 为互斥项，如果指定则每次用该镜像启动一个新容器作为节点。
+    runargs:  # 启动参数，当使用镜像启动容器时可以指定。
+      command: nix-daemon  # 启动命令

xflow/framework/statics/initdir/pipelines/example.py

@@ -0,0 +1,60 @@
+from xflow.framework.pipeline import Pipeline
+
+
+class example(Pipeline):
+    """
+    pipeline 示例。
+    """
+    class Options(Pipeline.Options):
+        """
+        流水线参数，根据需要自行定义。
+        """
+        pyver: int = Pipeline.Option(desc='Python version.',
+                                     default=3)
+        packtype: str = Pipeline.Option(desc='Package type.',
+                                        default='onedir',
+                                        choices=('onefile', 'onedir'))
+    
+    def setup(self) -> None:
+        """
+        前置步骤。
+        """
+        self.options: __class__.Options  # 用于类型推断
+        super().setup()
+
+    def stage1(self) -> None:
+        """
+        拉取代码。
+        """
+        self.node.exec('git clone https://ghfast.top/https://github.com/zhaowcheng/xbot.framework.git')
+
+    def stage2(self) -> None:
+        """
+        编译代码。
+        """
+        with self.node.dir('xbot.framework'):
+            self.node.exec(f'python{self.options.pyver} -m venv venv')
+            self.node.exec('venv/bin/pip install -r requirements.txt')
+            self.node.exec('venv/bin/pip install pyinstaller')
+            self.node.exec(f'venv/bin/python -m PyInstaller --{self.options.packtype} -n xbot xbot/framework/main.py')
+
+    def stage3(self) -> None:
+        """
+        打包。
+        """
+        self.node.putfile(
+            './requirements.txt',
+            self.node.cwd.joinpath('xbot.framework', 'dist')
+        )
+        with self.node.dir('xbot.framework/dist'):
+            self.node.exec('tar czvf xbot.tar.gz xbot/')
+        self.node.getfile(
+            self.node.cwd.joinpath('xbot.framework', 'dist', 'xbot.tar.gz'),
+            self.cwd
+        )
+
+    def teardown(self) -> None:
+        """
+        后置步骤。
+        """
+        super().teardown()