kyutil 0.2.1__tar.gz

kyutil-0.2.1/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: kyutil
+Version: 0.2.1
+Summary: Python utils for kylin python application
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: apscheduler>=3.10.4
+Requires-Dist: beautifulsoup4>=4.13.5
+Requires-Dist: celery>=5.2.7
+Requires-Dist: configparser>=5.3.0
+Requires-Dist: cryptography>=45.0.7
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: fastapi>=0.103.2
+Requires-Dist: gitpython>=3.1.45
+Requires-Dist: kobo>=0.40.0
+Requires-Dist: koji>=1.35.3
+Requires-Dist: logzero>=1.7.0
+Requires-Dist: lxml>=5.4.0
+Requires-Dist: msgpack>=1.0.5
+Requires-Dist: networkx>=2.6.3
+Requires-Dist: numpy>=1.21.6
+Requires-Dist: openpyxl>=3.1.3
+Requires-Dist: packaging>=24.0
+Requires-Dist: pandas>=1.1.5
+Requires-Dist: paramiko>=3.5.1
+Requires-Dist: pika>=1.3.2
+Requires-Dist: productmd>=1.48
+Requires-Dist: pydantic>=2.5.3
+Requires-Dist: pydot>=2.0.0
+Requires-Dist: pytest>=7.4.4
+Requires-Dist: python-jenkins>=1.8.3
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: redis>=5.0.8
+Requires-Dist: requests>=2.31.0
+Requires-Dist: retry>=0.9.2
+Requires-Dist: six>=1.17.0
+Requires-Dist: sqlalchemy>=2.0.43
+Requires-Dist: starlette>=0.27.0
+Requires-Dist: urllib3>=2.0.7
+Requires-Dist: wget>=3.2
+
+# kyutil
+麒麟python工具库
+
+    Changelog:
+        ### 0.1.12
+            - BUGFIX: 修复在mock环境内找不到celery.log的BUG    
+    Changelog:
+        ### 0.1.0
+            - initial release    

kyutil-0.2.1/README.md

@@ -0,0 +1,9 @@
+# kyutil
+麒麟python工具库
+
+    Changelog:
+        ### 0.1.12
+            - BUGFIX: 修复在mock环境内找不到celery.log的BUG    
+    Changelog:
+        ### 0.1.0
+            - initial release    

kyutil-0.2.1/kyutil/FTPUtils.py

@@ -0,0 +1,343 @@
+# -*- coding: UTF-8 -*-
+"""FTPUtils.py"""
+import ftplib
+import os
+import ssl
+import traceback
+
+from celery import states
+from logzero import logger
+
+
+class ReusedSslSocket(ssl.SSLSocket):
+    def unwrap(self):
+        pass
+
+
+# MyFTP_TLS is derived to support TLS_RESUME(filezilla server)
+class MyFtpTLS(ftplib.FTP_TLS):
+    """Explicit FTPS, with shared TLS session"""
+
+    def ntransfercmd(self, cmd, rest=None):
+        conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest)
+        if self._prot_p:
+            conn = self.context.wrap_socket(
+                conn,
+                server_hostname=self.host,
+                session=self.sock.session)
+
+            conn.__class__ = ReusedSslSocket
+
+        return conn, size
+
+
+MSG_FTP_CONNECT_SUCCESS = "FTP链接成功！"
+MSG_FTP_CONNECT_FAILED = "FTP链接失败！"
+MSG_FTP_LOGIN_SUCCESS = "FTP登录成功！"
+MSG_FTP_LOGIN_FAILED = "FTP登录失败！"
+
+STOR_CMD = 'STOR '
+
+
+class FTP(object):
+    """FTPUtils(object)"""
+    # 上传过程必需参数列表
+
+    celery_task_field = [
+        'ftpaddress',  # ftp连接地址
+        'ftpport',  # ftp连接端口
+        'ftpuser',  # ftp连接用户
+        'ftppassword',  # ftp连接密码
+        'upload_path',  # ftp上传远端路径
+        'isoname',  # 上传的本地iso路径
+    ]
+
+    # celery参数
+    process_rate_status = 0  # 当前处理的进度，取值从0-100
+
+    def __init__(self, update_status_func, cfg, task_id, logger=logger):
+        """
+        :param update_status_func:
+        :param cfg:
+            cfg = {
+            'ftpaddress': "server.kylinos.cn",
+            'ftpport': "21",
+            'ftpuser': "user_name",
+            'ftppassword': "***",
+            'upload_path': "/vsftpd/os/ISO/HOSTOS/x86_64/2022/06/test",
+            'isoname': "/v10-sp1-0518-kx-x86_64-b3532/iso/Kylin-Server-10-SP1-kx.iso",  # 上传的本地iso路径
+            }
+        :param task_id:
+        :param logger:
+        """
+        self.logger = logger
+        # celery状态更新函数以及taskid
+        self.call_back_task_process = update_status_func
+
+        # 本地ftp目录 默认上传标志关闭
+        self.process_rate_status = 10
+        self.celery_task_param = cfg
+        self.task_id = task_id
+        self.ftp_instance = self.ftp_connect()
+        self.log_and_send_task_status("ftp登录成功")
+
+    # log信息输出&日志信息回传&进度设置！
+    def log_and_send_task_status(self, msg):
+        self.logger.info(f"{msg},【{self.task_id[:4]}】 目前进度{self.process_rate_status}%")
+        # 得想办法把logger.info的输出放到msg里面，这样日志信息会携带时间戳等信息
+        if self.call_back_task_process is not None:
+            if self.process_rate_status == 0:
+                self.call_back_task_process(
+                    state=states.FAILURE,
+                    meta={'current': 0, 'total': 100, 'status': msg, "exc_type": "RuntimeException", "exc_message": msg}
+                )
+            elif self.process_rate_status == 100:
+                self.call_back_task_process(
+                    state=states.SUCCESS,
+                    meta={'current': 100, 'total': 100, 'status': msg, "exc_type": "RuntimeException", "exc_message": msg}
+                )
+            else:
+                self.call_back_task_process(
+                    state=states.STARTED,
+                    meta={'current': self.process_rate_status, 'total': 100, 'status': msg, "exc_type": "RuntimeException", "exc_message": msg}
+                )
+
+    def ftp_connect(self):
+        ftp_address = self.celery_task_param.get('ftpaddress')
+        ftp_port = int(self.celery_task_param.get('ftpport'))
+        ftp = MyFtpTLS(host=ftp_address, timeout=15)
+        ftp.auth()
+        ftp.port = ftp_port
+        ftp.prot_p()
+        ftp.set_pasv(1)
+        ftp.encoding = 'utf-8'
+        ftp_user = self.celery_task_param.get('ftpuser')
+        ftp_password = self.celery_task_param.get('ftppassword')
+
+        try:
+            self.process_rate_status = 20
+            ftp.connect(ftp_address, ftp_port)
+            self.log_and_send_task_status(MSG_FTP_CONNECT_SUCCESS)
+        except Exception:
+            self.log_and_send_task_status(MSG_FTP_CONNECT_FAILED)
+            raise ConnectionError(MSG_FTP_CONNECT_FAILED)
+        try:
+            self.process_rate_status = 40
+            ftp.login(ftp_user, ftp_password)  # user/passwd
+            ftp.prot_p()
+            self.log_and_send_task_status(MSG_FTP_LOGIN_SUCCESS)
+        except Exception:
+            traceback.print_exc()
+            self.log_and_send_task_status(MSG_FTP_LOGIN_FAILED)
+            raise ConnectionError(MSG_FTP_LOGIN_FAILED)
+        return ftp
+
+    def is_same_size(self, local_file, remote_file):
+        """
+        判断远程文件和本地文件大小是否一致
+
+        Args:
+            local_file: 本地文件
+            remote_file: 远程文件
+
+        Returns:
+
+        """
+        try:
+            remote_file_size = self.ftp_instance.size(remote_file)
+        except Exception as err:
+            self.logger.debug("get remote file_size failed, Err:%s" % err)
+            remote_file_size = -1
+
+        try:
+            local_file_size = os.path.getsize(local_file)
+        except Exception as err:
+            self.logger.debug("get local file_size failed, Err:%s" % err)
+            local_file_size = -1
+
+        result = True if (remote_file_size == local_file_size) else False
+
+        return result, remote_file_size, local_file_size
+
+    def upload_file(self, local_file, remote_file, ftp):
+        # 本地是否有此文件
+        if not os.path.exists(local_file):
+            self.logger.debug('no such file or directory %s.' % local_file)
+            return False
+
+        result, remote_file_size, local_file_size = self.is_same_size(local_file, remote_file)
+        if not result:
+            self.logger.debug('remote_file %s is not exist, now trying to upload...' % remote_file)
+            buff_size = 8192
+            try:
+                with open(local_file, 'rb') as file_handler:
+                    if ftp.storbinary(STOR_CMD + remote_file, file_handler, buff_size):
+                        result, remote_file_size, local_file_size = self.is_same_size(local_file, remote_file)
+            except Exception as err:
+                self.logger.debug(
+                    'some error happened in storbinary file :%s. Err:%s' % (local_file, err))
+                result = False
+
+        self.logger.debug('Upload 【%s】 %s , remote_file_size = %d, local_file_size = %d.' \
+                          % (
+                              remote_file, 'success' if (result is True) else 'failed', remote_file_size,
+                              local_file_size))
+        self.logger.info('Upload 【%s】 %s , remote_file_size = %d, local_file_size = %d.' \
+                         % (
+                             remote_file, 'success' if (result is True) else 'failed', remote_file_size,
+                             local_file_size))
+
+    # 01-ftp-connect连接及登录
+
+    # 02-创建远端文件目录
+    def create_remote_dir(self, upload_path):
+        try:
+            self.ftp_instance.cwd(upload_path)
+        except Exception as e:
+            self.log_and_send_task_status(f"ftp cwd异常：{e}")
+            self.ftp_instance.cwd('/')
+            # 分割目录名
+            base_dir, part_path = self.ftp_instance.pwd(), upload_path.split('/')
+            for p in part_path[1:]:
+                # 拼接子目录
+                base_dir = base_dir + p + '/'
+                try:
+                    # 尝试切换子目录
+                    self.ftp_instance.cwd(base_dir)
+                except Exception as e:
+                    self.log_and_send_task_status(f"ftp cwd异常：{e}")
+                    # 不存在创建当前子目录
+                    self.ftp_instance.mkd(base_dir)
+        # 确保最后路径修改
+        self.ftp_instance.cwd(upload_path)
+        return True
+
+    # 03-上传本地指定文件
+    def ftp_upload_file(self, root_path_iso_path="/opt/integration_iso_files/") -> str:
+        iso_file = self.celery_task_param.get('isoname')
+        iso_rename = self.celery_task_param.get('filename_rename')
+
+        buffsize = 1024
+        self.ftp_instance.cwd(self.ftp_instance.pwd())
+
+        with open(root_path_iso_path + iso_file, 'rb') as f:
+            self.log_and_send_task_status('正在上传' + str(f))
+            if iso_rename is not None:
+                self.ftp_instance.storbinary(STOR_CMD + iso_rename, f, buffsize)
+                remote_file = str(self.ftp_instance.pwd()) + '/' + iso_rename
+            else:
+                self.ftp_instance.storbinary(STOR_CMD + iso_file.split('/')[-1], f, buffsize)
+                remote_file = str(self.ftp_instance.pwd()) + '/' + iso_file.split('/')[-1]
+        return remote_file
+
+    def instance_cwd(self, remote_path):
+        try:
+            self.ftp_instance.cwd(remote_path)  # 切换工作路径
+        except Exception as e:
+            self.logger.error('Except INFO:', e)
+            base_dir, part_path = self.ftp_instance.pwd(), remote_path.split('/')
+            for sub_path in part_path:
+                # 针对类似  '/home/billing/scripts/zhf/send' 和 'home/billing/scripts/zhf/send' 两种格式的目录
+                # 如果第一个分解后的元素是''这种空字符，说明根目录是从/开始，如果最后一个是''这种空字符，说明目录是以/结束
+                # 例如 /home/billing/scripts/zhf/send/ 分解后得到 ['', 'home', 'billing', 'scripts', 'zhf', 'send', '']
+                # 首位和尾都不是有效名称
+                if '' == sub_path:
+                    continue
+                base_dir = str(os.path.join(base_dir, sub_path))  # base_dir + subpath + '/'  # 拼接子目录
+                try:
+                    self.ftp_instance.cwd(base_dir)  # 切换到子目录, 不存在则异常
+                except Exception as e:
+                    self.logger.error('Except INFO:', e)
+                    self.logger.error('remote not exist directory %s , create it.' % base_dir)
+                    self.ftp_instance.mkd(base_dir)  # 不存在创建当前子目录 直到创建所有
+                    continue
+
+    # 03-上传目录
+    def upload_file_tree(self, local_path, remote_path, recursively):
+        # 创建服务器目录 如果服务器目录不存在 就从当前目录创建目标外层目录
+        # 打开该远程目录
+        self.instance_cwd(remote_path)
+
+        # 本地目录切换
+        try:
+            # 远端目录通过ftp对象已经切换到指定目录或创建的指定目录
+            file_list = os.listdir(local_path)
+            for file_name in file_list:
+                if os.path.isdir(os.path.join(local_path, file_name)):
+                    self.logger.debug('%s is a directory...' % file_name)
+                    if recursively:  # 递归目录上传
+                        # 创建相关的子目录 创建不成功则目录已存在
+                        try:
+                            cwd = self.ftp_instance.pwd()
+                            self.ftp_instance.cwd(file_name)  # 如果cwd成功 则表示该目录存在 退出到上一级
+                            self.ftp_instance.cwd(cwd)
+                        except Exception as e:
+                            self.logger.error(
+                                'check remote directory %s not eixst, now trying to create it! Except INFO:%s.' % (
+                                    file_name, e))
+                            self.ftp_instance.mkd(file_name)
+
+                        self.logger.debug('trying to upload directory %s --> %s ...' % (file_name, remote_path))
+                        p_local_path = os.path.join(local_path, file_name)
+                        p_remote_path = os.path.join(self.ftp_instance.pwd(), file_name)
+                        self.upload_file_tree(p_local_path, p_remote_path, recursively)
+                        # 对于递归 ftp 每次传输完成后需要切换目录到上一级
+                        self.ftp_instance.cwd("..")
+                    else:
+                        self.logger.debug(
+                            'translate mode is UnRecursively, %s is a directory, continue ...' % file_name)
+                        continue
+                else:
+                    # 是文件 直接上传
+                    local_file = os.path.join(local_path, file_name)
+                    remote_file = os.path.join(remote_path, file_name)
+                    self.upload_file(local_file, remote_file, self.ftp_instance)
+        except Exception as e:
+            traceback.format_exc()
+            self.logger.debug(f'FTP 无法上传目录。 错误消息:{e}')
+
+    # 04-核实iso名称
+    def ftp_check_iso(self):
+        upload_path = self.celery_task_param.get('upload_path')
+        iso_file = self.celery_task_param.get('isoname')
+        iso_rename = self.celery_task_param.get('filename_rename')
+
+        flist = self.ftp_instance.nlst(upload_path)
+        for f in flist:
+            if f.endswith('.iso') and f.split('/')[-1] == iso_file.split('/')[-1] or (
+                    f.endswith('.iso') and f.split('/')[-1] == iso_rename):
+                self.ftp_instance.quit()
+                return f
+        return None
+
+    # 外部调用ftp主函数入口
+    def upload_iso(self, root_path_iso_path="/opt/integration_iso_files/") -> str:
+        try:
+            self.process_rate_status = 50
+            self.create_remote_dir(self.celery_task_param.get('upload_path'))
+            self.log_and_send_task_status(
+                f'ftp上传路径:{self.celery_task_param.get("ftpaddress")}:{self.ftp_instance.pwd()}')
+
+            self.process_rate_status = 60
+
+            # 是否为空
+            if self.ftp_instance.nlst():
+                raise RuntimeError("目录不为空，无法上传")
+
+            self.log_and_send_task_status("ftp开始上传本地iso文件")
+            local_dir = root_path_iso_path + self.celery_task_param.get("isoname")[
+                                             :self.celery_task_param.get("isoname").rfind("/")]
+            self.upload_file_tree(local_dir, self.celery_task_param.get('upload_path'), True)
+
+            self.process_rate_status = 80
+            self.log_and_send_task_status("校验ftp远端iso文件")
+            if not self.ftp_check_iso():
+                self.log_and_send_task_status("ftp上传iso失败")
+
+            self.process_rate_status = 100
+            self.log_and_send_task_status("ftp上传iso完成")
+            return self.celery_task_param.get('upload_path')
+        except Exception as e:
+            self.process_rate_status = 0
+            self.log_and_send_task_status(f"ftp上传失败！{e}")
+        return ""

kyutil-0.2.1/kyutil/__init__.py

@@ -0,0 +1 @@
+# -*- coding: UTF-8 -*-

kyutil-0.2.1/kyutil/ansible.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+@Project ：kyutil 
+@File    ：ansible.py
+@IDE     ：PyCharm 
+@Author  ：xuyong@kylinos.cn
+@Date    ：2025/5/22 下午4:39 
+@Desc    ：说明：
+"""
+import base64
+import os
+import random
+import sys
+from shutil import copyfile
+
+import ansible_runner
+from logzero import logger as log
+
+
+def gen_random_base_64_str() -> bytes:
+    flag = "flag{**flag**}".encode("utf-8")
+    b64list = [lambda x: base64.b16encode(x), lambda x: base64.b32encode(x), lambda x: base64.b64encode(x)]
+    for _ in range(10):
+        choice = random.choice([0, 1, 2])
+        flag = b64list[choice](flag)
+    encoded = {
+        '16': lambda x: base64.b16encode(x),
+        '32': lambda x: base64.b32encode(x),
+        '64': lambda x: base64.b64encode(x)
+    }
+    choice = random.choice(['16', '32', '64'])
+    flag = encoded[choice](flag)
+    return flag[7:32]
+
+
+def execute_playbook(operation_type, host_ip, logger=log) -> bool:
+    bp_prefix = "/opt/ctdy/build_iso_server/src/app/bp/"
+    b64random = bytes.decode(gen_random_base_64_str())
+    host_file = f"{bp_prefix}ansible/hosts.{str(b64random)}"
+    random_file = f"{bp_prefix}ansible/{operation_type}.yaml." + str(b64random)
+
+    copyfile(f"{bp_prefix}ansible/{operation_type}.yaml", random_file)
+    copyfile(bp_prefix + "ansible/hosts", host_file)
+    ssh_pass_sk_raw = "RmExYzBuQEt5bGluMTAyMzYxCg=="
+    ssh_pass_sk = base64.b64decode(bytes(ssh_pass_sk_raw, 'utf-8')).decode()
+    with open(host_file, encoding="utf-8", mode="a") as f:
+        f.write("\n" + host_ip + " ansible_ssh_user=root ansible_ssh_pass=\"" + ssh_pass_sk.strip() + "\"")
+    out, err, rc = ansible_runner.run_command(
+        executable_cmd='ansible-playbook',
+        cmdline_args=[random_file, '-i', host_file, '--extra-vars', "ansible_become_pass=" + ssh_pass_sk],
+        input_fd=sys.stdin,
+        output_fd=sys.stdout,
+        error_fd=sys.stderr
+    )
+    os.remove(random_file)
+    os.remove(host_file)
+    if rc == 0:
+        logger.debug(f"Playbook执行成功:{out}. RC:{rc} ERR:{err}")
+        return True
+    else:
+        logger.error(f"Playbook执行失败:{err}. RC:{rc}")
+        return False
+
+
+def hosts_restart(host_list, logger=log):
+    oks, errs = [], []
+    for host in host_list:
+        ok = execute_playbook("restart", host.hostip)
+        if ok:
+            oks.append(host.hostip)
+        else:
+            errs.append(host.hostip)
+    return oks, errs