ryry-cli 1.0__py3-none-any.whl

ryry/ryry_service.py

@@ -0,0 +1,220 @@
+import sys, os, time, signal, subprocess, logging, json, platform, socket, calendar
+from logging.handlers import RotatingFileHandler
+from urllib.parse import *
+from datetime import datetime, timedelta
+from threading import Thread
+from pkg_resources import get_distribution
+
+from ryry import store
+from ryry import ryry_webapi
+from ryry import utils
+from ryry import taskUtils
+from ryry import ryry_widget
+from ryry import ryry_server_socket as TaskConnector
+
+thisFileDir = os.path.dirname(os.path.abspath(__file__)) 
+pid_file = os.path.join(thisFileDir, "ryryService.pid")
+stop_file = os.path.join(thisFileDir, "stop.now")
+stop_thread_file = os.path.join(thisFileDir, "stop.thread")
+all_thread_stoped = os.path.join(thisFileDir, "all_stoped.now")
+def notify_other_stoped():
+    with open(all_thread_stoped, 'w') as f:
+        f.write("")
+
+class LogStdout(object):
+    def __init__(self):
+        self.stdout = sys.stdout
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(logging.INFO)
+
+        logFilePath = f"{os.path.dirname(os.path.abspath(__file__))}/log.log"
+        file_handler = RotatingFileHandler(logFilePath, maxBytes=1024*1024*20, backupCount=30)
+        file_handler.setLevel(logging.INFO)
+        formatter = logging.Formatter(fmt='%(asctime)s - %(message)s', 
+                                    datefmt='%Y-%m-%d %H:%M:%S')
+        file_handler.setFormatter(formatter)
+        self.logger.addHandler(file_handler)
+
+    def write(self, message):
+        if message != '\n':
+            self.logger.info(message)
+        self.stdout.write(message)
+
+    def flush(self):
+        self.stdout.flush()
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        sys.stdout = self.stdout
+
+class ryryService:
+    def __init__(self):
+        self.THEADING_LIST = []
+
+    def start(self, threadNum=1):
+        if os.path.exists(pid_file):
+            #check pre process is finish successed!
+            with open(pid_file, 'r') as f:
+                pre_pid = str(f.read())
+            if len(pre_pid) > 0:
+                if utils.process_is_zombie_but_cannot_kill(int(pre_pid)):
+                    print(f'start service fail! pre process {pre_pid} is uninterruptible sleep')
+                    taskUtils.notifyWechatRobot({
+                        "msgtype": "text",
+                        "text": {
+                            "content": f"机器<{socket.gethostname()}>无法启动服务 进程<{pre_pid}>为 uninterruptible sleep"
+                        }
+                    })
+                    return False
+        #1: service init 
+        sys.stdout = LogStdout()
+        with open(pid_file, 'w') as f:
+            f.write(str(os.getpid()))
+        signal.signal(signal.SIGTERM, self.stop)
+        store.save_multithread(threadNum)
+        store.writeDeviceInfo(utils.deviceInfo())
+        TaskConnector._clearTask()
+        #2: service step
+        executor = TaskConnector.ryryTaskExecutorThread()
+        self.THEADING_LIST.append(executor)
+        for _ in range(0, threadNum):
+            self.THEADING_LIST.append(TaskConnector.ryryShortConnectThread(executor))
+        self.THEADING_LIST.append(ryryStateThread())
+        self.THEADING_LIST.append(ryryPackageThread())
+        #3: service step
+        while (os.path.exists(stop_file) == False):
+            time.sleep(10)
+        print("prepare stop")
+        with open(stop_thread_file, 'w') as f:
+            f.write("")
+        for t in self.THEADING_LIST:
+            t.markStop()
+        for t in self.THEADING_LIST:
+            t.join()
+        if pid_file and os.path.exists(pid_file):
+            os.remove(pid_file)
+        #4: clean
+        if os.path.exists(stop_thread_file):
+            os.remove(stop_thread_file)
+        if os.path.exists(stop_file):
+            os.remove(stop_file)
+        taskUtils.offlineNotify()
+        print("ryryService has ended!")
+        utils.check_restart()
+        sys.stdout.close()
+
+    def is_running(self):
+        if pid_file and os.path.exists(pid_file):
+            with open(pid_file, 'r', encoding='UTF-8') as f:
+                pid = int(f.read())
+                try:
+                    if utils.process_is_alive(pid):
+                        return True
+                    else:
+                        return False
+                except OSError:
+                    return False
+        else:
+            return False
+        
+    def stop(self, signum=None, frame=None):
+        with open(stop_file, 'w') as f:
+            f.write("")
+        print("ryryService waiting stop...")
+        taskUtils.restartNotify("手动")
+        while os.path.exists(stop_file):
+            time.sleep(1)
+        print("ryryService has ended!")
+    
+class ryryPackageThread(Thread):
+    def __init__(self):
+        super().__init__()
+        self.name = f"ryryPackageThread"
+        if platform.system() == 'Windows':
+            self.time_task_file = os.path.join(thisFileDir, "update_ryry.bat")
+        elif platform.system() == 'Linux' or platform.system() == 'Darwin':
+            self.time_task_file = os.path.join(thisFileDir, "update_ryry.sh")
+        if os.path.exists(self.time_task_file):
+            os.remove(self.time_task_file)
+        self.last_check_time = calendar.timegm(time.gmtime())
+        self.start()
+    def getCommandResult(self, cmd):
+        try:
+            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            if result.returncode == 0:
+                return result.stdout.decode(encoding="utf8", errors="ignore").replace("\n","").strip()
+        except subprocess.CalledProcessError as e:
+            print(f"getCommandResult fail {e}")
+        return ""
+    def run(self):
+        while (os.path.exists(stop_thread_file) == False):
+            time.sleep(10)
+            if calendar.timegm(time.gmtime()) - self.last_check_time > 300:
+                self.last_check_time = calendar.timegm(time.gmtime())
+                try:
+                    #update widget
+                    ryry_widget.UpdateWidgetFromPypi()
+                except Exception as ex:
+                    print(f'update widget fail, {ex}')
+
+                try:
+                    #update cli
+                    remote_version = ryry_widget._remote_package_version("ryry-cli")
+                    simple = "https://pypi.python.org/simple/"
+                    local_version = ryry_widget._local_package_version("ryry-cli")
+                    if ryry_widget.compare_versions(remote_version, local_version) > 0:
+                        print("start update progress...")
+                        utils.begin_restart("auto upgrade ryry-cli", True, simple)
+                        device_id = utils.generate_unique_id()
+                        machine_name = socket.gethostname()
+                        ver = get_distribution("ryry-cli").version
+                        taskUtils.notifyWechatRobot({
+                            "msgtype": "text",
+                            "text": {
+                                "content": f"机器<{machine_name}[{device_id}]>[{ver}] ryry-cli开始升级[{local_version}]->[{remote_version}]"
+                            }
+                        })
+                        break
+                except Exception as ex:
+                    print(f'update ryry-cli fail, {ex}')
+            time.sleep(10)
+        print(f"   PackageChecker stop")
+        notify_other_stoped() #because other thread is waiting some signal to close
+    def markStop(self):
+        print(f"   PackageChecker waiting stop")
+
+class ryryStateThread(Thread):
+    def __init__(self):
+        super().__init__()
+        self.name = f"ryryStateThread"
+        self.daemon = True
+        self.tik_time = 30.0
+        self.start()
+    def run(self):
+        taskUtils.onlineNotify()
+        while (os.path.exists(stop_thread_file) == False):
+            time.sleep(self.tik_time)
+            try:
+                task_config = TaskConnector._getTaskConfig()
+                if task_config["last_task_pts"] > 0:
+                    cnt = (calendar.timegm(time.gmtime()) - task_config["last_task_pts"]) #second
+                    if cnt >= (60*60) and cnt/(60*60)%1 <= self.tik_time/3600:
+                        taskUtils.idlingNotify(cnt)
+                        #clear trush
+                        for root,dirs,files in os.walk(thisFileDir):
+                            for file in files:
+                                if file.find(".") <= 0:
+                                    continue
+                                ext = file[file.rindex("."):]
+                                if ext in [ ".in", ".out" ]:
+                                    os.remove(os.path.join(thisFileDir, file))
+                            if root != files:
+                                break
+            except:
+                time.sleep(60)
+        print(f"   StateChecker stop")
+    def markStop(self):
+        print(f"   StateChecker waiting stop")
+        

ryry/ryry_webapi.py

@@ -0,0 +1,251 @@
+import json, os
+import datetime
+import uuid, requests
+import calendar, time
+
+from ryry import store 
+from ryry import utils 
+from ryry import taskUtils 
+from ryry import constant 
+
+def _aigc_post(func, params, files={}, timeout=10):
+    s = requests.session()
+    try:
+        requests.adapters.DEFAULT_RETRIES = 2
+        s.keep_alive = False
+        s.headers.update({'Connection':'close'})
+        s.headers.update({'Usertoken':"0cce7cfe3629e16ee9b3ea8563305d3a15c524804da1861db07c2c867d79da63"})
+        if len(files) <= 0:
+            s.headers.update({'Content-Type':'application/json'})
+        res = s.post(url=f"https://api.dalipen.com/{func}", json=params, files=files, timeout=timeout, verify=False)
+        if res.status_code == 200:
+            result_data = json.loads(res.content)
+            res.close()
+            if result_data["code"] == 0:
+                return 0, "", result_data["data"]
+            else:
+                return result_data["code"], result_data["msg"], ""
+        else:
+            raise Exception(f'''request {func} status_code={res.status_code}''')
+    except Exception as e:
+        taskUtils.taskPrint(None, f'''=============== ryry {func} server error {datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')} ===============
+url={func}
+
+req=
+{params}
+exception={e}
+===============
+''')
+        return -1, str(e), ""
+    finally:
+        s.close()
+    return -1, "", ""
+    
+#======================================== Task Function ==============================
+def _extend():
+    extInfo = store.readDeviceInfo()
+    extInfo["app_version"] = constant.app_version
+    extInfo["app_bulld_number"] = constant.app_bulld_number
+    extInfo["app_name"] = constant.app_name
+    extInfo["dts"] = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S_%f')
+    extInfo["trace_id"] = ''.join(str(uuid.uuid4()).split('-'))
+    extInfo["host_name"] = utils.get_hostname()
+    return json.dumps(extInfo)
+
+def GetTask():
+    widget_map = ["TESTTEST"]
+    map = store.widgetMap()
+    for it in map:
+        if isinstance(map[it], (dict)):
+            if map[it]["isBlock"] == False:
+                widget_map.append(it)
+        else:
+            widget_map.append(it)
+    req = {
+        "widget_map": json.dumps(widget_map),
+        "limit": 1,
+    }
+    r1, r2, r3 = _aigc_post("aigc/task/getTask", req)
+    if r1 != 0:
+        return [], 10
+    datas = []
+    for it in r3:
+        datas.append({
+            "taskUUID": it["task_uuid"],
+            "config": it["config"],
+            "data": it["data"],
+        })
+    #Previous tasks may have failed due to network problems, so collect and resend
+    retryLastTaskNotify()
+    return datas, 600
+
+TASK_NOTIFY_DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"task_notify_data.json")
+def saveTaskNotifyData(taskUUID, status, msg, dataStr):
+    taskUtils.taskPrint(taskUUID, f"save {taskUUID} to next notify")
+    data = []
+    try:
+        if not os.path.exists(TASK_NOTIFY_DATA):
+            with open(TASK_NOTIFY_DATA, 'w') as f:
+                json.dump([], f)
+        with open(TASK_NOTIFY_DATA, 'r') as f:
+            data = json.load(f)
+    except:
+        pass
+    try:
+        data.append({
+            "taskUUID":taskUUID,
+            "status":status,
+            "msg":msg,
+            "dataStr":dataStr,
+            "pts": calendar.timegm(time.gmtime())
+        })
+        with open(TASK_NOTIFY_DATA, 'w') as f:
+            json.dump(data, f)
+    except:
+        pass
+def resetLastTaskNotify(taskUUID):
+    data = []
+    try:
+        if os.path.exists(TASK_NOTIFY_DATA):
+            with open(TASK_NOTIFY_DATA, 'r') as f:
+                data = json.load(f)
+    except:
+        pass
+    try:
+        newData = []
+        for it in data:
+            if it["taskUUID"] != taskUUID:
+                newData.append(it)
+        with open(TASK_NOTIFY_DATA, 'w') as f:
+            json.dump(newData, f)
+    except:
+        pass
+def retryLastTaskNotify():
+    data = []
+    try:
+        if os.path.exists(TASK_NOTIFY_DATA):
+            with open(TASK_NOTIFY_DATA, 'r') as f:
+                data = json.load(f)
+    except:
+        pass
+    try:
+        newData = []
+        for it in data:
+            msg = it["msg"]
+            fail_count = 0
+            if "fail_count" in it:
+                fail_count = it["fail_count"]
+            else:
+                it["fail_count"] = 0
+            pts = 0
+            if "pts" in it:
+                pts = it["pts"]
+            if abs(calendar.timegm(time.gmtime())-pts) < 10: #notify late after 10 sec
+                newData.append(it)
+                continue
+            if fail_count < 10:
+                if TaskNotify(it["taskUUID"], it["status"], f"{msg} *", it["dataStr"], False) == False:
+                    it["fail_count"] += 1
+                    newData.append(it)
+        with open(TASK_NOTIFY_DATA, 'w') as f:
+            json.dump(newData, f)
+    except:
+        pass
+def TaskNotify(taskUUID, status, msg, dataStr, failSaveNotify=True):
+    req = {
+        "task_uuid": taskUUID,
+        "status": status,
+        "msg": msg,
+        "dataStr": dataStr
+    }
+    r1, r2, r3 = _aigc_post("aigc/task/taskNotify", req)
+    if r1 != 0:
+        if failSaveNotify:
+            #tasks may have failed due to network problems, so collect and resend
+            saveTaskNotifyData(taskUUID, status, msg, dataStr)
+        return False
+    return r3
+  
+retryLastTaskNotify()
+def TaskUpdateProgress(taskUUID, progress, dataStr):
+    # req = aigc_ext_pb2.TaskUpdateReq()
+    # req.taskUUID = taskUUID
+    # req.progress = progress
+    # req.data = dataStr
+    
+    # rsp = aigc_ext_pb2.TaskUpdateRes()
+    # r1, r2, r3 = _aigc_post(req, "TaskUpdate")
+    # if r1 != 0:
+    #     return False
+    # rsp.ParseFromString(r3)
+    return True
+
+def upload(localFilePath, ext):
+    with open(localFilePath, "rb") as f:
+        files = { 'file':f }
+        params = { 'ext':ext }
+        r1, r2, r3 = _aigc_post("upload/create", params, files=files,timeout=100)
+    if r1 != 0:
+        raise Exception(f"upload fail!, reason={r2}")
+    return r3
+
+def uploadWidget(widget_id, name, whl, ext, version):
+    with open(whl, "rb") as f:
+        files = { 'file':f }
+        params = {
+            'widget_id':widget_id,
+            'name':name,
+            'version':version,
+            'ext':ext
+        }
+        r1, r2, r3 = _aigc_post("aigc/task/publishWidget", params, files=files, timeout=100)
+    if r1 != 0:
+        raise Exception(f"upload fail!, reason={r2}")
+    return r3
+
+def CreateWidgetUUID():
+    req = {}
+    r1, r2, r3 = _aigc_post("aigc/task/createWidget", req)
+    if r1 != 0:
+        raise Exception(f"CreateWidgetUUID fail!, reason={r2}")
+    return r3
+
+#======================================== Task Function ==============================
+def createTask(widget_id, params):
+    req = {
+        "widget_id": widget_id,
+        "widget_data": json.dumps(params),
+        "extend": _extend()
+    }
+    r1, r2, r3 = _aigc_post("aigc/task/createTask", req)
+    if r1 != 0:
+        raise Exception(f"create task fail!, reason={r2}")
+    return r3
+
+def findWidget(name):
+    req = {
+        "name": name
+    }
+    r1, r2, r3 = _aigc_post("aigc/task/findWidget", req)
+    if r1 != 0:
+        return 0
+    for it in r3:
+        widget_id = it["widget_id"]
+        widget_name = it["name"]
+        if widget_name.strip().lower() == name.strip().lower():
+            return widget_id
+    return 0
+
+def checkTask(checkUUID):
+    req = {
+        "task_uuid": checkUUID
+    }
+    r1, r2, r3 = _aigc_post("aigc/task/checkTask", req)
+    if r1 != 0:
+        return False, False, "server fail"
+    if r3["status"] < 3:
+        return False, False, ""
+    elif r3["status"] == 3:
+        return True, True, json.loads(r3["task_result"])
+    elif r3["status"] == 4:
+        return True, False, r3["fail_reason"]