ryry-cli 1.0__py3-none-any.whl

ryry/task.py

@@ -0,0 +1,203 @@
+import os, time, calendar
+import json
+from urllib.parse import *
+import sys
+import signal
+import subprocess, multiprocessing
+from threading import Thread, current_thread, Lock
+
+from ryry import ryry_webapi
+from ryry import store
+from ryry import taskUtils
+from ryry import utils
+from pathlib import Path
+
+def runTask(it, timeout):
+    start_time = calendar.timegm(time.gmtime())
+    taskUUID = it["taskUUID"]
+    config = json.loads(it["config"])
+    params = json.loads(it["data"])
+    widget_id = config["widget_id"]
+    #cmd
+    cmd = cmdWithWidget(widget_id)
+    #params
+    params["task_id"] = taskUUID
+    #run
+    taskUtils.taskPrint(taskUUID, f"{current_thread().name}=== start execute task : {taskUUID}")
+    executeSuccess, result_obj = executeLocalPython(taskUUID, cmd, params, timeout)
+    #result
+    is_ok = executeSuccess and result_obj["status"] == 0
+    msg = ""
+    if len(result_obj["message"]) > 0:
+        msg = str(result_obj["message"])
+    if is_ok:
+        checkResult(taskUUID, result_obj)
+    taskUtils.taskPrint(taskUUID, f"{current_thread().name}=== task {taskUUID} is_ok={is_ok} ")
+    taskUtils.saveCounter(taskUUID, (calendar.timegm(time.gmtime()) - start_time), is_ok)
+    return is_ok, msg, json.dumps(result_obj["result"], separators=(',', ':'))
+
+def cmdWithWidget(widget_id):
+    map = store.widgetMap()
+    if widget_id in map:
+        path = ""
+        is_block = False
+        if isinstance(map[widget_id], (dict)):
+            is_block = map[widget_id]["isBlock"]
+            path = map[widget_id]["path"]
+        else:
+            is_block = False
+            path = map[widget_id]
+        if len(path) > 0 and is_block == False:
+            return path
+    return None
+
+def executeLocalPython(taskUUID, cmd, param, timeout):
+    inputArgs = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"{taskUUID}.in")
+    if os.path.exists(inputArgs):
+        os.remove(inputArgs)
+    with open(inputArgs, 'w') as f:
+        json.dump(param, f)
+    outArgs = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"{taskUUID}.out")
+    if os.path.exists(outArgs):
+        os.remove(outArgs)
+        
+    outData = {
+        "result" : [ 
+        ],
+        "status" : -1,
+        "message" : "script error"
+    }
+    executeSuccess = False
+    command = [sys.executable, cmd, "--run", inputArgs, "--out", outArgs]
+    taskUtils.taskPrint(taskUUID, f"{current_thread().name}=== exec => {command}")
+    process = None
+    try:
+        if timeout == 0:
+            timeout = 60*60 #max 1 hour expire time
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        timeout_killprocess(process, timeout)
+        output, error = process.communicate()
+        if process.returncode == 0:
+            taskUtils.taskPrint(taskUUID, output.decode(encoding="utf8", errors="ignore"))
+            if os.path.exists(outArgs) and os.stat(outArgs).st_size > 0:
+                try:
+                    with open(outArgs, 'r', encoding='UTF-8') as f:
+                        outData = json.load(f)
+                    executeSuccess = True
+                    taskUtils.taskPrint(taskUUID, f"[{taskUUID}]exec success result => {outData}")
+                except:
+                    taskUtils.taskPrint(taskUUID, f"[{taskUUID}]task result format error, please check => {outData}")
+            else:
+                taskUtils.taskPrint(taskUUID, f"[{taskUUID}]task result is empty!, please check {cmd}")
+        else:
+            taskUtils.taskPrint(taskUUID, f"====================== script error [{taskUUID}]======================")
+            o1 = output.decode(encoding="utf8", errors="ignore")
+            o2 = error.decode(encoding="utf8", errors="ignore")
+            error_msg = f"{o1}\n{o2}"
+            short_error_msg = ""
+            if len(error_msg) > 310:
+                short_error_msg = f"{error_msg[0:99]}\n...\n{error_msg[len(error_msg)-200:]}"
+            else:
+                short_error_msg = error_msg
+            outData["message"] = short_error_msg
+            taskUtils.taskPrint(taskUUID, error_msg)
+            taskUtils.taskPrint(taskUUID, "======================     end      ======================")
+            taskUtils.notifyScriptError(taskUUID)
+    except Exception as e:
+        time.sleep(1) 
+        taskUtils.taskPrint(taskUUID, f"====================== process error [{taskUUID}]======================")
+        taskUtils.taskPrint(taskUUID, e)
+        taskUtils.taskPrint(taskUUID, "======================      end      ======================")
+        if process:
+            os.kill(process.pid, signal.SIGTERM) 
+            if process.poll() is None:
+                os.kill(process.pid, signal.SIGKILL)  
+        taskUtils.notifyScriptError(taskUUID)
+        outData["message"] = str(e)
+    finally:
+        if process and process.returncode is None:
+            try:
+                print("kill -9 " + str(process.pid))
+                os.system("kill -9 " + str(process.pid))
+            except ProcessLookupError:
+                pass
+        if os.path.exists(inputArgs):
+            os.remove(inputArgs)
+        if os.path.exists(outArgs):
+            os.remove(outArgs)
+    return executeSuccess, outData
+
+def _needChangeValue(taskUUID, data, type, key):
+    if "type" not in data:
+        taskUtils.taskPrint(taskUUID, "result is not avalid")
+        return False
+    if data["type"] != type:
+        return False
+    if "extension" not in data or key not in data["extension"] or len(data["extension"][key]) == 0:
+        return True
+    return False
+            
+def checkResult(taskUUID, data):
+    try:
+        for it in data["result"]:
+            if "extension" not in it:
+                continue
+            if _needChangeValue(taskUUID, it, "text", "cover_url"):
+                it["extension"]["cover_url"] = ""
+            if _needChangeValue(taskUUID, it, "audio", "cover_url"):
+                it["extension"]["cover_url"] = ""
+            if _needChangeValue(taskUUID, it, "image", "cover_url"):
+                it["extension"]["cover_url"] = ""
+            if _needChangeValue(taskUUID, it, "video", "cover_url"):
+                it["extension"]["cover_url"] = ""
+                
+            if "cover_url" in it["extension"] and len(it["extension"]["cover_url"]) > 0:
+                cover_url = str(it["extension"]["cover_url"]).replace('\\u0026', '&')
+                parsed_url = urlparse(cover_url)
+                params = parse_qs(parsed_url.query)
+                #add width & height if need
+                if "width" not in params and "height" not in params:
+                    w, h = utils.getOssImageSize(cover_url)
+                    if w > 0 and h > 0:
+                        params["width"] = w
+                        params["height"] = h
+                        it["extension"]["width"] = w
+                        it["extension"]["height"] = h
+                #remove optional parameters
+                for k in ["Expires","OSSAccessKeyId","Signature","security-token"]:
+                    params.pop(k, None)
+                if "width" in it["extension"]:
+                    if isinstance(it["extension"]["width"], str):
+                        it["extension"]["width"] = int(it["extension"]["width"])
+                if "height" in it["extension"]:
+                    if isinstance(it["extension"]["height"], str):
+                        it["extension"]["height"] = int(it["extension"]["height"])
+                updated_query_string = urlencode(params, doseq=True)
+                final_url = parsed_url._replace(query=updated_query_string).geturl()
+                it["extension"]["cover_url"] = final_url
+    except Exception as ex:
+        taskUtils.taskPrint(taskUUID, f"result: {data} status is not valid, exception is {ex} ")
+        pass
+
+def updateProgress(data, progress=0.5, taskUUID=None):
+    realTaskUUID = taskUUID
+    if realTaskUUID == None or len(realTaskUUID) <= 0:
+        realTaskUUID = taskUtils.taskInfoWithFirstTask()
+        
+    if progress < 0:
+        progress = 0
+    if progress > 1:
+        progress = progress / 100.0
+    return ryry_webapi.TaskUpdateProgress(realTaskUUID, progress, json.dumps(data["result"]))
+
+def timeout_killprocess(proc, timeout): # """超过指定的秒数后杀死进程"""
+    import threading
+    timer = threading.Timer(timeout, lambda p: p.kill(), [proc])
+    try:
+        timer.start()
+        proc.communicate()
+    except Exception as e:
+        print(e)
+    finally:
+        timer.cancel()
+

ryry/taskUtils.py

@@ -0,0 +1,283 @@
+import os
+import requests
+import datetime
+import json
+import socket
+from requests_toolbelt import MultipartEncoder
+from urllib import parse
+import base64
+import hashlib
+from ryry import utils
+from pkg_resources import get_distribution
+
+task_config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)) , f"task_config.txt")
+def taskInfoWithFirstTask():
+    if os.path.exists(task_config_file):
+        with open(task_config_file, 'r') as f:
+            data = json.load(f)
+        for it in data:
+            if it not in ["last_task_pts"]:
+                return it
+    return None
+
+WECHAT_URL = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=f40c5fb0-2734-48b0-a2d0-0faedf7dd2e4"
+def uploadFile2Wechat(filepath):
+    real_robot_url = WECHAT_URL
+    params = parse.parse_qs( parse.urlparse( real_robot_url ).query )
+    webHookKey=params['key'][0]
+    upload_url = f'https://qyapi.weixin.qq.com/cgi-bin/webhook/upload_media?key={webHookKey}&type=file'
+    headers = {"Accept": "application/json, text/plain, */*", "Accept-Encoding": "gzip, deflate",
+               "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36"}
+    filename = os.path.basename(filepath)
+    try:
+        multipart = MultipartEncoder(
+            fields={'filename': filename, 'filelength': '', 'name': 'media', 'media': (filename, open(filepath, 'rb'), 'application/octet-stream')},
+            boundary='-------------------------acebdf13572468')
+        headers['Content-Type'] = multipart.content_type
+        resp = requests.post(upload_url, headers=headers, data=multipart, timeout=300)
+        json_res = resp.json()
+        if json_res.get('media_id'):
+            return json_res.get('media_id')
+    except Exception as e:
+        return ""
+def notifyWechatRobot(param):
+    real_robot_url = WECHAT_URL
+    try:
+        s = requests.session()
+        s.headers.update({'Connection':'close'})
+        headers = dict()
+        headers['Content-Type'] = "application/json"
+        res = s.post(real_robot_url, json.dumps(param), headers=headers, verify=False, timeout=30)
+        s.close()
+    except Exception as e:
+        print(f"===== qyapi.weixin.qq.com fail ", True)
+
+logs = {}
+def taskPrint(taskUUID, msg):
+    global logs
+    if (taskUUID == None or len(taskUUID) == 0) and msg == None:
+        return
+    if taskUUID and msg == None:
+        del logs[taskUUID]
+        return
+    if taskUUID and msg:
+        if taskUUID not in logs:
+            logs[taskUUID] = []
+        logs[taskUUID].append(msg)
+    if taskUUID == None and msg:
+        for uuid in logs:
+            logs[uuid].append(msg)
+    print(msg)
+def getTaskLog(taskUUID):
+    if taskUUID in logs:
+        return logs[taskUUID]
+    return []
+
+def _uploadLog(taskUUID):
+    try:
+        log_path = f"{os.path.dirname(os.path.abspath(__file__))}/log_{taskUUID}.log"
+        with open(log_path, 'w') as f:
+            f.write("\n".join(getTaskLog(taskUUID)))
+        notifyWechatRobot({
+            "msgtype": "file",
+            "file": {
+                "media_id": uploadFile2Wechat(log_path)
+            }
+        })
+        if os.path.exists(log_path):
+            os.remove(log_path)
+    except:
+        pass
+
+def notifyTaskFail(taskUUID, reason):
+    try:
+        real_reason = ""
+        if len(reason) > 610:
+            real_reason = f"{reason[0:300]}\n...\n{reason[len(reason)-300:]}"
+        else:
+            real_reason = reason
+        notifyWechatRobot({
+            "msgtype": "markdown",
+            "markdown": {
+                "content": f"机器<<font color=\"warning\">{socket.gethostname()}</font>> 执行任务<{taskUUID}>失败\n<{real_reason}>"
+            }
+        })
+        _uploadLog(taskUUID)
+    except:
+        pass
+
+def notifyServerError(taskUUID):
+    try:
+        notifyWechatRobot({
+            "msgtype": "markdown",
+            "markdown": {
+                "content": f"机器<<font color=\"warning\">{socket.gethostname()}</font>> 执行任务<{taskUUID}>上报失败, retry..."
+            }
+        })
+    except:
+        pass
+
+def notifyScriptError(taskUUID):
+    try:
+        notifyWechatRobot({
+            "msgtype": "markdown",
+            "markdown": {
+                "content": f"机器<<font color=\"warning\">{socket.gethostname()}</font>> 执行任务<{taskUUID}>异常"
+            }
+        })
+        _uploadLog(taskUUID)
+    except:
+        pass
+
+def idlingNotify(cnt):
+    device_id = utils.generate_unique_id()
+    machine_name = socket.gethostname()
+    hour = int(float(cnt)/(60.0*60.0))
+    if hour<72:
+        if hour not in [1, 2, 3, 10, 30, 50, 70]:
+            return
+    notifyWechatRobot({
+        "msgtype": "text",
+        "text": {
+            "content": f"机器<{machine_name}[{device_id}]> 空转{hour}小时"
+        }
+    })
+
+def onlineNotify():
+    device_id = utils.generate_unique_id()
+    machine_name = socket.gethostname()
+    ver = get_distribution("ryry-cli").version
+    notifyWechatRobot({
+        "msgtype": "text",
+        "text": {
+            "content": f"机器<{machine_name}[{device_id}]>[{ver}] 上线"
+        }
+    })
+
+def restartNotify(msg):
+    device_id = utils.generate_unique_id()
+    machine_name = socket.gethostname()
+    notifyWechatRobot({
+        "msgtype": "text",
+        "text": {
+            "content": f"机器<{machine_name}[{device_id}]> 即将下线，原因：{msg}"
+        }
+    })
+    
+def offlineNotify():
+    device_id = utils.generate_unique_id()
+    machine_name = socket.gethostname()
+    ver = get_distribution("ryry-cli").version
+    notifyWechatRobot({
+        "msgtype": "text",
+        "text": {
+            "content": f"机器<{machine_name}[{device_id}]> 下线"
+        }
+    })
+
+#===================================================== counter ===============================================#
+task_counter_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "task_counter.txt")
+def notifyCounterIfNeed():
+    if os.path.exists(task_counter_file) == False:
+        return
+    with open(task_counter_file, 'r') as f:
+        data = json.load(f)
+    now_hour = datetime.datetime.now().hour
+    if now_hour == 0 and len(data) > 2:
+        yesterday = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d')
+        s_cnt = 0
+        f_cnt = 0
+        all_day_usage = 0
+        t_l = []
+        s_l = []
+        f_l = []
+        for i in range(0,24):
+            tips = ""
+            if str(i) in data:
+                s_cnt += data[str(i)]["success"]
+                f_cnt += data[str(i)]["fail"]
+                s_l.append(data[str(i)]["success"])
+                f_l.append(data[str(i)]["fail"])
+                if "usage" in data[str(i)]:
+                    all_day_usage += data[str(i)]["usage"]
+                    usage_percentage = int((float(data[str(i)]["usage"])/float(60*60))*100)
+                    tips = f"({usage_percentage}%)"
+            else:
+                s_cnt += 0
+                f_cnt += 0
+                s_l.append(0)
+                f_l.append(0)
+            t_l.append(f"{i}{tips}")
+        usage_percentage = int((float(all_day_usage)/float(24*60*60))*100)
+        notifyWechatRobot({
+            "msgtype": "markdown",
+            "markdown": {
+                "content": f"机器<<font color=\"warning\">{socket.gethostname()}</font>> {yesterday} 日报 \n\n\
+                                >过去24小时执行任务<<font color=\"warning\">{s_cnt+f_cnt}</font>>个, 负载<<font color=\"warning\">{usage_percentage}%</font>> \n\
+                                >成功<font color=\"warning\">{s_cnt}</font>个 \n\
+                                >失败<font color=\"warning\">{f_cnt}</font>个"
+            }
+        })
+        
+        try:
+            import matplotlib.pyplot as plt
+            plt.figure(figsize=(8,3))
+            plt.rcParams.update({
+                'font.size': 7
+            })
+            plt.bar(t_l, s_l, color='g', label='success')
+            plt.bar(t_l, f_l, bottom=s_l, color='r', label='fail')
+            plt.title(f'[{socket.gethostname()}] [{yesterday}] success/fail={s_cnt}/{f_cnt}')
+            plt.xlabel('time')
+            plt.xticks(ticks=t_l,rotation=45)
+            plt.ylabel('count')
+            plt.subplots_adjust(bottom=0.25)
+            plt.legend()
+            fff = os.path.join(os.path.dirname(os.path.abspath(__file__)), "plt.png")
+            plt.savefig(fff)
+            with open(fff, "rb") as f:
+                encode_string = str(base64.b64encode(f.read()), encoding='utf-8')
+            md5 = hashlib.md5()
+            md5.update(base64.b64decode(encode_string))
+            hash = md5.hexdigest()
+            notifyWechatRobot({
+                "msgtype": "image",
+                "image": {
+                    "base64": encode_string,
+                    "md5": hash
+                }
+            })
+            os.remove(fff)
+            os.remove(task_counter_file)
+        except:
+            pass
+
+def saveCounter(taskUUID, duration, isSuccess):
+    try:
+        notifyCounterIfNeed()
+        if os.path.exists(task_counter_file) == False:
+            with open(task_counter_file, 'w') as f:
+                json.dump({}, f)
+        with open(task_counter_file, 'r') as f:
+            data = json.load(f)
+        #update
+        now_hour = str(datetime.datetime.now().hour)
+        if now_hour in data:
+            if isSuccess:
+                data[now_hour]["success"] += 1
+            else:
+                data[now_hour]["fail"] += 1
+            if "usage" not in data[now_hour]:
+                data[now_hour]["usage"] = 0
+            data[now_hour]["usage"] += duration
+        else:
+            data[now_hour] = {
+                "success" : 1 if isSuccess else 0,
+                "fail" : 0 if isSuccess else 1,
+                "usage" : 0
+            }
+        #save
+        with open(task_counter_file, 'w') as f:
+            json.dump(data, f)
+    except:
+        pass

ryry/upload.py

@@ -0,0 +1,139 @@
+from urllib.parse import *
+
+def addtionExif(srcFile, taskUUID):
+    if taskUUID == None or len(taskUUID) == 0:
+        return
+    try:
+        from pathlib import Path
+        file_name = Path(srcFile).name
+        ext = file_name[file_name.index("."):].lower()
+        if ext in [".jpg", ".png", ".jpeg", ".bmp", ".webp", ".gif"]:
+            from PIL import Image
+            img = Image.open(srcFile)
+            exif_dict = {
+                "0th": { }, 
+                "Exif": { }, 
+                "1st": { },
+                "thumbnail": None, 
+                "GPS": { }
+            }
+            import piexif
+            if taskUUID:
+                exif_dict["0th"] = { 
+                    piexif.ImageIFD.Software: f'make with ryry({taskUUID})'.encode(),
+                    piexif.ImageIFD.Copyright: f'dalipen'.encode(),
+                }
+                exif_dict["Exif"] = {
+                    piexif.ExifIFD.UserComment: f'make with ryry({taskUUID})'.encode(),
+                }
+            exif_dat = piexif.dump(exif_dict)
+            img.save(srcFile, "webp", quality=90, exif=exif_dat)
+        # elif ext in [".mp4",".mov",".avi",".wmv",".mpg",".mpeg",".rm",".ram",".flv",".swf",".ts"]:
+        #     params = {}
+        # elif ext in [".mp3",".aac",".wav",".wma",".cda",".flac",".m4a",".mid",".mka",".mp2",".mpa",".mpc",".ape",".ofr",".ogg",".ra",".wv",".tta",".ac3",".dts"]:
+        #     params = {}
+        # else:
+        #     params = {}
+    except:
+        return
+
+def transcode(srcFile):
+    try:
+        from pathlib import Path
+        from PIL import Image
+        file_name = Path(srcFile).name
+        ext = file_name[file_name.index("."):].lower()
+        if ext in [".jpg", ".png", ".jpeg", ".bmp"]:
+            image = Image.open(srcFile, "r")
+            format = image.format
+            if format.lower() != "webp":
+                fname = Path(srcFile).name
+                newFile = srcFile.replace(fname[fname.index("."):], ".webp")
+                image.save(newFile, "webp", quality=90)
+                image.close()
+                return True, newFile
+    except Exception as e:
+        pass
+    return False, srcFile
+
+def additionalUrl(srcFile, ossUrl):
+    from pathlib import Path
+    from PIL import Image
+    try:
+        file_name = Path(srcFile).name
+        ext = file_name[file_name.index("."):].lower()
+        params = {}
+        if ext in [".jpg", ".png", ".jpeg", ".bmp", ".webp", ".gif"]:
+            img = Image.open(srcFile)
+            params["width"] = img.width
+            params["height"] = img.height
+        elif ext in [".mp4",".mov",".avi",".wmv",".mpg",".mpeg",".rm",".ram",".flv",".swf",".ts"]:
+            params = {}
+        elif ext in [".mp3",".aac",".wav",".wma",".cda",".flac",".m4a",".mid",".mka",".mp2",".mpa",".mpc",".ape",".ofr",".ogg",".ra",".wv",".tta",".ac3",".dts"]:
+            params = {}
+        else:
+            params = {}
+        parsed_url = urlparse(ossUrl)
+        updated_query_string = urlencode(params, doseq=True)
+        final_url = parsed_url._replace(query=updated_query_string).geturl()
+        return final_url
+    except:
+        return ossUrl
+
+def upload(src, taskUUID, timeout=300):
+    import os
+    from ryry import store
+    from pathlib import Path
+    from ryry import taskUtils
+    from ryry import ryry_webapi
+    import requests
+    if os.path.exists(src) == False:
+        raise Exception(f"upload file not found")
+    if taskUUID==None or len(taskUUID) <= 0:
+        taskUUID = taskUtils.taskInfoWithFirstTask()
+
+    needDeleteSrc, newSrc = transcode(src)
+    addtionExif(newSrc, taskUUID)
+    file_name = Path(newSrc).name
+    ossurl, content_type = ryry_webapi.GetOssUrl(os.path.splitext(file_name)[-1][1:])
+    if len(ossurl) == 0:
+        raise Exception(f"oss server is not avalid, msg = {content_type}")
+
+    headers = dict()
+    headers['Content-Type'] = content_type
+    requests.adapters.DEFAULT_RETRIES = 3
+    s = requests.session()
+    s.keep_alive = False
+    res = s.put(ossurl, data=open(newSrc, 'rb').read(), headers=headers, timeout=timeout)
+    s.close()
+    if res.status_code == 200:
+        ossurl = additionalUrl(newSrc, ossurl)
+        if needDeleteSrc:
+            os.remove(newSrc)
+        return ossurl
+    else:
+        raise Exception(f"upload file fail! res = {res}")
+
+def uploadWidget(src, widgetid, timeout=300):
+    from ryry import ryry_webapi
+    import requests
+    ossurl, content_type = ryry_webapi.GetWidgetOssUrl(widgetid)
+    if len(ossurl) == 0:
+        raise Exception("oss server is not avalid")
+    
+    headers = dict()
+    headers['Content-Type'] = content_type
+    requests.adapters.DEFAULT_RETRIES = 3
+    s = requests.session()
+    s.keep_alive = False
+    res = s.put(ossurl, data=open(src, 'rb').read(), headers=headers, timeout=timeout)
+    s.close()
+    if res.status_code == 200:
+        ossurl = additionalUrl(src, ossurl)
+        checkid = ryry_webapi.WidgetUploadEnd(ossurl)
+        if checkid > 0:
+            return ossurl, checkid
+        else:
+            raise Exception("check fail!")
+    else:
+        raise Exception(f"upload file fail! res = {res}")