cloudpss 4.0.2__py3-none-any.whl → 4.1.1a1__py3-none-any.whl

cloudpss/__init__.py

@@ -6,17 +6,16 @@ from .model import Model, ModelRevision, ModelTopology
 from .project import Project
 from .utils import MatlabDataEncoder, DateTimeEncode
 from . import function
-
+from .job import Job
 from .function import FunctionExecution
 from .version import __version__
 __all__ = [
     'setToken', 'Model', 'ModelRevision', 'ModelTopology', 'Runner', 'Result',
     'PowerFlowResult', 'EMTResult', 'MatlabDataEncoder', 'DateTimeEncode',
-    'function', 'Project', 'currentJob', 'IESLabSimulation', 'IESLabPlan','__version__'
+    'function', 'Project', 'currentJob', 'IESLabSimulation', 'IESLabPlan','__version__','Job'
 ]
 
 
-
 def currentJob():
     """
         获取当前的 currentExecution 实例

cloudpss/function/functionExecution.py

@@ -163,7 +163,7 @@ class FunctionExecution(object):
         sys.exit(code)
 
     def print(self, data):
-        print(json.dumps(data, cls=MatlabDataEncoder), flush=True)
+        print(json.dumps(data, cls=MatlabDataEncoder)+'\n', flush=True)
 
     def log(self, content, level='info', html=False, key=None):
         '''

cloudpss/job/job.py

@@ -1,6 +1,10 @@
+import asyncio
 import random
 import re
 import time
+from cloudpss.utils.AsyncIterable import CustomAsyncIterable
+
+from cloudpss.utils.httpAsyncRequest import graphql_fetch
 from .view import getViewClass
 
 from cloudpss.utils.IO import IO
@@ -12,20 +16,33 @@ from .jobMachine import JobMachine
 from .messageStreamSender import MessageStreamSender
 
 
-
 class Job(object):
     """docstring for Job"""
 
-    def __init__(self, id, args, createTime, startTime, endTime, status,
-                 context, user, priority, policy, machine, input, output,
-                 position):
+    def __init__(
+        self,
+        id,
+        args,
+        createTime,
+        startTime,
+        endTime,
+        status,
+        context,
+        user,
+        priority,
+        policy,
+        machine,
+        input,
+        output,
+        position,
+    ):
         super(Job, self).__init__()
         self.id = id
         self.args = args
         self.createTime = createTime
         self.startTime = startTime
         self.endTime = endTime
-        self.status = status
+        self.job_status = status #这里的status字段与原本的status()冲突
         self.context = context
         self.user = user
         self.priority = priority
@@ -36,15 +53,16 @@ class Job(object):
         self.position = position
         self.__receiver = None
         self.__sender = None
+        self._result = None
 
     @staticmethod
-    def fetch(id):
+    async def fetch(id):
         """
         获取job信息
         """
         if id is None:
-            raise Exception('id is None')
-        query = '''query($_a:JobInput!){
+            raise Exception("id is None")
+        query = """query($_a:JobInput!){
             job(input:$_a){
                 id
                 args
@@ -79,165 +97,103 @@ class Job(object):
                 output
                 position
             }
-        }'''
+        }"""
         variables = {"_a": {"id": id}}
-        r = graphql_request(query, variables)
-        if 'errors' in r:
-            raise Exception(r['errors'])
-        return Job(**r['data']['job'])
+
+        r = await graphql_fetch(query, variables)
+        if "errors" in r:
+            raise Exception(r["errors"])
+        return Job(**r["data"]["job"])
 
     @staticmethod
-    def fetchMany(input):
+    def fetchMany(*args):
         """
         批量获取任务信息
         """
-        if input is None:
-            raise Exception('input is None')
-        query = '''query($_a:JobsInput!){
-            jobs(input:$_a){
-                item {
-                    id
-                    args
-                    createTime
-                    startTime
-                    endTime
-                    status
-                    context
-                    user
-                    priority
-                    policy  {
-                        id 
-                        name 
-                        users 
-                        functions 
-                        tres {
-                            cpu
-                            ecpu
-                            mem
-                        } 
-                        minPriority 
-                        maxPriority 
-                        maxDuration 
-                        createTime 
-                        updateTime 
-                        visibility 
-                        queue
-                    }
-                    machine {
-                        id
-                        name
-                        tres {
-                            cpu
-                            ecpu
-                            mem
-                        }
-                        valid
-                    }
-                    input
-                    output
-                    position
-                } 
-                cursor 
-                count 
-                total
-                }
-            }'''
-        variables = {"_a": input}
-        r = graphql_request(query, variables)
-        if 'errors' in r:
-            raise Exception(r['errors'])
-        jobs = []
-        for job in r['data']['jobs']['item']:
-            jobs.append(Job(**job))
+        jobs = CustomAsyncIterable(Job.fetch,*args)
         return jobs
+    
 
     @staticmethod
-    def create(revisionHash,
-               job,
-               config,
-               name=None,
-               rid='',
-               policy=None,
-               **kwargs):
-        '''
-            创建一个运行任务
+    async def create(revisionHash, job, config, name=None, rid="", policy=None, **kwargs):
+        """
+        创建一个运行任务
 
-            :params: revision 项目版本号
-            :params: job 调用仿真时使用的计算方案，为空时使用项目的第一个计算方案
-            :params: config 调用仿真时使用的参数方案，为空时使用项目的第一个参数方案
-            :params: name 任务名称，为空时使用项目的参数方案名称和计算方案名称
-            :params: rid 项目rid，可为空
+        :params: revision 项目版本号
+        :params: job 调用仿真时使用的计算方案，为空时使用项目的第一个计算方案
+        :params: config 调用仿真时使用的参数方案，为空时使用项目的第一个参数方案
+        :params: name 任务名称，为空时使用项目的参数方案名称和计算方案名称
+        :params: rid 项目rid，可为空
 
-            :return: 返回一个运行实例
+        :return: 返回一个运行实例
 
-            >>> runner = Runner.runRevision(revision,job,config,'')
-        '''
+        >>> runner = Runner.runRevision(revision,job,config,'')
+        """
 
-        #处理policy字段
+        # 处理policy字段
         if policy is None:
             policy = {}
-            if policy.get('tres', None) is None:
-                policy['tres'] = {}
-            policy['queue'] = job['args'].get('@queue', 1)
-            policy['priority'] = job['args'].get('@priority', 0)
-            tres = {'cpu': 1, 'ecpu': 0, 'mem': 0}
-            tresStr = job['args'].get('@tres', '')
-            for t in re.split('\s+', tresStr):
-                if t == '':
+            if policy.get("tres", None) is None:
+                policy["tres"] = {}
+            policy["queue"] = job["args"].get("@queue", 1)
+            policy["priority"] = job["args"].get("@priority", 0)
+            tres = {"cpu": 1, "ecpu": 0, "mem": 0}
+            tresStr = job["args"].get("@tres", "")
+            for t in re.split("\s+", tresStr):
+                if t == "":
                     continue
-                k, v = t.split('=')
-                tres[k] = float(v)  #type: ignore
-            policy['tres'] = tres
+                k, v = t.split("=")
+                tres[k] = float(v)  # type: ignore
+            policy["tres"] = tres
 
-        query = '''mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}'''
-        function = job['rid'].replace('job-definition/cloudpss/',
-                                      'function/CloudPSS/')
+        query = """mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}"""
+        function = job["rid"].replace("job-definition/cloudpss/", "function/CloudPSS/")
         variables = {
-            'input': {
+            "input": {
                 "args": {
-                    **job['args'], '_ModelRevision': revisionHash,
-                    '_ModelArgs': config['args']
+                    **job["args"],
+                    "_ModelRevision": revisionHash,
+                    "_ModelArgs": config["args"],
                 },
                 "context": [
                     function,
                     rid,
-                    f"model/@sdk/{str(int(time.time() * random.random()))}"
+                    f"model/@sdk/{str(int(time.time() * random.random()))}",
                 ],
-                "policy":
-                policy,
+                "policy": policy,
             }
         }
-        r = graphql_request(query, variables)
-        if 'errors' in r:
-            raise Exception(r['errors'])
-        id = r['data']['job']['id']
-        return Job.fetch(id)
+        r = await graphql_fetch(query, variables)
+        if "errors" in r:
+            raise Exception(r["errors"])
+        id = r["data"]["job"]["id"]
+        return await Job.fetch(id)
 
     @staticmethod
-    def abort(id, timeout):
+    async def abort(id, timeout):
         """
         结束当前运行的算例
 
         """
-        query = '''mutation ($input: AbortJobInput!) {
+        query = """mutation ($input: AbortJobInput!) {
             job: abortJob(input: $input) {
                 id
                 status
             }
         }
-        '''
-        variables = {'input': {'id': id, 'timeout': timeout}}
-        graphql_request(query, variables)
+        """
+        variables = {"input": {"id": id, "timeout": timeout}}
+        await graphql_fetch(query, variables)
 
     @staticmethod
-    def load(file, format='yaml'):
+    def load(file, format="yaml"):
         return IO.load(file, format)
 
     @staticmethod
-    def dump(job, file, format='yaml', compress='gzip'):
+    def dump(job, file, format="yaml", compress="gzip"):
         return IO.dump(job, file, format, compress)
 
-    def read(self, receiver=None, dev=False, **kwargs):
+    async def read(self, receiver=None, dev=False, **kwargs):
         """
         使用接收器获取当前运行实例的输出
         """
@@ -245,33 +201,40 @@ class Job(object):
             self.__sender = receiver
         if self.__receiver is None:
             self.__receiver = MessageStreamReceiver(self, dev)
-        self.__receiver.connect(**kwargs)
+        await self.__receiver.connect(**kwargs)
         return self.__receiver
 
-    def write(self,sender=None, dev=False, **kwargs)->MessageStreamSender:
+    async def write(self, sender=None, dev=False, **kwargs) -> MessageStreamSender:
         """
-            使用发送器为当前运行实例输入
+        使用发送器为当前运行实例输入
         """
-        
+
         if sender is not None:
             self.__sender = sender
         if self.__sender is None:
-            self.__sender =MessageStreamSender(self,dev)
-        self.__sender.connect(**kwargs)
+            self.__sender = MessageStreamSender(self, dev)
+        await self.__sender.connect(**kwargs)
         return self.__sender
     
+    def status(self):
+        if self.__receiver is not None:
+            return self.__receiver.status
+        return 0
+
     @property
     def result(self):
         """
         获取当前运行实例的输出
         """
-        viewType = getViewClass(self.context[0])
-        return self.view(viewType)
+        if self._result is None:
+            viewType = getViewClass(self.context[0])
+            self._result = asyncio.run(self.view(viewType))
+        return self._result
 
-    def view(self, viewType):
+    async def view(self, viewType):
         """
         获取当前运行实例的输出
         """
-        receiver = self.read()
-        sender=self.write()
-        return viewType(receiver,sender)
+        receiver = await self.read()
+        sender = await self.write()
+        return viewType(receiver, sender)

cloudpss/job/messageStreamReceiver.py

@@ -1,35 +1,30 @@
 import logging
+
+from cloudpss.utils.httpAsyncRequest import websocket_connect
 from .jobReceiver import JobReceiver
 import os
 from urllib.parse import urlparse
-import websocket
 import pytz
-import threading
-import time
-utc_tz = pytz.timezone('UTC')
+
+utc_tz = pytz.timezone("UTC")
 
 from ..utils.IO import IO
 
 
 class Message(object):
-
     def __init__(self, id, token):
         self.id = id
         self.token = token
-        
 
 
 class MessageStreamReceiver(JobReceiver):
-
     def __init__(self, job, dev=False):
         super().__init__()
         self.job = job
         self.dev = dev
-        self.origin = os.environ.get('CLOUDPSS_API_URL',
-                                     'https://cloudpss.net/')
-        self.__hasOpen=False
+        self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
 
-    def receive(self, id, fr0m, on_open, on_message, on_error, on_close):
+    async def receive(self, id, fr0m, on_open, on_message, on_error, on_close):
         """
         读取消息流中的数据
         id: 消息流id
@@ -40,28 +35,31 @@ class MessageStreamReceiver(JobReceiver):
         on_close: 连接关闭时的回调函数
         """
         if id is None:
-            raise Exception('id is None')
+            raise Exception("id is None")
         u = list(urlparse(self.origin))
-        head = 'wss' if u[0] == 'https' else 'ws'
+        head = "wss" if u[0] == "https" else "ws"
 
-        path = head + '://' + str(u[1]) + '/api/streams/id/' + id
+        path = head + "://" + str(u[1]) + "/api/streams/id/" + id
         if fr0m is not None:
-            path = path + '&from=' + str(fr0m)
+            path = path + "&from=" + str(fr0m)
         logging.info(f"receive data from websocket: {path}")
-        ws = websocket.WebSocketApp(path,
-                                    on_open=on_open,
-                                    on_message=on_message,
-                                    on_error=on_error,
-                                    on_close=on_close)
-        
+        ws = await websocket_connect(
+            path,
+            open_func=on_open,
+            receive_func=on_message,
+            error_func=on_error,
+            closed_func=on_close,
+        )
+
         return ws
-    
+
     ###下面是兼容Receiver部分功能实现
 
-    def on_message(self, ws, message):
-        data = IO.deserialize(message, 'ubjson')
-        msg = IO.deserialize(data['data'], 'ubjson')
-        if msg.get('type', None) == 'log' and self.dev:
+    def on_message(self, message):
+        data = IO.deserialize(message, "ubjson")
+        msg = IO.deserialize(data["data"], "ubjson")
+        logging.debug(f"receive message: {msg}")
+        if msg.get("type", None) == "log" and self.dev:
             print(msg)
         self.messages.append(msg)
         # if msg and type(msg) is dict and msg.get('type', None) == 'terminate':
@@ -70,12 +68,12 @@ class MessageStreamReceiver(JobReceiver):
     def on_error(self, ws, error):
         logging.info("MessageStreamReceiver error")
         msg = {
-            'type': 'log',
-            'verb': 'create',
-            'version': 1,
-            'data': {
-                'level': 'error',
-                'content': "websocket error",
+            "type": "log",
+            "verb": "create",
+            "version": 1,
+            "data": {
+                "level": "error",
+                "content": "websocket error",
             },
         }
         self.messages.append(msg)
@@ -84,37 +82,46 @@ class MessageStreamReceiver(JobReceiver):
         logging.info("MessageStreamReceiver close")
         self._status = 0
         msg = {
-            'type': 'log',
-            'verb': 'create',
-            'version': 1,
-            'data': {
-                'level': 'error',
-                'content': "websocket closed",
+            "type": "log",
+            "verb": "create",
+            "version": 1,
+            "data": {
+                "level": "error",
+                "content": "websocket closed",
             },
         }
         self.messages.append(msg)
 
-    def on_open(self, ws):
+    def on_open(self):
         self._status = 1
-        self.__hasOpen=True
+        self.__hasOpen = True
         pass
 
     def close(self, ws):
         self._status = 0
         ws.close()
 
+    @property
+    def status(self):
+        return self._status
+
     @property
     def end(self):
-        return  not self._status
+        return not self._status
 
-    def connect(self):
+    async def connect(self):
         self._status = 0
-        self.ws = self.receive(self.job.output, None, self.on_open,
-                               self.on_message, self.on_error, self.on_close)
-        
-        
-        thread = threading.Thread(target=self.ws.run_forever, args=(None , None , 6, 3))
-        thread.setDaemon(True)
-        thread.start()
-        while not self.__hasOpen:
-            time.sleep(0.2)
+        self.ws = await self.receive(
+            self.job.output,
+            None,
+            self.on_open,
+            self.on_message,
+            self.on_error,
+            self.on_close,
+        )
+
+        # thread = threading.Thread(target=self.ws.run_forever, args=(None, None, 6, 3))
+        # thread.setDaemon(True)
+        # thread.start()
+        # while not self.__hasOpen:
+        #     time.sleep(0.2)

cloudpss/job/messageStreamSender.py

@@ -1,7 +1,10 @@
-import sys,os
+import sys, os
 import threading
 from urllib.parse import urlparse
-sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
+
+from cloudpss.utils.httpAsyncRequest import websocket_connect
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
 import websocket
 
@@ -9,36 +12,34 @@ from cloudpss.utils.IO import IO
 import time
 import logging
 
-class MessageStreamSender():
 
+class MessageStreamSender:
     def __init__(self, job, dev=False):
         super().__init__()
         self.job = job
         self.dev = dev
-        self.origin = os.environ.get('CLOUDPSS_API_URL',
-                                     'https://cloudpss.net/')
+        self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
 
     ###下面是兼容Receiver部分功能实现
 
     def on_message(self, ws, message):
-        logging.info('on_message',message)
+        logging.info("on_message", message)
 
     def on_error(self, ws, error):
-        logging.info('on_error')
+        logging.info("on_error")
 
     def on_close(self, *args, **kwargs):
         time.sleep(0.5)
         self._status = 0
-        
-        logging.info('on_close')
 
-    def on_open(self, ws):
+        logging.info("on_close")
+
+    def on_open(self):
         self._status = 1
-        logging.info('on_open')
+        logging.info("on_open")
         pass
 
     def close(self):
-        
         self._status = 0
         self.ws.close()
 
@@ -47,28 +48,30 @@ class MessageStreamSender():
         return self._status
 
     def write(self, message):
-        data = IO.serialize(message, 'ubjson',None)
-        self.ws.send(data, websocket.ABNF.OPCODE_BINARY)
-        
-    def connect(self):
-        logging.info('connect')
+        data = IO.serialize(message, "ubjson", None)
+        self.ws.send_bytes(data)
+
+    async def connect(self):
+        logging.info("connect")
         self._status = 0
         if self.job.input is None:
-            raise Exception('id is None')
+            raise Exception("id is None")
         u = list(urlparse(self.origin))
-        head = 'wss' if u[0] == 'https' else 'ws'
+        head = "wss" if u[0] == "https" else "ws"
 
-        path = head + '://' + str(u[1]) + '/api/streams/token/' + self.job.input
+        path = head + "://" + str(u[1]) + "/api/streams/token/" + self.job.input
         logging.info(f"receive data from websocket: {path}")
-        
-        self.ws = websocket.WebSocketApp(path,
-                                    on_open=self.on_open,
-                                    on_message=self.on_message,
-                                    on_error=self.on_error,
-                                    on_close=self.on_close)
-        
-        thread = threading.Thread(target=self.ws.run_forever, args=(None , None , 6, 3))
-        thread.setDaemon(True)
-        thread.start()
-        while self.status != 1:
-            time.sleep(0.2)
+
+        self.ws = await websocket_connect(
+            path,
+            open_func=self.on_open,
+            receive_func=self.on_message,
+            error_func=self.on_error,
+            closed_func=self.on_close,
+        )
+
+        # thread = threading.Thread(target=self.ws.run_forever, args=(None, None, 6, 3))
+        # thread.setDaemon(True)
+        # thread.start()
+        # while self.status != 1:
+        #     time.sleep(0.2)