cloudpss 4.1.1b8__py3-none-any.whl → 4.5.0__py3-none-any.whl

cloudpss/job/job.py

@@ -2,23 +2,53 @@ import asyncio
 import random
 import re
 import time
-from cloudpss.utils.AsyncIterable import CustomAsyncIterable
 
-from cloudpss.utils.httpAsyncRequest import graphql_fetch
-from .view import getViewClass
+from cloudpss.job.result.result import Result
+from .result import getResultClass
 
 from cloudpss.utils.IO import IO
 from .messageStreamReceiver import MessageStreamReceiver
 
 from cloudpss.utils.graphqlUtil import graphql_request
-from .jobPolicy import JobPolicy
-from .jobMachine import JobMachine
 from .messageStreamSender import MessageStreamSender
-from typing import Any, Callable, TypeVar
+from typing import Any, Callable, Generic, TypeVar
 F = TypeVar('F', bound=Callable[..., Any])
-class Job(object):
+T = TypeVar('T', bound=Callable[..., Result])
+class Job(Generic[T]):
     """docstring for Job"""
-
+    __jobQuery = """query($_a:JobInput!){
+            job(input:$_a){
+                id
+                args
+                createTime
+                startTime
+                endTime
+                status
+                context
+                user
+                priority
+                policy  { 
+                    name
+                    queue
+                    tres {
+                        cpu
+                        ecpu
+                        mem
+                    } 
+                    priority 
+                    maxDuration 
+                }
+                machine {
+                    id
+                    name
+                }
+                input
+                output
+                position
+            }
+        }"""
+    
+    __createJobQuery = """mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}"""
     def __init__(
         self,
         id,
@@ -47,7 +77,7 @@ class Job(object):
         self.user = user
         self.priority = priority
         self.policy = policy  # type: ignore
-        self.machine = JobMachine(**machine)  # type: ignore
+        self.machine = machine # type: ignore
         self.input = input
         self.output = output
         self.position = position
@@ -57,82 +87,32 @@ class Job(object):
 
     @staticmethod
     def fetch(id):
-        return  asyncio.run(Job.fetchAsync(id))
-    @staticmethod
-    async def fetchAsync(id):
         """
         获取job信息
         """
         if id is None:
             raise Exception("id is None")
-        query = """query($_a:JobInput!){
-            job(input:$_a){
-                id
-                args
-                createTime
-                startTime
-                endTime
-                status
-                context
-                user
-                priority
-                policy  { 
-                    name
-                    queue
-                    tres {
-                        cpu
-                        ecpu
-                        mem
-                    } 
-                    priority 
-                    maxDuration 
-                }
-                machine {
-                    id
-                    name
-                    tres {
-                        cpu
-                        ecpu
-                        mem
-                    }
-                }
-                input
-                output
-                position
-            }
-        }"""
+       
         variables = {"_a": {"id": id}}
 
-        r = await graphql_fetch(query, variables)
+        r =  graphql_request(Job.__jobQuery, variables)
         if "errors" in r:
             raise Exception(r["errors"])
         return Job(**r["data"]["job"])
-
-    @staticmethod
-    def fetchMany(*args):
-        """
-        批量获取任务信息
-        """
-        jobs = CustomAsyncIterable(Job.fetchAsync,*args)
-        return jobs
+        
+    
+    # @staticmethod
+    # def fetchMany(*args):
+    #     """
+    #     批量获取任务信息
+    #     """
+    #     # jobs = CustomAsyncIterable(Job.fetch,*args)
+    #     # return jobs 
     
 
+    
     @staticmethod
-    async def create(revisionHash, job, config, name=None, rid="", policy=None, **kwargs):
-        """
-        创建一个运行任务
-
-        :params: revision 项目版本号
-        :params: job 调用仿真时使用的计算方案，为空时使用项目的第一个计算方案
-        :params: config 调用仿真时使用的参数方案，为空时使用项目的第一个参数方案
-        :params: name 任务名称，为空时使用项目的参数方案名称和计算方案名称
-        :params: rid 项目rid，可为空
-
-        :return: 返回一个运行实例
-
-        >>> runner = Runner.runRevision(revision,job,config,'')
-        """
-
+    def __createJobVariables(job, config, revisionHash, rid=None, policy=None, **kwargs):
         # 处理policy字段
         if policy is None:
             policy = {}
@@ -148,45 +128,62 @@ class Job(object):
                 k, v = t.split("=")
                 tres[k] = float(v)  # type: ignore
             policy["tres"] = tres
-
-        query = """mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}"""
         function = job["rid"].replace("job-definition/cloudpss/", "function/CloudPSS/")
+        implement = kwargs.get("implement", None)
+        debug = job["args"].get("@debug", None )
+        debugargs={}
+        if debug is not None:
+            t= [ i.split('=') for i in re.split(r'\s+',debug) if i.find('=')>0]
+            for i in t:
+                debugargs[i[0]]=i[1]
+        context=[
+            function,
+            f"model/@sdk/{str(int(time.time() * random.random()))}",
+        ]
+        if rid is not None and rid != "":
+            context.append(rid)
+            
+        PARENT_JOB_ID =kwargs.get("PARENT_JOB_ID",None)
+        if PARENT_JOB_ID is not None:
+            context.append(f"job/parent/{PARENT_JOB_ID}")
+            
         variables = {
             "input": {
                 "args": {
                     **job["args"],
                     "_ModelRevision": revisionHash,
                     "_ModelArgs": config["args"],
+                    "implement":implement
                 },
-                "context": [
-                    function,
-                    rid,
-                    f"model/@sdk/{str(int(time.time() * random.random()))}",
-                ],
+                "context": context,
                 "policy": policy,
+                "debug":debugargs
             }
         }
-        r = await graphql_fetch(query, variables)
-        if "errors" in r:
-            raise Exception(r["errors"])
-        id = r["data"]["job"]["id"]
-        return await Job.fetchAsync(id)
-
+        return variables
     @staticmethod
-    async def abort(id, timeout):
+    def create(revisionHash, job, config, name=None, rid=None, policy=None, **kwargs):
         """
-        结束当前运行的算例
+        创建一个运行任务
 
+        :params: revision 项目版本号
+        :params: job 调用仿真时使用的计算方案，为空时使用项目的第一个计算方案
+        :params: config 调用仿真时使用的参数方案，为空时使用项目的第一个参数方案
+        :params: name 任务名称，为空时使用项目的参数方案名称和计算方案名称
+        :params: rid 项目rid，可为空
+
+        :return: 返回一个运行实例
+
+        >>> runner = Runner.runRevision(revision,job,config,'')
         """
-        query = """mutation ($input: AbortJobInput!) {
-            job: abortJob(input: $input) {
-                id
-                status
-            }
-        }
-        """
-        variables = {"input": {"id": id, "timeout": timeout}}
-        await graphql_fetch(query, variables)
+        variables=Job.__createJobVariables(job, config, revisionHash, rid=rid, policy=policy, **kwargs)
+        r =  graphql_request(Job.__createJobQuery, variables)
+        if "errors" in r:
+            raise Exception(r["errors"])
+        id = r["data"]["job"]["id"]
+        return  Job.fetch(id)
+        
+    
 
     @staticmethod
     def load(file, format="yaml"):
@@ -196,49 +193,30 @@ class Job(object):
     def dump(job, file, format="yaml", compress="gzip"):
         return IO.dump(job, file, format, compress)
 
-    async def read(self, receiver=None, dev=False, **kwargs):
-        """
-        使用接收器获取当前运行实例的输出
-        """
-        if receiver is not None:
-            self.__receiver = receiver
-        if self.__receiver is None:
-            self.__receiver = MessageStreamReceiver(self, dev)
-        await self.__receiver.connect(**kwargs)
-        return self.__receiver
+    
 
-    def read_legacy(self, receiver=None, dev=False, **kwargs):
+    def read(self, receiver=None, **kwargs):
         """
         使用接收器获取当前运行实例的输出
         """
         if receiver is not None:
             self.__receiver = receiver
         if self.__receiver is None:
-            self.__receiver = MessageStreamReceiver(self, dev)
-            self.__receiver.connect_legacy(**kwargs)
+            self.__receiver = MessageStreamReceiver(self)
+            self.__receiver.connect(**kwargs)
         return self.__receiver
 
-    async def write(self, sender=None, dev=False, **kwargs) -> MessageStreamSender:
-        """
-        使用发送器为当前运行实例输入
-        """
-
-        if sender is not None:
-            self.__sender = sender
-        if self.__sender is None:
-            self.__sender = MessageStreamSender(self, dev)
-        await self.__sender.connect(**kwargs)
-        return self.__sender
     
-    def write_legacy(self, sender=None, dev=False, **kwargs) -> MessageStreamSender:
+    
+    def write(self, sender=None,  **kwargs) -> MessageStreamSender:
         """
-        使用发送器为当前运行实例输入同步方法
+        使用发送器为当前运行实例输入
         """
 
         if sender is not None:
             self.__sender = sender
         if self.__sender is None:
-            self.__sender = MessageStreamSender(self, dev)
+            self.__sender = MessageStreamSender(self)
         self.__sender.connect_legacy(**kwargs)
         return self.__sender
     
@@ -246,43 +224,60 @@ class Job(object):
         """
         return: 0: 运行中 1: 运行完成 2: 运行失败
         """
-        
+        time.sleep(0)
         if self.__receiver is not None:
             return self.__receiver.status
         if self.__receiver is None:
-            self.__connect_legacy()
+            self.__connect()
+        
         return 0
     
-    def __connect_legacy(self):
+    def __connect(self):
         """
         连接接收器和发送器
         """
-        viewType = getViewClass(self.context[0])
-        self._result = self.view_legacy(viewType)
+        resultType = getResultClass(self.context[0])
+        
+        self._result = self._resultView(resultType)
         
     @property
-    def result(self):
+    def result(self)->T:
         """
         获取当前运行实例的输出
         """
         if self._result is None:
-            self.__connect_legacy()
+            self.__connect()
         return self._result
    
-    
-    def view_legacy(self, viewType:F)->F:
+        
+        
+    def _resultView(self, resultType=None):
         """
         获取当前运行实例的输出
         """
-        receiver =  self.read_legacy()
-        sender =  self.write_legacy()
-        return viewType(receiver, sender)
+        receiver =  self.read()
+        sender =  self.write()
+        if resultType is None:
+            resultType = getResultClass(self.context[0])
+        return resultType(receiver, sender)
+
+    
 
-    async def view(self, viewType:F)->F:
+    def abort(self,timeout=3):
         """
-        获取当前运行实例的输出
+        中断当前运行实例
         """
-        receiver = await self.read()
-        sender = await self.write()
-        self._result= viewType(receiver, sender)
-        return self._result
+        query = '''mutation ($input: AbortJobInput!) {
+            job: abortJob(input: $input) {
+                id
+                status
+            }
+        }
+        '''
+        variables = {
+            'input': {
+                'id': self.taskId,
+                'timeout': timeout
+            }
+        }
+        graphql_request(query, variables)

cloudpss/job/jobReceiver.py

@@ -1,3 +1,4 @@
+from deprecated import deprecated
 class JobReceiver(object):
     messages = []
     index = 0
@@ -20,7 +21,7 @@ class JobReceiver(object):
             return message
         raise StopIteration()
 
-    def view(self, ViewType):
+    def result(self, resultType):
         """
             获取指定类型的视图数据
 
@@ -30,4 +31,9 @@ class JobReceiver(object):
 
             >>> view= receiver.view(EMTView)
         """
-        return ViewType(self)
+        return resultType(self)
+    
+    @property
+    @deprecated(version='3.0', reason="该方法将在 5.0 版本移除")
+    def message(self):
+        return self.messages

cloudpss/job/messageStreamReceiver.py

@@ -1,9 +1,5 @@
 import logging
-
-import aiohttp
-from aiohttp import WSMsgType
-
-from cloudpss.utils.httpAsyncRequest import websocket_connect
+import sys
 from .jobReceiver import JobReceiver
 import os
 from urllib.parse import urlparse
@@ -24,91 +20,36 @@ class Message(object):
 
 
 class MessageStreamReceiver(JobReceiver):
-    def __init__(self, job, dev=False):
+    def __init__(self, job):
         super().__init__()
         self.job = job
-        self.dev = dev
+        self.id =self.job.output
         self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
         self.__hasOpen = False
 
-    def __receive_legacy(self, id, fr0m):
-        """
-        同步方法读取消息流中的数据
-        id: 消息流id
-        fr0m: 从哪个位置开始读取，如果为0则从头开始读取
-        on_open: 连接建立时的回调函数
-        on_message: 收到消息时的回调函数
-        on_error: 发生错误时的回调函数
-        on_close: 连接关闭时的回调函数
-        """
-        if id is None:
-            raise Exception("id is None")
-        u = list(urlparse(self.origin))
-        head = "wss" if u[0] == "https" else "ws"
-
-        path = head + "://" + str(u[1]) + "/api/streams/id/" + id
-        if fr0m is not None:
-            path = path + "&from=" + str(fr0m)
-        logging.info(f"receive data from websocket: {path}")
-        ws = websocket.WebSocketApp(
-            path,
-            on_open=self.__on_open,
-            on_message=self.__on_message_legacy,
-            on_error=self.__on_error,
-            on_close=self.__on_close,
-        )
-
-        return ws
-
-    async def __receive(self, id, fr0m):
-        """
-        读取消息流中的数据
-        id: 消息流id
-        fr0m: 从哪个位置开始读取，如果为0则从头开始读取
-        on_open: 连接建立时的回调函数
-        on_message: 收到消息时的回调函数
-        on_error: 发生错误时的回调函数
-        on_close: 连接关闭时的回调函数
-        """
-        if id is None:
+    def __path(self, from_=None):
+        if self.id is None:
             raise Exception("id is None")
         u = list(urlparse(self.origin))
         head = "wss" if u[0] == "https" else "ws"
-
-        path = head + "://" + str(u[1]) + "/api/streams/id/" + id
-        if fr0m is not None:
-            path = path + "&from=" + str(fr0m)
-        logging.info(f"MessageStreamReceiver data from websocket: {path}")
-        async for msg in websocket_connect(
-            path,
-            open_func=self.__on_open,
-        ):
-            if msg.type == WSMsgType.BINARY:
-                decode = self.__on_message(msg.data)
-                yield decode
-            elif msg.type == WSMsgType.TEXT:
-                decode = self.__on_message(msg.data)
-                yield decode
-            elif msg.type == WSMsgType.CLOSED:
-                logging.debug("WebSocket连接已关闭")
-                self.__on_close()
-                
-                break
-            elif msg.type == WSMsgType.ERROR:
-                logging.debug(f"WebSocket连接发生错误：{msg.data}")
-                self.__on_error(msg.data)
-                break
-        self._status=1
+        path = head + "://" + str(u[1]) + "/api/streams/id/" + self.id
+        if from_ is not None:
+            path = path + "?from=" + str(from_)
+        return path
+  
     ###下面是兼容Receiver部分功能实现
     def __on_message_legacy(self,  *args, **kwargs):
-        if len(args) == 1:
+        
+        if type(args[0]) != websocket.WebSocketApp:
             message = args[0]
         else:
             message = args[1]
         return self.__on_message(message)
 
     def __on_message(self, message):
+        
         data = IO.deserialize(message, "ubjson")
+        self.ws.url = self.__path(data["id"])
         msg = IO.deserialize(data["data"], "ubjson")
         self.messages.append(msg)
         if(msg['type']=='terminate'):
@@ -130,9 +71,10 @@ class MessageStreamReceiver(JobReceiver):
         self.messages.append(msg)
 
     def __on_close(self, *args, **kwargs):
-        if len(args)>0:
+        if len(args)>1:
             msg =args[2]
-            if msg.startswith("CMS_NO_STREAM_ID:"):
+            
+            if msg is not None and msg.startswith("CMS_NO_STREAM_ID:"):
                 self._status = 1
                 msg = {
                     "type": "log",
@@ -145,7 +87,6 @@ class MessageStreamReceiver(JobReceiver):
                 self.messages.append(msg)
                 return
         logging.debug("MessageStreamReceiver close")
-        self._status = 1
         msg = {
             "type": "log",
             "verb": "create",
@@ -156,7 +97,7 @@ class MessageStreamReceiver(JobReceiver):
             },
         }
         self.messages.append(msg)
-
+        self._status = 1
     
     def __on_open(self,ws, *args, **kwargs):
         self.ws = ws
@@ -173,31 +114,33 @@ class MessageStreamReceiver(JobReceiver):
     def status(self):
         return self._status
 
-    @property
-    def end(self):
-        return not self._status
+    def waitFor(self,timeOut=sys.maxsize):
+        """
+            阻塞方法，直到任务完成
 
-    def connect_legacy(self):
+            :params timeOut: 超时时间
+        """
+        start = time.time()
+        while self.status == 0:
+            time.sleep(0)
+            if time.time()-start>timeOut:
+                raise Exception("time out")
+       
+
+    def connect(self):
         self._status = 1
-        self.ws = self.__receive_legacy(
-            self.job.output,
-            None,
+        path = self.__path()
+        logging.info(f"receive data from websocket: {path}")
+        self.ws = websocket.WebSocketApp(
+            path,
+            on_open=self.__on_open,
+            on_message=self.__on_message_legacy,
+            on_error=self.__on_error,
+            on_close=self.__on_close,
         )
-        thread = threading.Thread(target=self.ws.run_forever, args=(None, None, 6, 3))
+        thread = threading.Thread(target=self.ws.run_forever, kwargs={'ping_interval':60,'ping_timeout':5,'reconnect':True})
         thread.setDaemon(True)
         thread.start()
         while not self.__hasOpen:
-            time.sleep(0.2)
-
-    async def connect(self):
-        self._status = 0
-        self.receiver= self.__receive(
-            self.job.output,
-            None,
-        )
-        # asyncio.create_task(
-        #     self.__receive(
-        #         self.job.output,
-        #         None
-        #     )
-        # )
+            time.sleep(0)
+        

cloudpss/job/messageStreamSender.py

@@ -2,10 +2,6 @@ import asyncio
 import sys, os
 import threading
 from urllib.parse import urlparse
-
-import aiohttp
-
-
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
 import websocket
@@ -16,10 +12,9 @@ import logging
 
 
 class MessageStreamSender:
-    def __init__(self, job, dev=False):
+    def __init__(self, job):
         super().__init__()
         self.job = job
-        self.dev = dev
         self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
         self.__hasOpen = False
 
@@ -56,19 +51,9 @@ class MessageStreamSender:
     def status(self):
         return self._status
 
-    async def write_async(self, message):
-        if self.websocket:
-            data = IO.serialize(message, "ubjson", None)
-            await self.websocket.send_bytes(data)
-        else:
-            logging.info("websocket is None")
-
     def write(self, message):
-        if self.ws:
-            data = IO.serialize(message, "ubjson", None)
-            self.ws.send(data,websocket.ABNF.OPCODE_BINARY)
-        else:
-            asyncio.run(self.write_async(message))
+        data = IO.serialize(message, "ubjson", None)
+        self.ws.send(data,websocket.ABNF.OPCODE_BINARY)
 
     def connect_legacy(self):
         """
@@ -99,28 +84,6 @@ class MessageStreamSender:
             time.sleep(0.2)
         return self.ws
 
-    async def receive_data(self):
-        if self.websocket:
-            data = await self.websocket.receive()
-            if data.type == aiohttp.WSMsgType.TEXT:
-                self.__on_message(data.data)
-            elif data.type == aiohttp.WSMsgType.CLOSED:
-                self.__on_close()
-            elif data.type == aiohttp.WSMsgType.ERROR:
-                self.__on_error(data.data)
-        else:
-            logging.info("WebSocket connection not established")
-
-    async def connect(self):
-        self._status = 0
-        if self.job.input is None:
-            raise Exception("id is None")
-        if self.job.input == "00000000-0000-0000-0000-000000000000":
-            return
-        u = list(urlparse(self.origin))
-        head = "wss" if u[0] == "https" else "ws"
+    
 
-        path = head + "://" + str(u[1]) + "/api/streams/token/" + self.job.input
-        logging.info(f"MessageStreamSender data from websocket: {path}")
-        async with aiohttp.ClientSession() as session:
-            self.websocket = await session.ws_connect(path)
+