cloudpss 4.0.2__tar.gz → 4.1.1a1__tar.gz

{cloudpss-4.0.2 → cloudpss-4.1.1a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudpss
-Version: 4.0.2
+Version: 4.1.1a1
 Summary: cloudpss sdk
 Home-page: https://www.cloudpss.net
 Author: cloudpss

{cloudpss-4.0.2 → cloudpss-4.1.1a1}/cloudpss/__init__.py

@@ -6,17 +6,16 @@ from .model import Model, ModelRevision, ModelTopology
 from .project import Project
 from .utils import MatlabDataEncoder, DateTimeEncode
 from . import function
-
+from .job import Job
 from .function import FunctionExecution
 from .version import __version__
 __all__ = [
     'setToken', 'Model', 'ModelRevision', 'ModelTopology', 'Runner', 'Result',
     'PowerFlowResult', 'EMTResult', 'MatlabDataEncoder', 'DateTimeEncode',
-    'function', 'Project', 'currentJob', 'IESLabSimulation', 'IESLabPlan','__version__'
+    'function', 'Project', 'currentJob', 'IESLabSimulation', 'IESLabPlan','__version__','Job'
 ]
 
 
-
 def currentJob():
     """
         获取当前的 currentExecution 实例

{cloudpss-4.0.2 → cloudpss-4.1.1a1}/cloudpss/function/functionExecution.py

@@ -163,7 +163,7 @@ class FunctionExecution(object):
         sys.exit(code)
 
     def print(self, data):
-        print(json.dumps(data, cls=MatlabDataEncoder), flush=True)
+        print(json.dumps(data, cls=MatlabDataEncoder)+'\n', flush=True)
 
     def log(self, content, level='info', html=False, key=None):
         '''

cloudpss-4.1.1a1/cloudpss/job/__init__.py

@@ -0,0 +1,5 @@
+from .job import Job
+
+__all__ = [
+    'Job'
+]

cloudpss-4.1.1a1/cloudpss/job/job.py

@@ -0,0 +1,240 @@
+import asyncio
+import random
+import re
+import time
+from cloudpss.utils.AsyncIterable import CustomAsyncIterable
+
+from cloudpss.utils.httpAsyncRequest import graphql_fetch
+from .view import getViewClass
+
+from cloudpss.utils.IO import IO
+from .messageStreamReceiver import MessageStreamReceiver
+
+from cloudpss.utils.graphqlUtil import graphql_request
+from .jobPolicy import JobPolicy
+from .jobMachine import JobMachine
+from .messageStreamSender import MessageStreamSender
+
+
+class Job(object):
+    """docstring for Job"""
+
+    def __init__(
+        self,
+        id,
+        args,
+        createTime,
+        startTime,
+        endTime,
+        status,
+        context,
+        user,
+        priority,
+        policy,
+        machine,
+        input,
+        output,
+        position,
+    ):
+        super(Job, self).__init__()
+        self.id = id
+        self.args = args
+        self.createTime = createTime
+        self.startTime = startTime
+        self.endTime = endTime
+        self.job_status = status #这里的status字段与原本的status()冲突
+        self.context = context
+        self.user = user
+        self.priority = priority
+        self.policy = policy  # type: ignore
+        self.machine = JobMachine(**machine)  # type: ignore
+        self.input = input
+        self.output = output
+        self.position = position
+        self.__receiver = None
+        self.__sender = None
+        self._result = None
+
+    @staticmethod
+    async def fetch(id):
+        """
+        获取job信息
+        """
+        if id is None:
+            raise Exception("id is None")
+        query = """query($_a:JobInput!){
+            job(input:$_a){
+                id
+                args
+                createTime
+                startTime
+                endTime
+                status
+                context
+                user
+                priority
+                policy  { 
+                    name
+                    queue
+                    tres {
+                        cpu
+                        ecpu
+                        mem
+                    } 
+                    priority 
+                    maxDuration 
+                }
+                machine {
+                    id
+                    name
+                    tres {
+                        cpu
+                        ecpu
+                        mem
+                    }
+                }
+                input
+                output
+                position
+            }
+        }"""
+        variables = {"_a": {"id": id}}
+
+        r = await graphql_fetch(query, variables)
+        if "errors" in r:
+            raise Exception(r["errors"])
+        return Job(**r["data"]["job"])
+
+    @staticmethod
+    def fetchMany(*args):
+        """
+        批量获取任务信息
+        """
+        jobs = CustomAsyncIterable(Job.fetch,*args)
+        return jobs
+    
+
+    @staticmethod
+    async def create(revisionHash, job, config, name=None, rid="", policy=None, **kwargs):
+        """
+        创建一个运行任务
+
+        :params: revision 项目版本号
+        :params: job 调用仿真时使用的计算方案，为空时使用项目的第一个计算方案
+        :params: config 调用仿真时使用的参数方案，为空时使用项目的第一个参数方案
+        :params: name 任务名称，为空时使用项目的参数方案名称和计算方案名称
+        :params: rid 项目rid，可为空
+
+        :return: 返回一个运行实例
+
+        >>> runner = Runner.runRevision(revision,job,config,'')
+        """
+
+        # 处理policy字段
+        if policy is None:
+            policy = {}
+            if policy.get("tres", None) is None:
+                policy["tres"] = {}
+            policy["queue"] = job["args"].get("@queue", 1)
+            policy["priority"] = job["args"].get("@priority", 0)
+            tres = {"cpu": 1, "ecpu": 0, "mem": 0}
+            tresStr = job["args"].get("@tres", "")
+            for t in re.split("\s+", tresStr):
+                if t == "":
+                    continue
+                k, v = t.split("=")
+                tres[k] = float(v)  # type: ignore
+            policy["tres"] = tres
+
+        query = """mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}"""
+        function = job["rid"].replace("job-definition/cloudpss/", "function/CloudPSS/")
+        variables = {
+            "input": {
+                "args": {
+                    **job["args"],
+                    "_ModelRevision": revisionHash,
+                    "_ModelArgs": config["args"],
+                },
+                "context": [
+                    function,
+                    rid,
+                    f"model/@sdk/{str(int(time.time() * random.random()))}",
+                ],
+                "policy": policy,
+            }
+        }
+        r = await graphql_fetch(query, variables)
+        if "errors" in r:
+            raise Exception(r["errors"])
+        id = r["data"]["job"]["id"]
+        return await Job.fetch(id)
+
+    @staticmethod
+    async def abort(id, timeout):
+        """
+        结束当前运行的算例
+
+        """
+        query = """mutation ($input: AbortJobInput!) {
+            job: abortJob(input: $input) {
+                id
+                status
+            }
+        }
+        """
+        variables = {"input": {"id": id, "timeout": timeout}}
+        await graphql_fetch(query, variables)
+
+    @staticmethod
+    def load(file, format="yaml"):
+        return IO.load(file, format)
+
+    @staticmethod
+    def dump(job, file, format="yaml", compress="gzip"):
+        return IO.dump(job, file, format, compress)
+
+    async def read(self, receiver=None, dev=False, **kwargs):
+        """
+        使用接收器获取当前运行实例的输出
+        """
+        if receiver is not None:
+            self.__sender = receiver
+        if self.__receiver is None:
+            self.__receiver = MessageStreamReceiver(self, dev)
+        await self.__receiver.connect(**kwargs)
+        return self.__receiver
+
+    async def write(self, sender=None, dev=False, **kwargs) -> MessageStreamSender:
+        """
+        使用发送器为当前运行实例输入
+        """
+
+        if sender is not None:
+            self.__sender = sender
+        if self.__sender is None:
+            self.__sender = MessageStreamSender(self, dev)
+        await self.__sender.connect(**kwargs)
+        return self.__sender
+    
+    def status(self):
+        if self.__receiver is not None:
+            return self.__receiver.status
+        return 0
+
+    @property
+    def result(self):
+        """
+        获取当前运行实例的输出
+        """
+        if self._result is None:
+            viewType = getViewClass(self.context[0])
+            self._result = asyncio.run(self.view(viewType))
+        return self._result
+
+    async def view(self, viewType):
+        """
+        获取当前运行实例的输出
+        """
+        receiver = await self.read()
+        sender = await self.write()
+        return viewType(receiver, sender)

cloudpss-4.1.1a1/cloudpss/job/jobMachine.py

@@ -0,0 +1,11 @@
+from cloudpss.job.jobTres import JobTres
+
+
+class JobMachine(object):
+    '''job machine'''
+
+    def __init__(self, id, name, tres, valid=None):
+        self.id = id
+        self.name = name
+        self.tres = JobTres(**tres)  # type: ignore
+        self.valid = valid

cloudpss-4.1.1a1/cloudpss/job/jobPolicy.py

@@ -0,0 +1,129 @@
+from cloudpss.job.jobQueue import JobQueue
+from cloudpss.job.jobTres import JobTres
+from cloudpss.utils.graphqlUtil import graphql_request
+
+
+class JobPolicy(object):
+
+    def __init__(self, id, name, users, functions, tres, minPriority,
+                 maxPriority, maxDuration, createTime, updateTime, visibility,
+                 queue):
+        self.id = id
+        self.name = name
+        self.users = users
+        self.functions = functions
+        print(tres)
+        self.tres = []
+        for tre in tres:
+            self.tres.append(JobTres(**tre))  # type: ignore
+        self.minPriority = minPriority
+        self.maxPriority = maxPriority
+        self.maxDuration = maxDuration
+        self.createTime = createTime
+        self.updateTime = updateTime
+        self.visibility = visibility
+        self.queue = JobQueue(**queue)  # type: ignore
+
+    @staticmethod
+    def fetch(id):
+        query = '''query($input:JobPolicyInput!)
+            {
+                jobPolicy(input:$input)
+                {
+                    id 
+                    name 
+                    users 
+                    functions 
+                    tres {
+                        cpu
+                        ecpu
+                        mem
+                    } 
+                    minPriority 
+                    maxPriority 
+                    maxDuration 
+                    createTime 
+                    updateTime 
+                    visibility 
+                    queue {
+                        id
+                        name
+                        scheduler
+                        machines {
+                            id
+                            name
+                            tres {
+                                cpu
+                                ecpu
+                                mem
+                            }
+                            valid
+                        }
+                        createTime
+                        updateTime
+                        load
+                    }
+                }
+                
+            }'''
+        variables = {'input': {'id': id}}
+        r = graphql_request(query, variables)
+        print (r)
+        if 'errors' in r:
+            raise Exception(r['errors'])
+        return JobPolicy(**r['data']['jobPolicy'])
+
+    @staticmethod
+    def fetchMany(input):
+        query = '''query($input:JobPoliciesInput!)
+            {
+                jobPolicies(input:$input)
+                {
+                    items {
+                        id 
+                        name 
+                        users 
+                        functions 
+                        tres {
+                            cpu
+                            ecpu
+                            mem
+                        } 
+                        minPriority 
+                        maxPriority 
+                        maxDuration 
+                        createTime 
+                        updateTime 
+                        visibility 
+                        queue {
+                            id
+                            name
+                            scheduler
+                            machines {
+                                id
+                                name
+                                tres {
+                                    cpu
+                                    ecpu
+                                    mem
+                                }
+                                valid
+                            }
+                            createTime
+                            updateTime
+                            load
+                        }
+                    }
+                    cursor
+                    count
+                    total
+                }
+            }'''
+        variables = {'input': input}
+        r = graphql_request(query, variables)
+        if 'errors' in r:
+            raise Exception(r['errors'])
+        policies = []
+        for policy in r['data']['jobPolicies']['items']:
+            policies.append(JobPolicy(**policy))
+        return policies

cloudpss-4.1.1a1/cloudpss/job/jobQueue.py

@@ -0,0 +1,14 @@
+from cloudpss.job.jobMachine import JobMachine
+
+
+class JobQueue(object):
+
+    def __init__(self, id, name, scheduler, machines, createTime, updateTime,
+                 load):
+        self.id = id
+        self.name = name
+        self.scheduler = scheduler
+        self.machines = [JobMachine(**m) for m in machines]
+        self.createTime = createTime
+        self.updateTime = updateTime
+        self.load = load

cloudpss-4.1.1a1/cloudpss/job/jobReceiver.py

@@ -0,0 +1,33 @@
+class JobReceiver(object):
+    messages = []
+    index = 0
+
+    def __init__(self):
+        self.index = 0
+        self.messages = []
+
+    def __len__(self):
+        return len(self.messages)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        maxLength = len(self.messages)
+        if self.index < maxLength:
+            message = self.messages[self.index]
+            self.index += 1
+            return message
+        raise StopIteration()
+
+    def view(self, ViewType):
+        """
+            获取指定类型的视图数据
+
+            :params viewType: 视图类型
+
+            :returns: 对应类型的视图数据
+
+            >>> view= receiver.view(EMTView)
+        """
+        return ViewType(self)

cloudpss-4.1.1a1/cloudpss/job/jobTres.py

@@ -0,0 +1,6 @@
+class JobTres(object):
+    '''tres'''
+    def __init__(self, cpu, mem, ecpu):
+        self.cpu = cpu
+        self.mem = mem
+        self.ecpu = ecpu

cloudpss-4.1.1a1/cloudpss/job/messageStreamReceiver.py

@@ -0,0 +1,127 @@
+import logging
+
+from cloudpss.utils.httpAsyncRequest import websocket_connect
+from .jobReceiver import JobReceiver
+import os
+from urllib.parse import urlparse
+import pytz
+
+utc_tz = pytz.timezone("UTC")
+
+from ..utils.IO import IO
+
+
+class Message(object):
+    def __init__(self, id, token):
+        self.id = id
+        self.token = token
+
+
+class MessageStreamReceiver(JobReceiver):
+    def __init__(self, job, dev=False):
+        super().__init__()
+        self.job = job
+        self.dev = dev
+        self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
+
+    async def receive(self, id, fr0m, on_open, on_message, on_error, on_close):
+        """
+        读取消息流中的数据
+        id: 消息流id
+        fr0m: 从哪个位置开始读取，如果为0则从头开始读取
+        on_open: 连接建立时的回调函数
+        on_message: 收到消息时的回调函数
+        on_error: 发生错误时的回调函数
+        on_close: 连接关闭时的回调函数
+        """
+        if id is None:
+            raise Exception("id is None")
+        u = list(urlparse(self.origin))
+        head = "wss" if u[0] == "https" else "ws"
+
+        path = head + "://" + str(u[1]) + "/api/streams/id/" + id
+        if fr0m is not None:
+            path = path + "&from=" + str(fr0m)
+        logging.info(f"receive data from websocket: {path}")
+        ws = await websocket_connect(
+            path,
+            open_func=on_open,
+            receive_func=on_message,
+            error_func=on_error,
+            closed_func=on_close,
+        )
+
+        return ws
+
+    ###下面是兼容Receiver部分功能实现
+
+    def on_message(self, message):
+        data = IO.deserialize(message, "ubjson")
+        msg = IO.deserialize(data["data"], "ubjson")
+        logging.debug(f"receive message: {msg}")
+        if msg.get("type", None) == "log" and self.dev:
+            print(msg)
+        self.messages.append(msg)
+        # if msg and type(msg) is dict and msg.get('type', None) == 'terminate':
+        #     self.close(ws)
+
+    def on_error(self, ws, error):
+        logging.info("MessageStreamReceiver error")
+        msg = {
+            "type": "log",
+            "verb": "create",
+            "version": 1,
+            "data": {
+                "level": "error",
+                "content": "websocket error",
+            },
+        }
+        self.messages.append(msg)
+
+    def on_close(self, *args, **kwargs):
+        logging.info("MessageStreamReceiver close")
+        self._status = 0
+        msg = {
+            "type": "log",
+            "verb": "create",
+            "version": 1,
+            "data": {
+                "level": "error",
+                "content": "websocket closed",
+            },
+        }
+        self.messages.append(msg)
+
+    def on_open(self):
+        self._status = 1
+        self.__hasOpen = True
+        pass
+
+    def close(self, ws):
+        self._status = 0
+        ws.close()
+
+    @property
+    def status(self):
+        return self._status
+
+    @property
+    def end(self):
+        return not self._status
+
+    async def connect(self):
+        self._status = 0
+        self.ws = await self.receive(
+            self.job.output,
+            None,
+            self.on_open,
+            self.on_message,
+            self.on_error,
+            self.on_close,
+        )
+
+        # thread = threading.Thread(target=self.ws.run_forever, args=(None, None, 6, 3))
+        # thread.setDaemon(True)
+        # thread.start()
+        # while not self.__hasOpen:
+        #     time.sleep(0.2)

cloudpss-4.1.1a1/cloudpss/job/messageStreamSender.py

@@ -0,0 +1,77 @@
+import sys, os
+import threading
+from urllib.parse import urlparse
+
+from cloudpss.utils.httpAsyncRequest import websocket_connect
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
+
+import websocket
+
+from cloudpss.utils.IO import IO
+import time
+import logging
+
+
+class MessageStreamSender:
+    def __init__(self, job, dev=False):
+        super().__init__()
+        self.job = job
+        self.dev = dev
+        self.origin = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
+
+    ###下面是兼容Receiver部分功能实现
+
+    def on_message(self, ws, message):
+        logging.info("on_message", message)
+
+    def on_error(self, ws, error):
+        logging.info("on_error")
+
+    def on_close(self, *args, **kwargs):
+        time.sleep(0.5)
+        self._status = 0
+
+        logging.info("on_close")
+
+    def on_open(self):
+        self._status = 1
+        logging.info("on_open")
+        pass
+
+    def close(self):
+        self._status = 0
+        self.ws.close()
+
+    @property
+    def status(self):
+        return self._status
+
+    def write(self, message):
+        data = IO.serialize(message, "ubjson", None)
+        self.ws.send_bytes(data)
+
+    async def connect(self):
+        logging.info("connect")
+        self._status = 0
+        if self.job.input is None:
+            raise Exception("id is None")
+        u = list(urlparse(self.origin))
+        head = "wss" if u[0] == "https" else "ws"
+
+        path = head + "://" + str(u[1]) + "/api/streams/token/" + self.job.input
+        logging.info(f"receive data from websocket: {path}")
+
+        self.ws = await websocket_connect(
+            path,
+            open_func=self.on_open,
+            receive_func=self.on_message,
+            error_func=self.on_error,
+            closed_func=self.on_close,
+        )
+
+        # thread = threading.Thread(target=self.ws.run_forever, args=(None, None, 6, 3))
+        # thread.setDaemon(True)
+        # thread.start()
+        # while self.status != 1:
+        #     time.sleep(0.2)