cloudpss 4.0.3__py3-none-any.whl → 4.0.5__py3-none-any.whl

cloudpss/runner/MessageStreamReceiver.py

@@ -0,0 +1,99 @@
+import os
+from urllib.parse import urlparse
+import requests
+import websocket
+import datetime
+import pytz
+utc_tz = pytz.timezone('UTC')
+
+from ..utils.IO import IO
+
+
+class Message(object):
+
+    def __init__(self, id, token):
+        self.id = id
+        self.token = token
+
+
+class MessageStreamReceiver(object):
+    """消息流读取函数结果"""
+
+    def __init__(self, id, db):
+        self.origin = os.environ.get('CLOUDPSS_API_URL',
+                                     'https://cloudpss.net/')
+        self.id = id
+        self.db = db
+        self._status = 0
+        self.error = None
+        self.isOpen = False
+
+    def receive(self, id, fr0m, on_open, on_message, on_error, on_close):
+        """
+        读取消息流中的数据
+        id: 消息流id
+        fr0m: 从哪个位置开始读取，如果为0则从头开始读取
+        on_open: 连接建立时的回调函数
+        on_message: 收到消息时的回调函数
+        on_error: 发生错误时的回调函数
+        on_close: 连接关闭时的回调函数
+        """
+        if id is None:
+            raise Exception('id is None')
+        u = list(urlparse(self.origin))
+        head = 'wss' if u[0] == 'https' else 'ws'
+
+        path = head + '://' + str(u[1]) + '/api/streams/id/' + id
+        if fr0m is not None:
+            path = path + '&from=' + str(fr0m)
+        ws = websocket.WebSocketApp(path,
+                                    on_open=on_open,
+                                    on_message=on_message,
+                                    on_error=on_error,
+                                    on_close=on_close)
+        ws.run_forever()
+        return ws
+
+    ###下面是兼容Receiver部分功能实现
+    def on_message(self, ws, message):
+        data = IO.deserialize(message, 'ubjson')
+        msg = IO.deserialize(data['data'], 'ubjson')
+        if "when" not in msg:
+            msg['when']= datetime.datetime.now()
+        self.db.storeMessage(msg)
+        if(msg['type']=='terminate'):
+            self.close(self.ws)
+
+    def on_error(self, ws, error):
+        msg = {
+            'type': 'log',
+            'verb': 'create',
+            'version': 1,
+            'data': {
+                'level': 'error',
+                'content': "websocket error",
+            },
+        }
+        self.db.storeMessage(msg)
+        self.error = error
+        self._status = -1
+
+
+    def on_close(self, ws,*args,**kwargs):
+        self.db.finished = datetime.datetime.now(tz=utc_tz).isoformat()
+        self._status = 1
+
+    def on_open(self, ws):
+        self.isOpen = True
+
+    def close(self, ws):
+        ws.close()
+
+    def status(self):
+        return self._status
+
+    def connect(self):
+        self._status = 0
+        self.ws = self.receive(self.id, None, self.on_open, self.on_message, self.on_error, self.on_close)
+
+

cloudpss/runner/runner.py

@@ -3,20 +3,27 @@ import threading
 import json
 import time
 import random
+
+from deprecated import deprecated
+
+from cloudpss.utils.graphqlUtil import graphql_request
 from .receiver import Receiver
+from .MessageStreamReceiver import MessageStreamReceiver
 from .result import IESLabSimulationResult, PowerFlowResult, EMTResult, Result, IESResult
 from .IESLabPlanResult import IESLabPlanResult
 from .IESLabEvaluationResult import IESLabEvaluationResult
 from .IESLabTypicalDayResult import IESLabTypicalDayResult
 from .storage import Storage
 from ..utils import request
-from typing import  TypeVar,  Generic
+from typing import TypeVar, Generic
+import re
 
 RECEIVER = {
-    'default': Receiver,
+    'default': MessageStreamReceiver,
 }
 T = TypeVar('T', Result, EMTResult, PowerFlowResult, IESResult,
-            IESLabSimulationResult, IESLabPlanResult, IESLabEvaluationResult,IESLabTypicalDayResult)
+            IESLabSimulationResult, IESLabPlanResult, IESLabEvaluationResult,
+            IESLabTypicalDayResult)
 
 IES_LAB_RESULT = {
     'function/ieslab/plan': IESLabPlanResult,
@@ -40,8 +47,9 @@ RESULT_DB = {
 }
 
 class Runner(Generic[T]):
-    def __init__(self, taskId, name, job, config, revision, modelRid,
+    def __init__(self, id,taskId, name, job, config, revision, modelRid, policy,
                  **kwargs):
+        self.id = id
         self.taskId = taskId
         self.db = Storage(taskId, name, job, config, revision, modelRid)
         rid =job['rid'].replace('job-definition/','function/').replace('/cloudpss/','/CloudPSS/')
@@ -87,10 +95,36 @@ class Runner(Generic[T]):
         结束当前运行的算例
 
         """
-        r = request("DELETE", 'api/simulation/runner/' + str(self.taskId))
+        self.abort(0)
+    def abort(self,timeout=3):
+        """
+        结束当前运行的算例
+
+        """
+        query = '''mutation ($input: AbortJobInput!) {
+            job: abortJob(input: $input) {
+                id
+                status
+            }
+        }
+        '''
+        variables = {
+            'input': {
+                'id': self.taskId,
+                'timeout': timeout
+            }
+        }
+        graphql_request(query, variables)
+
 
     @staticmethod
-    def create(revisionHash, job, config, name=None, rid='', **kwargs):
+    def create(revisionHash,
+               job,
+               config,
+               name=None,
+               rid='',
+               policy=None,
+               **kwargs):
         '''
             创建一个运行任务
 
@@ -104,35 +138,72 @@ class Runner(Generic[T]):
 
             >>> runner = Runner.runRevision(revision,job,config,'')
         '''
-        taskId = str(int(time.time() * random.random()))
 
-        runner = Runner(taskId, name, job, config, revisionHash, rid, **kwargs)
+        #处理policy字段
+        if policy is None:
+            policy = {}
+            if policy.get('tres', None) is None:
+                policy['tres'] = {}
+            policy['queue'] = job['args'].get('@queue', 1)
+            policy['priority'] = job['args'].get('@priority', 0)
+            tres = {'cpu': 1, 'ecpu': 0, 'mem': 0}
+            tresStr = job['args'].get('@tres', '')
+            for t in re.split('\s+', tresStr):
+                if t == '':
+                    continue
+                k, v = t.split('=')
+                tres[k] = float(v)  #type: ignore
+            policy['tres'] = tres
+
+        query = '''mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}'''
+        function = job['rid'].replace('job-definition/cloudpss/',
+                                      'function/CloudPSS/')
+        variables = {
+            'input': {
+                "args":
+                {
+                    **job['args'],
+                    '_ModelRevision' : revisionHash,
+                    '_ModelArgs' : config['args']
+                },
+                "context": [
+                    function,
+                    f"model/@sdk/{str(int(time.time() * random.random()))}",
+                    rid
+                ],
+                "policy":
+                policy,
+            }
+        }
+        r = graphql_request(query, variables)
+        if 'errors' in r:
+            raise Exception(r['errors'])
+        messageId = r['data']['job']['output']
+        id = r['data']['job']['id']
+        runner = Runner(id,messageId, name, job, config, revisionHash, rid,
+                        policy, **kwargs)
+
         event = threading.Event()
         thread = threading.Thread(target=runner.__listen, kwargs=kwargs)
         thread.setDaemon(True)
         thread.start()
-        payload = {
-            'args': config['args'],
-            'job': job,
-            'implement': kwargs.get('topology', None)
-        }
+
         while not runner.__listenStatus():
             time.sleep(0.1)
-        r = request('POST',
-                    'api/simulation/runner/' + revisionHash + '/' +
-                    str(taskId),
-                    data=json.dumps(payload))
 
         return runner
 
+    
 
 class HttpRunner(Runner[T]):
+
     def __init__(self, job, simulationId, **kwargs):
         self.simulationId = simulationId
         self.job = job
         self.__taskId = self.__getLastTask()
         result = IES_LAB_RESULT.get(job.get('rid', ''), IESLabPlanResult)
-        self.result: T = result(self.simulationId, self.__taskId, **kwargs)
+        self.result: T = result(self.simulationId, self.__taskId,
+                                **kwargs)  # type: ignore
 
     def __getLastTask(self):
         r = request('GET',

cloudpss/utils/IO.py

@@ -0,0 +1,164 @@
+import io
+import json
+import ubjson
+import yaml
+import gzip
+import base64
+import struct
+import zstandard as zstd
+
+
+def float32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('f' * (len(data) // 4), data))
+
+
+def float64Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('d' * (len(data) // 8), data))
+
+
+def uint8Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('B' * (len(data) // 1), data))
+
+
+def uint8ClampedArray(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('B' * (len(data) // 1), data))
+
+
+def uint16Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('H' * (len(data) // 2), data))
+
+
+def uint32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('I' * (len(data) // 4), data))
+
+
+def int8Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('b' * (len(data) // 1), data))
+
+
+def int16Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('h' * (len(data) // 2), data))
+
+
+def int32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('i' * (len(data) // 4), data))
+
+
+#type:ignore
+yaml.add_constructor('tag:yaml.org,2002:js/Float32Array', float32Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Float64Array', float64Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Uint8Array', uint8Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Uint8ClampedArray',
+                     uint8ClampedArray)
+yaml.add_constructor('tag:yaml.org,2002:js/Uint16Array', uint16Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Uint32Array', uint32Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Int8Array', int8Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Int16Array', int16Array)
+yaml.add_constructor('tag:yaml.org,2002:js/Int32Array', int32Array)
+
+
+class IO(object):
+    """
+    IO 模块，抽象 bytes/file <-> object 的 load/dump 操作，支持 gzip 压缩和 yaml、ubjson 序列化 读取时依据 magic number 自动识别压缩格式，写入时默认使用 gzip
+    """
+
+    @staticmethod
+    def serialize(obj, format, compress='gzip') -> bytes:
+        """
+        根据format序列化模型
+        format 支持 json, ubjson, yaml, zstd
+        compress 支持 gzip
+        """
+        result = None
+        if format == 'json':
+            result = json.dumps(obj).encode(encoding="utf-8")
+        if format == 'ubjson':
+            result = ubjson.dumpb(obj)
+        if format == 'yaml':
+            result = yaml.dump(obj).encode(encoding="utf-8")
+        if result is None:
+            assert False, 'format not support'
+        if compress == 'gzip':
+            return gzip.compress(result)
+        if compress == 'zstd':
+            return zstd.ZstdCompressor().compress(result)
+        return result
+
+    @staticmethod
+    def deserialize(byt, format):
+        """
+        根据format反序列化模型
+        format 支持 json, ubjson, yaml, zstd
+        """
+        if format == 'json':
+            return json.loads(byt)
+        if format == 'ubjson':
+            return ubjson.loadb(byt)
+        if format == 'yaml':
+            return yaml.load(io.BytesIO(byt), yaml.FullLoader)
+        assert False, 'format not support'
+
+    @staticmethod
+    def load(file, format):
+        """
+        根据format从文件中加载模型
+        format 支持 json, ubjson, yaml
+        """
+        ### 读取文件
+        f = open(file, 'r+', encoding='utf-8')
+        t = f.buffer.read(4)
+        f.close()
+        ### 判断文件格式是否是gzip或其他格式
+        if t[0:2] == b'\x1f\x8b':
+            with gzip.open(file, 'rb') as input_file:
+                return IO.deserialize(input_file.read(), format)  # type:ignore
+        if t == b'\x28\xb5\x2f\xfd':
+            with open(file, 'rb') as input_file:
+                return IO.deserialize(
+                    zstd.ZstdDecompressor().decompress(input_file.read()),
+                    format)
+        else:
+            with open(file, 'rb') as f:
+                data = f.read()
+                f.close()
+            return IO.deserialize(data, format)
+
+    @staticmethod
+    def dump(obj, file, format, compress='gzip'):
+        """
+        根据format将模型保存到文件中
+        format 支持 json, ubjson, yaml, zstd
+        compress 支持 gzip
+        """
+        ### 序列化
+        data = IO.serialize(obj, format, compress)
+        ### 写入文件
+        with open(file, 'wb') as f:
+            f.write(data)
+            f.close()
+
+
+if __name__ == '__main__':
+    obj = [
+        123, 1.25, 43121609.5543, 12345.44e40, 'a', 'here is a string', None,
+        True, False, [[1, 2], 3, [4, 5, 6], 7], {
+            'a dict': 456
+        }
+    ]
+    IO.dump(obj, 'output.json.gz', 'json')
+    IO.dump(obj, 'output.ubjson.gz', 'ubjson')
+    IO.dump(obj, 'output.yaml.gz', 'yaml')  # type: ignore
+    IO.dump(obj, 'output.yaml.zstd', 'yaml', 'zstd')  # type: ignore
+    print(IO.load('output.json.gz', 'json'))
+    print(IO.load('output.ubjson.gz', 'ubjson'))
+    print(IO.load('output.yaml.gz', 'yaml'))
+    print(IO.load('output.yaml.zstd', 'yaml'))

cloudpss/version.py

@@ -1 +1 @@
-__version__ = '4.0.3'
+__version__ = '4.0.5'

{cloudpss-4.0.3.dist-info → cloudpss-4.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudpss
-Version: 4.0.3
+Version: 4.0.5
 Summary: cloudpss sdk
 Home-page: https://www.cloudpss.net
 Author: cloudpss
@@ -21,6 +21,8 @@ Requires-Dist: requests
 Requires-Dist: websocket-client
 Requires-Dist: pytz
 Requires-Dist: deprecated
+Requires-Dist: zstandard
+Requires-Dist: py-ubjson
 
 
 CloudPSS SDK

{cloudpss-4.0.3.dist-info → cloudpss-4.0.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cloudpss/__init__.py,sha256=Ov8K45KxSsPalPErvTRUb2GvPdJ3myucFqx7s37_YZo,777
 cloudpss/verify.py,sha256=KF4Gd59DGvCyIEkRD7rNnekWw22XxJpi3DW6keb6j4c,1498
-cloudpss/version.py,sha256=WVDh-CWReADU6VLpUNeW8EQnxCJUNqQrdVIoGVLRXxQ,21
+cloudpss/version.py,sha256=F9_5WzXCbDxelG0Xz9hBfGhy0BP1_F8FDyJ64EnDdCM,21
 cloudpss/function/__init__.py,sha256=TNzChB-gQF6oB62F423evpUlgcp6s03D4pKKs89Lz4Q,1629
 cloudpss/function/function.py,sha256=llomkfnTmKMiFJYJ2mKnmAEoIjTNVCnjCUUyiJMOV1s,29
 cloudpss/function/functionExecution.py,sha256=GcyhRlgYjRnJrreKOtERExbCp6XDedUlT0L7k-uqyZM,14640
@@ -25,19 +25,21 @@ cloudpss/project/project.py,sha256=sGvJEd9yEB5NARRa64gwF96hK69gQSmwX8lQANLhBos,1
 cloudpss/runner/IESLabEvaluationResult.py,sha256=uFNLiN5kzlzSGlLIq3ybuRwB-ufYytflVrXV5g7voOU,5841
 cloudpss/runner/IESLabPlanResult.py,sha256=dCyTbMtITy9XDjE42knrpJrvaH7dvFgfQMZbx9hybRs,6862
 cloudpss/runner/IESLabTypicalDayResult.py,sha256=3egAzmv32sr4wKaVgLxsM75ScSb08F4-iLEbuPkoi_c,8001
+cloudpss/runner/MessageStreamReceiver.py,sha256=_G_7LIFdm7EtOf046nT-V3HNmygAm2oo5WYl4m27jKY,2832
 cloudpss/runner/__init__.py,sha256=aZ00V2AzQNBbER0xi6pqFFyiLFM-6r1fxvzdxxUJA_4,241
 cloudpss/runner/receiver.py,sha256=d8vOtBJ_Iqo3syJXgFFxwii1MXF7CyHRoXy6eT81Zq4,4096
 cloudpss/runner/result.py,sha256=MkoTQly7FjKj9mde3bnv4vBDfGJ2U4eK-HhxxdwmemE,13125
-cloudpss/runner/runner.py,sha256=NbBJfCVFk9n3eYsQSjTaK12zohwqKYak-lEEOo8vWDo,5316
+cloudpss/runner/runner.py,sha256=207PVoD3mjPiqWy6Qt3L5EU0woO7jmv_xj_bRuE9tDU,7299
 cloudpss/runner/storage.py,sha256=zFET_zwPIOF2Cnh9sgFiS0HFxV1OmVsU34bGUQ6PpkA,4162
 cloudpss/runner/transform.py,sha256=krOgTZiJSJAb5QSwerAqlbC4Ma0PKi__0WOZlAxw4O8,11613
+cloudpss/utils/IO.py,sha256=Uh1NOlBn15Jl5_aWS2QRO2-QsKlFT9o8DPtP8-8Z2PA,5314
 cloudpss/utils/__init__.py,sha256=NU5ZAeXH733nJKooH_1cdT9ntPEYRG4Kk3PA7P6bK_s,291
 cloudpss/utils/dataEncoder.py,sha256=5PUPb844eOGgFnYrMM8bdjdKH_MZz0lk-67uo8TvwEo,885
 cloudpss/utils/graphqlUtil.py,sha256=swl6XT1UmUQHdHjMWw3M0i8meEmVKNQvX0SHsbgg8TU,255
 cloudpss/utils/httprequests.py,sha256=-RYi1Dc4QDqqupuw0x-lq74dFvCqPrZfCJKgrdolEpk,1643
 cloudpss/utils/matlab.py,sha256=SLwVt790BjklJK2XNELt9R2n_1ej9Y8QsTIdFkKXLWE,795
 cloudpss/utils/yamlLoader.py,sha256=KRlRkHFltOjqxa_sesdBLf1W5J7XTgvubszsSw-XR0U,2885
-cloudpss-4.0.3.dist-info/METADATA,sha256=uXlgX7FBN1vtept_TPaoufb-pEoAGMZskcgbHObajwM,2318
-cloudpss-4.0.3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-cloudpss-4.0.3.dist-info/top_level.txt,sha256=wS9qPU4-aWM9ouzMOx34Nlq-GkdQKpr9vBskwut1BD8,9
-cloudpss-4.0.3.dist-info/RECORD,,
+cloudpss-4.0.5.dist-info/METADATA,sha256=iDbbOA3p31OHB5tsRdsAMTDpX03a_nQVwW8Fin3-07E,2368
+cloudpss-4.0.5.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+cloudpss-4.0.5.dist-info/top_level.txt,sha256=wS9qPU4-aWM9ouzMOx34Nlq-GkdQKpr9vBskwut1BD8,9
+cloudpss-4.0.5.dist-info/RECORD,,