cloudpss 4.0.2__py3-none-any.whl → 4.1.1a1__py3-none-any.whl

cloudpss/model/revision.py

@@ -1,3 +1,4 @@
+from cloudpss.utils.httpAsyncRequest import graphql_fetch
 from .topology import ModelTopology
 from .implements import ModelImplement
 from ..utils import request, graphql_request
@@ -21,6 +22,7 @@ class ModelRevision(object):
 
 
     """
+
     def __init__(self, revision: dict = {}):
         """
             初始化
@@ -54,7 +56,7 @@ class ModelRevision(object):
 
         return self.implements
 
-    def run(self, job, config, name=None, rid='', **kwargs):
+    async def run(self, job, config, name=None, rid='', policy=None, **kwargs):
         """
             运行某个指定版本的项目
 
@@ -63,16 +65,14 @@ class ModelRevision(object):
             :params name:  任务名称，为空时使用项目的参数方案名称和计算方案名称
             :params rid:  项目rid，可为空
 
-            :return: 返回一个运行实例
+            :return: 返回一个ModelRevision
 
             >>> revision.run(revision,job,config,'')
         """
-        revision = ModelRevision.create(self)
-        return Runner.create(revision['hash'], job, config, name, rid,
-                             **kwargs)
+        return await ModelRevision.create(self)
 
     @staticmethod
-    def create(revision, parentHash=None):
+    async def create(revision, parentHash=None):
         """
             创建一个新版本
 
@@ -93,12 +93,12 @@ class ModelRevision(object):
         del r['hash']
         variables = {'a': {**r, 'parent': parentHash}}
 
-        r = graphql_request(query, variables)
+        r = await graphql_fetch(query, variables)
         if 'errors' in r:
             raise Exception(r['errors'])
         return r['data']['createModelRevision']
 
-    def fetchTopology(self, implementType, config, maximumDepth):
+    async def fetchTopology(self, implementType, config, maximumDepth):
         """
             获取当前项目版本的拓扑数据
 
@@ -115,6 +115,6 @@ class ModelRevision(object):
         """
 
         if self.hash is not None:
-            return ModelTopology.fetch(self.hash, implementType, config,
+            return await ModelTopology.fetch(self.hash, implementType, config,
                                        maximumDepth)
         return None

cloudpss/model/topology.py

@@ -1,4 +1,6 @@
 import json
+
+from cloudpss.utils.httpAsyncRequest import graphql_fetch
 from ..utils import request, graphql_request
 
 
@@ -44,7 +46,7 @@ class ModelTopology():
         f.close()
 
     @staticmethod
-    def fetch(hash, implementType, config, maximumDepth=None):
+    async def fetch(hash, implementType, config, maximumDepth=None):
         """
             获取拓扑
 
@@ -75,7 +77,7 @@ class ModelTopology():
                 'maximumDepth': maximumDepth
             }
         }
-        data = graphql_request(query, variables)
+        data = await graphql_fetch(query, variables)
         if 'errors' in data:
             raise Exception(data['errors'][0]['message'])
 

cloudpss/runner/__init__.py

@@ -1,8 +1,10 @@
-
 from .receiver import Receiver
 from .result import PowerFlowResult, EMTResult, Result
 from .storage import Storage
 from .runner import Runner
+from .MessageStreamReceiver import MessageStreamReceiver, Message
 
-__all__ = ['Runner', 'Result', 'PowerFlowResult',
-           'EMTResult', 'Receiver', 'Storage']
+__all__ = [
+    'Runner', 'Result', 'PowerFlowResult', 'EMTResult', 'Receiver', 'Storage',
+    'MessageStreamReceiver', 'Message'
+]

cloudpss/runner/receiver.py

@@ -8,6 +8,7 @@ from .transform import Transformer
 import datetime
 import pytz
 import ssl
+import time
 
 utc_tz = pytz.timezone('UTC')
 
@@ -62,6 +63,7 @@ class Receiver(object):
             return
         try:
             payload = json.loads(message)
+            payload['when'] = datetime.datetime.now()
             msg = payload
             try:
                 if ('version' in payload and payload['version'] == 1):

cloudpss/runner/runner.py

@@ -3,20 +3,27 @@ import threading
 import json
 import time
 import random
+
+from deprecated import deprecated
+
+from cloudpss.utils.graphqlUtil import graphql_request
 from .receiver import Receiver
+from .MessageStreamReceiver import MessageStreamReceiver
 from .result import IESLabSimulationResult, PowerFlowResult, EMTResult, Result, IESResult
 from .IESLabPlanResult import IESLabPlanResult
 from .IESLabEvaluationResult import IESLabEvaluationResult
 from .IESLabTypicalDayResult import IESLabTypicalDayResult
 from .storage import Storage
 from ..utils import request
-from typing import  TypeVar,  Generic
+from typing import TypeVar, Generic
+import re
 
 RECEIVER = {
-    'default': Receiver,
+    'default': MessageStreamReceiver,
 }
 T = TypeVar('T', Result, EMTResult, PowerFlowResult, IESResult,
-            IESLabSimulationResult, IESLabPlanResult, IESLabEvaluationResult,IESLabTypicalDayResult)
+            IESLabSimulationResult, IESLabPlanResult, IESLabEvaluationResult,
+            IESLabTypicalDayResult)
 
 IES_LAB_RESULT = {
     'function/ieslab/plan': IESLabPlanResult,
@@ -39,9 +46,12 @@ RESULT_DB = {
     'function/CloudPSS/ieslab-optimization': IESResult,
 }
 
+@deprecated(version='4.0', reason="该类将在 6.0 版本移除，请使用 Job 类代替")
 class Runner(Generic[T]):
-    def __init__(self, taskId, name, job, config, revision, modelRid,
+
+    def __init__(self, id,taskId, name, job, config, revision, modelRid, policy,
                  **kwargs):
+        self.id = id
         self.taskId = taskId
         self.db = Storage(taskId, name, job, config, revision, modelRid)
         rid =job['rid'].replace('job-definition/','function/').replace('/cloudpss/','/CloudPSS/')
@@ -87,10 +97,37 @@ class Runner(Generic[T]):
         结束当前运行的算例
 
         """
-        r = request("DELETE", 'api/simulation/runner/' + str(self.taskId))
+        self.abort(0)
+    
+    def abort(self,timeout=3):
+        """
+        结束当前运行的算例
+
+        """
+        query = '''mutation ($input: AbortJobInput!) {
+            job: abortJob(input: $input) {
+                id
+                status
+            }
+        }
+        '''
+        variables = {
+            'input': {
+                'id': self.taskId,
+                'timeout': timeout
+            }
+        }
+        graphql_request(query, variables)
+
 
     @staticmethod
-    def create(revisionHash, job, config, name=None, rid='', **kwargs):
+    def create(revisionHash,
+               job,
+               config,
+               name=None,
+               rid='',
+               policy=None,
+               **kwargs):
         '''
             创建一个运行任务
 
@@ -104,35 +141,71 @@ class Runner(Generic[T]):
 
             >>> runner = Runner.runRevision(revision,job,config,'')
         '''
-        taskId = str(int(time.time() * random.random()))
 
-        runner = Runner(taskId, name, job, config, revisionHash, rid, **kwargs)
+        #处理policy字段
+        if policy is None:
+            policy = {}
+            if policy.get('tres', None) is None:
+                policy['tres'] = {}
+            policy['queue'] = job['args'].get('@queue', 1)
+            policy['priority'] = job['args'].get('@priority', 0)
+            tres = {'cpu': 1, 'ecpu': 0, 'mem': 0}
+            tresStr = job['args'].get('@tres', '')
+            for t in re.split('\s+', tresStr):
+                if t == '':
+                    continue
+                k, v = t.split('=')
+                tres[k] = float(v)  #type: ignore
+            policy['tres'] = tres
+
+        query = '''mutation($input:CreateJobInput!){job:createJob(input:$input){id input output status position}}'''
+        function = job['rid'].replace('job-definition/cloudpss/',
+                                      'function/CloudPSS/')
+        variables = {
+            'input': {
+                "args":
+                {
+                    **job['args'],
+                    '_ModelRevision' : revisionHash,
+                    '_ModelArgs' : config['args']
+                },
+                "context": [
+                    function,
+                    f"model/@sdk/{str(int(time.time() * random.random()))}"
+                ],
+                "policy":
+                policy,
+            }
+        }
+        r = graphql_request(query, variables)
+        if 'errors' in r:
+            raise Exception(r['errors'])
+        messageId = r['data']['job']['output']
+        id = r['data']['job']['id']
+        runner = Runner(id,messageId, name, job, config, revisionHash, rid,
+                        policy, **kwargs)
+
         event = threading.Event()
         thread = threading.Thread(target=runner.__listen, kwargs=kwargs)
         thread.setDaemon(True)
         thread.start()
-        payload = {
-            'args': config['args'],
-            'job': job,
-            'implement': kwargs.get('topology', None)
-        }
+
         while not runner.__listenStatus():
             time.sleep(0.1)
-        r = request('POST',
-                    'api/simulation/runner/' + revisionHash + '/' +
-                    str(taskId),
-                    data=json.dumps(payload))
 
         return runner
 
+    
 
 class HttpRunner(Runner[T]):
+
     def __init__(self, job, simulationId, **kwargs):
         self.simulationId = simulationId
         self.job = job
         self.__taskId = self.__getLastTask()
         result = IES_LAB_RESULT.get(job.get('rid', ''), IESLabPlanResult)
-        self.result: T = result(self.simulationId, self.__taskId, **kwargs)
+        self.result: T = result(self.simulationId, self.__taskId,
+                                **kwargs)  # type: ignore
 
     def __getLastTask(self):
         r = request('GET',

cloudpss/utils/AsyncIterable.py

@@ -0,0 +1,27 @@
+import asyncio
+
+
+class CustomAsyncIterable:
+    async def __init__(self, async_func, *args):
+        self.async_func = async_func
+        self.queue = asyncio.Queue()
+        self.index = 0
+        self.args = args
+
+    async def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        try:
+            result = await self.queue.get()
+            if result is None:
+                raise StopAsyncIteration
+            return result
+        except asyncio.QueueEmpty:
+            tasks = []
+            for arg in self.args:
+                tasks.append(asyncio.create_task(self.async_func(arg)))
+            await asyncio.gather(*tasks)
+            for task in tasks:
+                await self.queue.put(await task)
+            return await self.__anext__()

cloudpss/utils/__init__.py

@@ -3,5 +3,9 @@ from .httprequests import request
 from .yamlLoader import fileLoad
 from .dataEncoder import MatlabDataEncoder, DateTimeEncode
 from .graphqlUtil import graphql_request
+from .IO import IO
 
-__all__ = ['request', 'fileLoad', 'MatlabDataEncoder', 'DateTimeEncode', 'graphql_request']
+__all__ = [
+    'request', 'fileLoad', 'MatlabDataEncoder', 'DateTimeEncode',
+    'graphql_request', 'IO'
+]

cloudpss/utils/httpAsyncRequest.py

@@ -0,0 +1,88 @@
+import json
+import os
+from aiohttp import ClientSession, WSMsgType
+from urllib.parse import urljoin
+import logging
+from ..version import __version__
+
+
+def graphql_version_check(uri, response):
+    if uri.startswith("graphql"):
+        if "X-Cloudpss-Version" not in response.headers:
+            raise Exception("当前SDK版本（ver 3.X.X）与服务器版本（3.0.0 以下）不兼容，请更换服务器地址或更换SDK版本。")
+        os.environ["X_CLOUDPSS_VERSION"] = response.headers["X-Cloudpss-Version"]
+        if float(response.headers["X-Cloudpss-Version"]) >= 5:
+            raise Exception(
+                "当前SDK版本（ver "
+                + __version__
+                + "）与服务器版本（ver "
+                + response.headers["X-Cloudpss-Version"]
+                + ".X.X）不兼容，请更换服务器地址或更换SDK版本(pip 使用 pip install -U cloudpss 命令更新, conda 使用 conda update cloudpss 命令更新)。"
+            )
+
+
+### 通过aiohttp实现请求
+async def fetch_data(method: str, uri, data, baseUrl=None, params={}, **kwargs):
+    if baseUrl == None:
+        baseUrl = os.environ.get("CLOUDPSS_API_URL", "https://cloudpss.net/")
+    url = urljoin(baseUrl, uri)
+    token = os.environ.get("CLOUDPSS_TOKEN", None)
+    if token:
+        headers = {
+            "Authorization": "Bearer " + token,
+            "Content-Type": "application/json; charset=utf-8",
+        }
+    else:
+        raise Exception("token undefined")
+    logging.debug("fetch start:")
+    async with ClientSession() as session:
+        async with session.request(
+            method, url, data=data, params=params, headers=headers
+        ) as response:
+            if response.status == 200:
+                data = await response.json()
+                logging.debug("data:")
+                logging.debug(data)
+                logging.debug("params:")
+                logging.debug(params)
+                graphql_version_check(uri, response)
+                # if "statusCode" in r.text:
+                #     t = json.loads(r.text)
+                #     raise  Exception(str(t['statusCode']) + " "+t['message'])
+                return data
+            elif 400 <= response.status < 500:
+                raise Exception(f"请求失败，状态码：{response.status}")
+            elif 500 <= response.status < 600:
+                raise Exception(f"请求失败，状态码：{response.status}")
+            else:
+                return
+
+
+# graphql实现方法
+async def graphql_fetch(query, variables=None):
+    payload = {"query": query, "variables": variables}
+    return await fetch_data("POST", "graphql", data=json.dumps(payload))
+
+
+# websocket
+async def websocket_connect(url, open_func, receive_func, closed_func, error_func):
+    async with ClientSession() as session:
+        async with session.ws_connect(url) as ws:
+            # ws.register_callback("connecting", lambda: logging.debug("连接正在建立..."))
+            # ws.register_callback("connected", lambda: open_func())
+            # ws.register_callback("closed", lambda: closed_func())
+            # ws.register_callback("error", lambda: error_func())
+            open_func()
+            async for msg in ws:
+                if msg.type == WSMsgType.BINARY:
+                    receive_func(msg.data)
+                if msg.type == WSMsgType.TEXT:
+                    receive_func(msg.data)
+                elif msg.type == WSMsgType.CLOSED:
+                    logging.debug("WebSocket连接已关闭")
+                    closed_func()
+                    break
+                elif msg.type == WSMsgType.ERROR:
+                    logging.debug(f"WebSocket连接发生错误：{msg.data}")
+                    error_func(msg.data)
+                    break

cloudpss/utils/httprequests.py

@@ -5,6 +5,9 @@ import requests
 import os
 from collections import OrderedDict
 from ..version import __version__
+import logging
+
+
 def request(method, uri, baseUrl=None, params={}, token=None, **kwargs):
     if baseUrl == None:
         baseUrl = os.environ.get('CLOUDPSS_API_URL', 'https://cloudpss.net/')
@@ -17,6 +20,7 @@ def request(method, uri, baseUrl=None, params={}, token=None, **kwargs):
         }
     else:
         raise Exception('token undefined')
+    logging.debug(kwargs['data'])
     r = requests.request(method, url, params=params, headers=headers, **kwargs)
 
     if (uri.startswith('graphql')):
@@ -31,9 +35,10 @@ def request(method, uri, baseUrl=None, params={}, token=None, **kwargs):
 
     if r.ok:
         return r
+    logging.debug(r.text)
     if r.text =="":
         r.raise_for_status()
     if "statusCode" in r.text:
         t = json.loads(r.text)
-        raise  Exception( str(t['statusCode']) + " "+t['message'])
+        raise  Exception(str(t['statusCode']) + " "+t['message'])
     

cloudpss/utils/yamlLoader.py

@@ -5,63 +5,62 @@ import struct
 import base64
 
 
-class JavascriptSchema():
-    def float32Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('f' * (len(data) // 4), data))
 
-    def float64Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('d' * (len(data) // 8), data))
+def float32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('f' * (len(data) // 4), data))
 
-    def uint8Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('B' * (len(data) // 1), data))
+def float64Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('d' * (len(data) // 8), data))
 
-    def uint8ClampedArray(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('B' * (len(data) // 1), data))
+def uint8Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('B' * (len(data) // 1), data))
 
-    def uint16Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('H' * (len(data) // 2), data))
+def uint8ClampedArray(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('B' * (len(data) // 1), data))
 
-    def uint32Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('I' * (len(data) // 4), data))
+def uint16Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('H' * (len(data) // 2), data))
 
-    def int8Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('b' * (len(data) // 1), data))
+def uint32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('I' * (len(data) // 4), data))
 
-    def int16Array(self, node):
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('h' * (len(data) // 2), data))
+def int8Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('b' * (len(data) // 1), data))
 
-    def int32Array(self, node):
-
-        data = base64.b64decode(node.value)
-        return list(struct.unpack('i' * (len(data) // 4), data)).toList()
+def int16Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('h' * (len(data) // 2), data))
 
+def int32Array(_loader, node):
+    data = base64.b64decode(node.value)
+    return list(struct.unpack('i' * (len(data) // 4), data))
 
+#type:ignore
 yaml.add_constructor('tag:yaml.org,2002:js/Float32Array',
-                     JavascriptSchema.float32Array)
+                     float32Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Float64Array',
-                     JavascriptSchema.float64Array)
+                     float64Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Uint8Array',
-                     JavascriptSchema.uint8Array)
+                     uint8Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Uint8ClampedArray',
-                     JavascriptSchema.uint8ClampedArray)
+                     uint8ClampedArray)
 yaml.add_constructor('tag:yaml.org,2002:js/Uint16Array',
-                     JavascriptSchema.uint16Array)
+                     uint16Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Uint32Array',
-                     JavascriptSchema.uint32Array)
+                     uint32Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Int8Array',
-                     JavascriptSchema.int8Array)
+                     int8Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Int16Array',
-                     JavascriptSchema.int16Array)
+                     int16Array)
 yaml.add_constructor('tag:yaml.org,2002:js/Int32Array',
-                     JavascriptSchema.int32Array)
+                     int32Array)
 
 
 def fileLoad(fileName):
@@ -71,11 +70,11 @@ def fileLoad(fileName):
     data = None
     if t == b'\x1f\x8b':
         with gzip.open(fileName, 'rb') as input_file:
-            with io.TextIOWrapper(input_file, encoding='utf-8') as dec:
+            with io.TextIOWrapper(input_file, encoding='utf-8') as dec: # type: ignore
                 r = dec.read()
-                data = yaml.load(r, Loader=yaml.FullLoader)
+                data = yaml.load(r, _loader=yaml.Full_loader) # type: ignore
     else:
         f = open(fileName, 'r+', encoding='utf-8')
-        data = yaml.load(f, Loader=yaml.FullLoader)
+        data = yaml.load(f, _loader=yaml.Full_loader) # type: ignore
         f.close()
     return data

cloudpss/version.py

@@ -1 +1 @@
-__version__ = '4.0.2'
+__version__ = '4.1.1-alpha.1'

{cloudpss-4.0.2.dist-info → cloudpss-4.1.1a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudpss
-Version: 4.0.2
+Version: 4.1.1a1
 Summary: cloudpss sdk
 Home-page: https://www.cloudpss.net
 Author: cloudpss
@@ -21,6 +21,8 @@ Requires-Dist: requests
 Requires-Dist: websocket-client
 Requires-Dist: pytz
 Requires-Dist: deprecated
+Requires-Dist: py-ubjson
+Requires-Dist: aiohttp
 
 
 CloudPSS SDK