openbayes-cli 0.4.5__py3-none-any.whl

bayes/root.py

@@ -0,0 +1,170 @@
+from typing import Optional
+
+import typer
+
+from .client import user_client
+from .client.base import BayesGQLClient
+from .commands import gear, ssh
+from .commands import org
+from .error import credential_notfound_error, Error
+from bayes.model.file.settings import BayesSettings, BayesEnvConfig
+from bayes.usercases.switch_usercase import clean, is_url, is_exist
+from .usercases import auth_usecase
+from .utils import Utils
+
+app = typer.Typer(
+    help="OpenBayes CLI",
+    context_settings={"help_option_names": ["-h", "--help"]}
+)
+
+app.add_typer(ssh.app, name="ssh")
+app.add_typer(gear.app, name="gear")
+app.add_typer(org.app, name="org")
+
+
+@app.callback()
+def main(ctx: typer.Context):
+    ctx.obj = BayesSettings()
+
+
+@app.command()
+def login(ctx: typer.Context,
+          username: str = typer.Argument(
+              None, envvar="OPENBAYES_TOKEN", help="用户名 | 令牌"
+          )):
+    """
+    登录
+
+    用法：
+      bayes login [用户名 | 令牌] [选项]
+
+    可用选项：
+      -h, --help   查看 login 的帮助
+    """
+    # 使用账户密码登录时：当用户名为 null 的时候要去 credentials 文件中看看 username 是否为null，因为登出的时候 username 是不覆盖为None
+    # 如果 username is None，让用户输入，否则直接显示 请输入 xxx 的密码
+    bayes_settings = ctx.obj
+    default_env: Optional[BayesEnvConfig] = bayes_settings.default_env
+    if default_env.token:
+        print(f"{default_env.username} 已成功登入 {default_env.name}")
+        raise typer.Exit()
+
+    if username is None:
+        env_username = default_env.username
+        if env_username:
+            username = env_username
+        else:
+            username = input("请输入用户名：")
+
+    gql_client = BayesGQLClient(default_env.graphQL, None)
+
+    if len(username) > 40:
+        try:
+            result = user_client.login_with_token(gql_client, username)
+            bayes_settings.login(result.username, result.token)
+
+            print(f"{result.username} 已成功登入 {default_env.name}")
+            raise typer.Exit()
+        except Error as e:
+            print(e.message)
+            raise typer.Exit(code=1)
+    else:
+        password = typer.prompt(f"请输入 {username} 的密码", hide_input=True)
+        try:
+            result = user_client.login(gql_client, username, password)
+            bayes_settings.login(result.username, result.token)
+
+            print(f"{result.username} 已成功登入 {default_env.name}")
+            raise typer.Exit()
+        except Error as e:
+            print(e.message)
+            raise typer.Exit(code=1)
+
+
+@app.command()
+def logout(ctx: typer.Context):
+    """
+    注销
+
+    用法:
+      bayes logout [选项]
+
+    可用选项:
+      -h, --help   查看 logout 的帮助
+    """
+    bayes_settings = ctx.obj
+    logout_result = bayes_settings.logout()
+    if logout_result:
+        print("已成功登出")
+    else:
+        print("登出失败")
+
+
+@app.command()
+def switch(ctx: typer.Context, name: str,
+           endpoint: str = typer.Option(None, "--endpoint", "-e", help="填入 endpoint")):
+    """
+    切换服务端环境
+
+    用法：
+       bayes switch [配置名] [选项]
+
+    可用选项：
+      -e, --endpoint string   [可选] 填入 endpoint
+      -h, --help              查看 switch 的帮助
+    """
+
+    if name == "default" and endpoint:
+        print("配置名 default 已存在，且不能被修改，请选择其他配置名")
+        raise typer.Exit(code=1)
+
+    bayes_settings = ctx.obj
+
+    if endpoint:
+        if not is_url(endpoint):
+            print(f"{endpoint} 不是合法的链接")
+            raise typer.Exit(code=1)
+
+        # 临时创建一个新的环境，然后判断是否可以访问
+        new_env = BayesEnvConfig(name=name, endpoint=clean(endpoint))
+        if not is_exist(new_env.graphQL):
+            print(f"{endpoint} 无法访问，请再次确认你所输入的链接")
+            raise typer.Exit(code=1)
+
+        bayes_settings.add_new_env(name, new_env.endpoint)
+
+    switch_result = bayes_settings.switch_default_env(name)
+    if switch_result:
+        print(f"已成功切换到 {name}")
+    else:
+        error = credential_notfound_error(name)
+        print(error)
+
+
+@app.command()
+def status(ctx: typer.Context):
+    """
+    登录信息
+
+    用法：
+        bayes status [选项]
+
+    可用选项：
+        -h, --help   查看 status 的帮助
+    """
+    settings: BayesSettings = ctx.obj
+    default_env: Optional[BayesEnvConfig] = settings.default_env
+    userinfo = auth_usecase.get_default_credential_userinfo()
+    if userinfo is not None:
+        print(f"当前环境: {default_env.endpoint}")
+        print(f"当前组织: {default_env.orgName}")
+        print(f"用户名: {default_env.username}")
+        print(f"邮箱: {userinfo.email}")
+    else:
+        print(f"当前环境: {default_env.endpoint}")
+
+
+@app.command()
+def version(ctx: typer.Context):
+    version_number = Utils.get_version_from_pyproject_toml()
+    print(f"openbayes 的版本号为:{version_number}")

bayes/templates/openbayes_zh-Hans.yaml

@@ -0,0 +1,134 @@
+## 有关「 OpenBayes 配置文件」的最新说明，请查阅 https://openbayes.com/docs/cli/config-file/
+
+## data_bindings
+#  指绑定的数据，支持「容器输出」以及「数据集」，最多同时绑定三个
+#
+#  一个完整的 data_bindings 样例如下：
+#
+#    data_bindings:
+#      - data: openbayes/mnist/1
+#        path: /input0
+#        type: ro
+#      - data: openbayes/jobs/jfaqJeLMcPM/output
+#        path: output
+#        type: rw
+#
+#  亦可将 data_bindings 替换成 bindings， 简写成如下样例：
+#
+#    bindings:
+#      - openbayes/mnist/1:/input0
+#      - openbayes/mnist/1:/input1:rw
+#      - openbayes/jobs/jfaqJeLMcPM/output:/output
+#
+data_bindings: []
+
+## resource
+#  指使用什么算力容器，通过命令 bayes gear resource 可以看到支持的算力类型
+#
+resource: "{resource}"
+
+## env
+#  指使用什么运行时环境，通过命令 bayes gear env 可以查看支持的运行时环境
+#
+env: "{env}"
+
+## command
+#  只有在创建「脚本执行」时需要，指任务执行时的入口命令
+#
+command: ""
+
+## node
+#  指定运行节点数量
+#
+node: 1
+
+## parameters
+#  支持 key / value 形式的参数，该参数会在容器执行时生成 openbayes_params.json 并补充在 command 参数后面
+#  样例如下：
+#
+#    parameters:
+#      input: /input0
+#      epochs: 5
+#
+#    在执行时会生成一个内容为 {"input": "/input0", "epochs": 5} 的 openbayes_params.json，
+#    并且会在执行命令后面追加 `--input=/input0 --epochs=5`
+#
+parameters: {}
+
+
+## 有关「 OpenBayes 自动调参」的最新说明，请查阅 https://openbayes.com/docs/hypertuning/
+#
+#  一个完整的 hyper_tuning 样例如下：
+#    hyper_tuning:
+#      max_job_count: 3
+#      hyperparameter_metric: precision
+#      goal: MINIMIZE
+#      algorithm: Bayesian
+#      parameter_specs:
+#      - name: regularization
+#        type: DOUBLE
+#        min_value: 0.001
+#        max_value: 10.0
+#        scale_type: UNIT_LOG_SCALE
+#      - name: latent_factors
+#        type: INTEGER
+#        min_value: 5
+#        max_value: 50
+#        scale_type: UNIT_LINEAR_SCALE
+#      - name: unobs_weight
+#        type: DOUBLE
+#        min_value: 0.001
+#        max_value: 5.0
+#        scale_type: UNIT_LOG_SCALE
+#      - name: feature_wt_factor
+#        type: DOUBLE
+#        min_value: 1
+#        max_value: 200
+#        scale_type: UNIT_LOG_SCALE
+#      - name: level
+#        type: DISCRETE
+#        discrete_values: [1, 2, 3, 4]
+#      - name: category
+#        type: CATEGORICAL
+#        categorical_values: ["A", "B", "C"]
+#
+hyper_tuning:
+
+  ## max_job_count
+  #  一次自动调参的尝试次数，最多支持 100 次
+  #
+  max_job_count: 0
+
+  ## parallel_count
+  #  并行的尝试个数受限于用户的单个资源类型的最大并行个数，通常是 1 或者 2
+  #
+  parallel_count: "{parallel_count}"
+
+  ## hyperparameter_metric
+  #  目标变量
+  #  有关目标变量的上报，请查阅 https://openbayes.com/docs/hypertuning/#2-上报目标变量
+  hyperparameter_metric: ""
+
+  ## goal
+  #  最优解的方向 ( MAXIMIZE 或 MINIMIZE )
+  #
+  goal: ""
+
+  ## algorithm
+  #  采用的算法，支持的算法如下：
+  #  Grid      对于只有 DISCRETE 以及 CATEGORICAL 类型参数的场景可以通过 GridSearch 遍历所有参数的组合
+  #  Random    针对 INTEGER 以及 DOUBLE 类型，依据其所支持的分布类型，在 min_value 和 max_value 之间随机选择数值，对于 DISCRETE 和 CATEGORICAL 类型，其行为和 Grid 方式类似
+  #  Bayesian  每次生成参数时考虑之前的「参数」-「目标变量」的结果，通过更新后的分布函数提供参数以期望获取更好的结果，其算法可以参考该文章
+  #
+  algorithm: ""
+
+  ## parameter_specs
+  #  输入参数的规约
+  #  参数规约的定义请查阅：https://openbayes.com/docs/hypertuning/#参数规约的定义
+  #
+  parameter_specs: []
+
+  ## side_metrics
+  #  其他参考指标
+  #
+  side_metrics: []

bayes/templates/openbayesignore.yaml

@@ -0,0 +1,3 @@
+__MACOSX
+.DS_Store
+.git

bayes/usercases/archive_usecase.py

@@ -0,0 +1,79 @@
+import logging
+import time
+import zipfile
+import archiver
+import shutil
+import os
+
+from bayes.model.file.openbayes_ignore import IGNORE_FILE_NAME, IGNORE_CLEANUPS
+from bayes.usercases.disk_usecase import IgnoreService
+
+
+def is_file_exist(zip_name, filename):
+    try:
+        with zipfile.ZipFile(zip_name, 'r') as zr:
+            for file in zr.filelist:
+                if not file.is_dir():
+                    if file.filename == filename:
+                        return True
+    except Exception as err:
+        return False, err
+
+    return False, None
+
+
+def unzip(source, target):
+    try:
+        if source.endswith('.zip'):
+            with zipfile.ZipFile(source, 'r') as zip_ref:
+                zip_ref.extractall(target)
+    except Exception as e:
+        return e
+
+
+def append_file(zip_writer, source, filename):
+    path = os.path.join(source, filename)
+
+    try:
+        # 打开文件
+        with open(path, 'rb') as file_to_zip:
+            # 获取文件信息
+            info = os.stat(path)
+
+            # 创建 ZIP 文件头
+            header = zipfile.ZipInfo(filename)
+            header.compress_type = zipfile.ZIP_DEFLATED
+            header.file_size = info.st_size
+
+            # 获取文件的修改时间并设置为 ZIP 文件头的时间
+            mtime = time.localtime(info.st_mtime)
+            header.date_time = mtime[:6]
+
+            # 将文件内容写入 ZIP 文件
+            zip_writer.writestr(header, file_to_zip.read())
+
+    except Exception as e:
+        return e
+
+    return None
+
+
+def archive(source, destination):
+    continue_on_error = True
+
+    try:
+        with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+            ignore_service = IgnoreService(IGNORE_FILE_NAME, IGNORE_CLEANUPS)
+            left_files, _, error = ignore_service.left(source)
+            if error is not None:
+                return error
+
+            for filename in left_files:
+                err = append_file(zip_file, source, filename)
+                if err is not None and not continue_on_error:
+                    return err
+    except Exception as e:
+        logging.error(f"archive: Error calling ignore_service.left: {e}")
+        return e
+
+    return None

bayes/usercases/auth_usecase.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from bayes.client import user_client
+from bayes.client.base import BayesGQLClient
+from bayes.model.file.settings import BayesSettings, BayesEnvConfig
+
+
+def get_default_credential_userinfo():
+    default_env: Optional[BayesEnvConfig] = BayesSettings().default_env
+    gql_client = BayesGQLClient(default_env.graphQL, None)
+
+    if not default_env.token:
+        print("用户还未登陆")
+        return
+
+    login_model = user_client.login_with_token(gql_client, default_env.token)
+    # 登陆成功，获得用户信息，去修改文件
+    BayesSettings().login(login_model.username, login_model.token)
+    return login_model
+
+
+def check_login():
+    default_env: Optional[BayesEnvConfig] = BayesSettings().default_env
+    if default_env.name and default_env.token:
+        return True
+    else:
+        return False
+
+
+def is_working_on_org():
+    default_env: Optional[BayesEnvConfig] = BayesSettings().default_env
+    if default_env.token and default_env.orgName:
+        return True
+    else:
+        return False

bayes/usercases/dataset_version_usecase.py

@@ -0,0 +1,61 @@
+from typing import Optional
+
+from bayes.client.base import BayesGQLClient
+from bayes.client import dataset_version_client
+from bayes.model.file.settings import BayesEnvConfig, BayesSettings
+from bayes.utils import Utils
+
+
+def get_dataset_version_for_gear_binding(party_name, flag):
+    default_env: Optional[BayesEnvConfig] = BayesSettings().default_env
+    gql_client = BayesGQLClient(default_env.graphQL, default_env.token)
+
+    list_data = []
+
+    # 先得到 PUBLIC
+    public_dataset_version_list = dataset_version_client.get_public_dataset_version_for_gear_binding(gql_client)
+
+    if public_dataset_version_list is not None:
+        for dataset_version in public_dataset_version_list.data:
+            data = [
+                "PUBLIC",
+                dataset_version.semanticBindingName,
+                Utils.date_from_now(dataset_version.createdAt),
+                f"{flag} {dataset_version.semanticBindingName}:/input[0-4]"
+            ]
+            list_data.append(data)
+
+    # user PRIVATE
+    party_dataset_version_list = dataset_version_client.get_party_private_dataset_version_for_gear_binding(gql_client,
+                                                                                                           party_name)
+
+    if party_dataset_version_list and len(party_dataset_version_list.data) > 0:
+        list_data.append(["-"])
+
+    if party_dataset_version_list is not None:
+        for dataset_version in party_dataset_version_list.data:
+            data = [
+                "PRIVATE",
+                dataset_version.semanticBindingName,
+                Utils.date_from_now(dataset_version.createdAt),
+                f"{flag} {dataset_version.semanticBindingName}:/input[0-4]"
+            ]
+            list_data.append(data)
+
+    # job output
+    party_job_output_list = dataset_version_client.get_party_job_output_for_gear_binding(gql_client, party_name)
+
+    if party_job_output_list and len(party_job_output_list.data) > 0:
+        list_data.append(["-"])
+
+    if party_job_output_list is not None:
+        for job_output in party_job_output_list.data:
+            data = [
+                "OUTPUT",
+                job_output.output.path,
+                Utils.date_from_now(job_output.createdAt),
+                f"{flag} {job_output.output.path}:/output"
+            ]
+            list_data.append(data)
+
+    return list_data

bayes/usercases/disk_usecase.py

@@ -0,0 +1,98 @@
+import os
+import logging
+from typing import List, Tuple, Optional
+
+import pathspec
+
+
+class IgnoreService:
+    def __init__(self, ignore_file_name: str, cleanups: List[str]):
+        self.ignore_file_name = ignore_file_name
+        self.cleanups = cleanups
+
+    def ignored(self, basedir: str) -> Tuple[List[str], List[str], Optional[Exception]]:
+        try:
+            basedir = os.path.realpath(basedir)
+
+            with open(self.ignore_file_name, 'r') as file:
+                ignore_patterns = file.read().splitlines()
+
+            spec = pathspec.PathSpec.from_lines('gitwildmatch', ignore_patterns + self.cleanups)
+
+            matched_files = []
+            matched_dirs = []
+
+            for root, dirs, files in os.walk(basedir):
+                for name in files:
+                    file_path = os.path.relpath(os.path.join(root, name), basedir)
+                    if spec.match_file(file_path):
+                        matched_files.append(file_path)
+                for name in dirs:
+                    dir_path = os.path.relpath(os.path.join(root, name), basedir)
+                    if spec.match_file(dir_path):
+                        matched_dirs.append(dir_path)
+
+            return matched_files, matched_dirs, None
+        except Exception as e:
+            return [], [], e
+
+    def left(self, basedir: str) -> Tuple[List[str], List[str], Optional[Exception]]:
+        try:
+            basedir = os.path.realpath(basedir)
+
+            with open(self.ignore_file_name, 'r') as file:
+                ignore_patterns = file.read().splitlines()
+
+            spec = pathspec.PathSpec.from_lines('gitwildmatch', ignore_patterns + self.cleanups)
+
+            unmatched_files = []
+            unmatched_dirs = []
+
+            for root, dirs, files in os.walk(basedir):
+                for name in files:
+                    file_path = os.path.relpath(os.path.join(root, name), basedir)
+                    if not spec.match_file(file_path):
+                        unmatched_files.append(file_path)
+                for name in dirs:
+                    dir_path = os.path.relpath(os.path.join(root, name), basedir)
+                    if not spec.match_file(dir_path):
+                        unmatched_dirs.append(dir_path)
+
+            return unmatched_files, unmatched_dirs, None
+        except Exception as e:
+            return [], [], e
+
+
+def mbyte_to_byte(mb):
+    return mb * (1 << 20)
+
+
+class DiskService:
+    def __init__(self, ignore_service):
+        self.ignore_service = ignore_service
+
+    def directory_computing(self, dir, mb_limit):
+        try:
+            left_files, _, err = self.ignore_service.left(dir)
+            if err is not None:
+                return 0, 0, err
+
+            files = 0
+            total_bytes = 0
+
+            for file in left_files:
+                file_path = os.path.join(dir, file)
+                if os.path.isfile(file_path):
+                    stat = os.stat(file_path)
+                    total_bytes += stat.st_size
+                    files += 1
+
+                if total_bytes >= mbyte_to_byte(mb_limit):
+                    return 0, 0, Exception(f"文件总大小超出限制的 {mb_limit} MB")
+
+            logging.debug(f"Get files [{files}] and sizes [{total_bytes}]")
+
+
+            return files, total_bytes, None
+        except Exception as e:
+            return 0, 0, e

bayes/usercases/gear_download_usecese.py

@@ -0,0 +1,113 @@
+import os
+import queue
+import sys
+import threading
+import time
+from typing import Optional
+
+import typer
+
+from bayes.client import gear_client
+from bayes.client.base import BayesGQLClient
+from bayes.client.gear_client import DownloadInfoPayload
+from bayes.model.file.settings import BayesSettings
+from bayes.usercases import archive_usecase
+from bayes.utils import Utils
+
+
+def is_folder_empty(target):
+    if not os.path.exists(target):
+        return True
+
+    files = os.listdir(target)
+    return len(files) == 0
+
+
+def get_download_url(id, party_name, download_from):
+    default_env = BayesSettings().default_env
+    gql_client = BayesGQLClient(default_env.graphQL, default_env.token)
+
+    return gear_client.get_output_download_link(gql_client, id, party_name, download_from)
+
+
+def get_target_file_name(target, payload: DownloadInfoPayload):
+    return os.path.join(target, payload.get_file_name())
+
+
+def is_file_exist(filename):
+    if os.path.exists(filename):
+        return True
+    else:
+        return False
+
+
+def is_continuing(filename: str) -> bool:
+    is_continuing = typer.prompt(f"{filename} 已存在在目标路径中，是否需要覆盖？ [y/N]")
+    if not is_continuing:
+        print("Operation cancelled by the user.")
+        sys.exit(1)
+    if is_continuing.lower() in ("y", "yes"):
+        return True
+    return False
+
+
+def download(target, payload):
+    filename = get_target_file_name(target, payload)
+    is_finished = queue.Queue()
+
+    print("正在下载中，请稍候")
+
+    def download_process():
+        is_done = False
+        while not is_done:
+            if not is_finished.empty():
+                result = is_finished.get()
+                if isinstance(result, tuple):
+                    is_done, err = result
+                else:
+                    raise TypeError("Queue item is not a tuple")
+
+                if is_done:
+                    print(f"\r下载完成，文件保存在 {filename}")
+                else:
+                    print(f"\r下载失败: {err}")
+                print()
+                break
+            else:
+                try:
+                    file_stat = os.stat(filename)
+                    size = Utils.byte_size(file_stat.st_size, False)
+                    print(f"\r已下载 {size}", end="")
+                except FileNotFoundError:
+                    pass
+                time.sleep(1)
+
+    download_thread = threading.Thread(target=download_process)
+    download_thread.start()
+
+    err = gear_client.download(payload.url, filename, is_finished)
+
+    if payload.is_file():
+        print(f"payload.is_file:{payload.is_file()}")
+        return "", err
+
+    return filename, err
+
+
+def rename_zip(zip_name, filename):
+    is_exist, err = archive_usecase.is_file_exist(zip_name, filename)
+    if is_exist and err is None:
+        new_file_name = zip_name.replace(".zip", "_" + Utils.generate_uid() + ".zip")
+        os.rename(zip_name, new_file_name)
+        return new_file_name
+    return zip_name
+
+
+def unzip(source, target):
+    try:
+        err = archive_usecase.unzip(source, target)
+        if err:
+            return err
+        os.remove(source)
+    except Exception as e:
+        return e