gsnet-sdk 0.1.0__py3-none-any.whl

gsnet/__init__.py

@@ -0,0 +1,32 @@
+import asyncio
+import os
+import sys
+
+from gsnet._login import login, logout, whoami
+from gsnet.refactor.constants import BOLD, DIM, RESET
+from gsnet.refactor.core import GSnetRouter
+from gsnet.refactor.plugins.log import setup_logging
+from gsnet.refactor.sdk import notice, gsnet_download_file, gsnet_upload_file
+
+from .downloader import download_file, download_model, download_model_file
+from .uploader import upload_file, upload_model, upload_model_file
+
+setup_logging()
+
+# if windows machine, do not set uvloop
+if os.name == "nt":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+else:
+    try:
+        import uvloop
+    except ImportError:
+        import subprocess
+
+        print(f"{DIM}Module {BOLD}`uvloop`{RESET} {DIM}undetected, installing...{RESET}")
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", "-qq", "uvloop"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        import uvloop
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

gsnet/_dataclass.py

@@ -0,0 +1,404 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from .utils import parse_time
+
+
+@dataclass
+class UserInfo:
+    """
+    User Info Dataclass
+
+    Attributes:
+        username (str):
+            username for GSnet links and repo_id
+        id (int):
+            unique number id of user
+        login (str):
+            username used for login purpose
+        subject_name (str):
+            self-defined name for display
+        email (str):
+            registered email address
+        avatar_url (str):
+            user avatar url
+        language (str):
+            user default language
+        is_admin (bool):
+            whether the user is admin or not
+        is_active (bool):
+            whether the user is active or not
+        last_login (str):
+            last login time
+        created (str):
+            created time
+    """
+
+    username: str
+    id: Optional[int]
+    login: Optional[str]
+    subject_name: Optional[str]
+    email: Optional[str]
+    avatar_url: Optional[str]
+    language: Optional[str]
+    is_admin: Optional[bool]
+    is_active: Optional[bool]
+    last_login: Optional[str]
+    created: Optional[str]
+
+
+@dataclass
+class RepoPermission:
+    """
+    Permission Info Dataclass
+
+    Attributes:
+        admin (bool):
+            whether the user is admin or not
+        push (bool):
+            whether the user can push to the repository or not
+        pull (bool):
+            whether the user can pull from the repository or not
+    """
+
+    admin: bool
+    push: bool
+    pull: bool
+
+
+@dataclass
+class RepoInfo:
+    """
+    Repo Info Dataclass
+
+    Attributes:
+        id (int):
+            repo id
+        owner (UserInfo):
+            owner info
+        name (str):
+            repo name
+        repo_id (str):
+            repo id in `Username/Reponame` format
+        size (int):
+            repo size
+        html_url (str):
+            repo url
+        permissions (RepoPermission):
+            repo permissions
+        created_at (str):
+            created time
+        updated_at (str):
+            updated time
+    """
+
+    id: int
+    owner: UserInfo
+    name: str
+    repo_id: str
+    size: Optional[int]
+    html_url: Optional[str]
+    permissions: Optional[RepoPermission]
+    created_at: Optional[datetime]
+    updated_at: Optional[datetime]
+
+    def __init__(self, **kwargs):
+        self.id = kwargs.pop("id")
+        self.owner = UserInfo(**kwargs.pop("owner"))
+        self.name = kwargs.pop("name")
+        self.repo_id = kwargs.pop("subject_name")
+
+        self.size = kwargs.pop("size", None)
+        self.html_url = kwargs.pop("html_url", None)
+        permissions = kwargs.pop("permissions", None)
+        self.permissions = RepoPermission(**permissions) if permissions else None
+
+        created_at = kwargs.pop("created_at", None)
+        self.created_at = parse_time(created_at) if created_at else None
+        updated_at = kwargs.pop("updated_at", None)
+        self.updated_at = parse_time(updated_at) if updated_at else None
+
+
+@dataclass
+class DatasetFile:
+    """TODO"""
+
+    uuid: str
+    name: str
+    size: int
+    type: int
+    id: int
+    datasetId: int
+    repo_id: str
+    uploaderId: Optional[int]
+    isPrivate: Optional[bool]
+    decompressState: Optional[int]
+
+    def __init__(self, **kwargs):
+        self.uuid = kwargs.pop("uuid")
+        self.name = kwargs.pop("name")
+        self.size = kwargs.pop("size")
+        self.type = kwargs.pop("type")
+        self.id = kwargs.pop("id")
+        self.datasetId = kwargs.pop("datasetId")
+        self.repo_id = kwargs.pop("repo_id")
+
+        self.uploaderId = kwargs.pop("uploaderId", None)
+        self.isPrivate = kwargs.pop("isPrivate", None)
+        self.decompressState = kwargs.pop("decompressState", None)
+
+
+@dataclass
+class DatasetInfo:
+    """TODO"""
+
+    id: str
+    userId: int
+    repoId: int
+    title: str
+    repo_id: str
+    attachments: Optional[List[DatasetFile]]
+    status: Optional[int]
+
+    def __init__(self, **kwargs):
+        self.id = kwargs.pop("id")
+        self.userId = kwargs.pop("userId")
+        self.repoId = kwargs.pop("repoId")
+        self.title = kwargs.pop("title")
+        self.repo_id = kwargs.pop("repo_id")
+
+        attachments = kwargs.pop("attachments", [])
+        if attachments:
+            for data in attachments:
+                data.update({"repo_id": self.repo_id})
+            self.attachments = [DatasetFile(**data) for data in attachments]
+        else:
+            self.attachments = None
+
+        self.status = kwargs.pop("status", None)
+        self.recommend = kwargs.pop("recommend", None)
+
+
+@dataclass
+class FileChunkInfo:
+    """
+    File Chunk Info Dataclass
+
+    API Response (dict):
+        - dataset "/attachments/get_chunks"
+            "uuid":        fileChunk.UUID,
+            "uploaded":    strconv.Itoa(fileChunk.IsUploaded),
+            "uploadID":    fileChunk.UploadID,
+            "chunks":      string(chunks),
+            "attachID":    "0",
+            "datasetID":   "0",
+            "fileName":    "",
+            "datasetName": "",
+
+        - model "/attachments/model/get_chunks"
+            "uuid":      fileChunk.UUID,
+            "uploaded":  strconv.Itoa(fileChunk.IsUploaded),
+            "uploadID":  fileChunk.UploadID,
+            "chunks":    string(chunks),
+            "attachID":  "0",
+            "modeluuid": modeluuid,
+            "datasetID": "0",
+            "fileName":  "",
+            "modelName": "",
+
+        - error response
+            "result_code": "-1",
+            "msg":         errStr,
+
+    Attributes:
+        result_code (int):
+            return 0 when success, otherwise return -1 with error message
+        msg (str):
+            return plain string error message, return "success" when None
+        uploaded (Optional[bool]):
+            whether the file is uploaded or not
+        attachID (Optional[int]):
+            attachment id
+        uploaded_chunks (Optional[list]):
+            list of uploaded chunks
+        uploadID (Optional[str]):
+            upload id
+        uuid (Optional[str]):
+            file uuid
+        repo_id (Optional[str]):
+            repo id in `Username/Reponame` format
+        dataset_or_model_name (Optional[str]):
+            dataset or model name
+    """
+
+    result_code: int
+    msg: str
+    uploaded: Optional[bool]
+    md5: Optional[str]
+    fileName: Optional[str]
+    attachID: Optional[int]
+    uploaded_chunks: Optional[list]
+    uploadID: Optional[str]
+    uuid: Optional[str]
+    repo_id: Optional[str]
+    dataset_or_model_name: Optional[str]
+    realName: Optional[str]
+    datasetID: Optional[str]
+    upload_mode: Optional[str]
+
+    def __init__(self, **kwargs):
+        result_code = kwargs.pop("result_code", 0)
+        self.result_code = int(result_code)
+        self.msg = kwargs.pop("msg", "success")
+
+        uploaded = kwargs.pop("uploaded", "")
+        self.uploaded = True if uploaded == "1" else False
+        self.attachID = kwargs.pop("attachID", None)
+
+        raw_chunks = kwargs.pop("chunks", "")
+        uploaded_chunks = [i for i in raw_chunks.split(",") if i != ""]
+        self.uploaded_chunks = uploaded_chunks
+
+        uploadID = kwargs.pop("uploadID", "")
+        self.uploadID = uploadID if uploadID != "" else None
+        uuid = kwargs.pop("uuid", "")
+        self.uuid = uuid if uuid != "" else None
+
+        repo = kwargs.pop("repoName", None)
+        owner = kwargs.pop("repoOwner", None)
+        self.repo_id = owner + "/" + repo if owner and repo else None
+
+        if kwargs.pop("upload_mode") == "dataset":
+            self.dataset_or_model_name = kwargs.pop("datasetName", "")
+        else:
+            self.dataset_or_model_name = kwargs.pop("modelName", "")
+
+        self.fileName = kwargs.pop("fileName", "")
+        self.realName = self.fileName
+        self.datasetID = kwargs.pop("datasetID", "")
+        self.md5 = kwargs.pop("md5", None)
+        self.upload_mode = kwargs.pop("upload_mode", None)
+
+
+@dataclass
+class NewMultipart:
+    """TODO"""
+
+    result_code: int
+    msg: str
+    uuid: Optional[str]
+    uploadID: Optional[str]
+
+    def __init__(self, **kwargs):
+        result_code = kwargs.pop("result_code", 0)
+        self.result_code = int(result_code)
+        self.msg = kwargs.pop("msg", "success")
+        self.uuid = kwargs.pop("uuid", None)
+        self.uploadID = kwargs.pop("uploadID", None)
+
+
+@dataclass
+class MultipartUrl:
+    """TODO"""
+
+    url: Optional[str] = None
+    msg: Optional[str] = None
+
+
+@dataclass
+class ModelFile:
+    FileName: Optional[str]
+    ModTime: Optional[str]
+    IsDir: Optional[bool]
+    Size: Optional[int]
+    ParenDir: Optional[str]
+    UUID: Optional[str]
+    RelativePath: Optional[str]
+    repo_id: Optional[str]
+    model_id: Optional[str]
+    IsSupportPreview: Optional[str] = ""
+
+
+@dataclass
+class ModelInfo:
+    id: Optional[str]
+    name: Optional[str]
+    size: Optional[int]
+    path: Optional[str]
+    repo_id: Optional[str]
+    repoId: Optional[int]
+    userId: Optional[int]
+    userName: Optional[str]
+    isCanOper: Optional[bool]
+    isCanDelete: Optional[bool]
+    isCanDownload: Optional[bool]
+    repoName: Optional[str]
+    repoOwnerName: Optional[str]
+    modelFileList: Optional[List[ModelFile]]
+    modelType: Optional[int]
+
+    def __init__(self, **kwargs):
+        self.id = kwargs.pop("id", None)
+        self.name = kwargs.pop("name", None)
+        self.size = kwargs.pop("size", None)
+        self.path = kwargs.pop("path", None)
+        self.repo_id = kwargs.pop("repo_id", None)
+        self.repoId = kwargs.pop("repoId", None)
+        self.userId = kwargs.pop("userId", None)
+        self.userName = kwargs.pop("userName", None)
+        self.isCanOper = kwargs.pop("isCanOper", None)
+        self.isCanDelete = kwargs.pop("isCanDelete", None)
+        self.isCanDownload = kwargs.pop("isCanDownload", None)
+        self.repoName = kwargs.pop("repoName", None)
+        self.repoOwnerName = kwargs.pop("repoOwnerName", None)
+
+        modelFileList = kwargs.pop("modelFileList", [])
+        if modelFileList:
+            for data in modelFileList:
+                data.update({"repo_id": self.repo_id, "model_id": self.id})
+            self.modelFileList = [ModelFile(**data) for data in modelFileList]
+        else:
+            self.modelFileList = None
+
+        self.modelType = kwargs.pop("modelType", None)
+
+
+@dataclass
+class ModelCreate:
+    code: str
+    msg: Optional[str] = None
+    id: Optional[int] = None
+
+
+@dataclass
+class BasicResp:
+    code: int
+    msg: str
+    data: any
+
+
+@dataclass
+class CreateRepoOption:
+    name: str
+    alias: Optional[str] = None
+    description: Optional[str] = None
+    private: bool = False
+    issue_labels: Optional[str] = None
+    auto_init: bool = False
+    gitignores: Optional[str] = None
+    license: Optional[str] = None
+    readme: Optional[str] = None
+    default_branch: Optional[str] = None
+
+    def __post_init__(self):
+        if len(self.name) > 100:
+            raise ValueError("Name must be at most 100 characters long")
+        if self.alias and len(self.alias) > 100:
+            raise ValueError("Alias must be at most 100 characters long")
+        if self.description and len(self.description) > 255:
+            raise ValueError("Description must be at most 255 characters long")
+        if self.default_branch and len(self.default_branch) > 100:
+            raise ValueError("Default branch must be at most 100 characters long")

gsnet/_exceptions.py

@@ -0,0 +1,273 @@
+import logging
+import re
+from functools import wraps
+from inspect import signature
+from pathlib import Path
+from typing import Callable, Type, TypeVar
+
+from gsnet.refactor.constants import REPO_ID_REGEX
+
+logger = logging.getLogger(__name__)
+
+
+class TokenNotFoundWarn(Warning):
+    def __init__(self):
+        self.message = (
+            "未找到本地 token；若遇到提示 PageNotFound 或 UnauthorizedError，"
+            "则需要使用 `login` 保存GSnet账户 token 到本地以获取授权。"
+        )
+
+    def __str__(self):
+        return repr(self.message)
+
+
+class GSnetError(BaseException):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return "❌ " + self.msg
+
+    def __repr__(self):
+        return "❌ " + self.msg
+
+
+class ServerError(GSnetError):
+    def __init__(self, status_code: int):
+        self.msg = f"{status_code} GSnet服务器暂时不可用，请联系社区管理员或稍后再试"
+
+
+class HttpProxyError(GSnetError):
+    def __init__(self):
+        self.msg = "无法发送请求，请检查网络代理"
+
+
+class HttpConnectionError(GSnetError):
+    def __init__(self):
+        self.msg = "无法连接网络，请检查网络设置"
+
+
+class InvalidTokenError(GSnetError):
+    def __init__(self):
+        self.msg = f"Invalid or broken GSnet token."
+
+
+class TokenNotFoundError(GSnetError):
+    def __init__(self):
+        token_help_doc_url = "https://gsnet.pcl.ac.cn/docs/index.html#/api/token"
+        self.msg = f"未找到本地 token；请先使用 `gsnet login` 保存GSnet账户 token 到本地以获取授权。具体使用方法请参考: {token_help_doc_url}"
+
+
+class UnauthorizedError(GSnetError):
+    def __init__(self, status_code: int = 401):
+        self.msg = f"{status_code} 您无权建立此连接。请确保您拥有对应的权限。"
+
+
+class PageNotFound(GSnetError):
+    def __init__(self):
+        self.msg = "无效请求，请确保仓库路径填写正确并且您拥有对应的权限"
+
+
+class ParamsTypeError(GSnetError):
+    def __init__(
+        self,
+        arg_name: str,
+        arg_value: any,
+        accepted_type: Type[any],
+    ):
+        self.msg = f"`{arg_name}` 参数类型错误 {type(arg_value)}: `{arg_value}`. " f"正确的参数类型为 {accepted_type}"
+
+
+class RepoIdError(GSnetError):
+    def __init__(self, repo_id):
+        self.msg = f"`{repo_id}` 参数格式错误. 正确的 `repo_id` 参数格式应为: `用户名/仓库名`"
+
+
+class RepoNotFoundError(GSnetError):
+    def __init__(self, repo_id):
+        self.msg = f"仓库不存在: `{repo_id}`"
+
+
+class RepoExistsError(GSnetError):
+    def __init__(self, repo_id):
+        self.msg = f"仓库已存在: `{repo_id}`，请勿重复创建"
+
+
+class KeyboardInterruptError(GSnetError):
+    def __init__(self, msg):
+        self.msg = f"用户中断操作; {msg}"
+
+
+class DatasetNotFound(GSnetError):
+    def __init__(self, repo_id, dataset_url):
+        self.msg = f"`{repo_id}` 仓库尚未创建数据集: {dataset_url}"
+
+
+class EmptyDataset(GSnetError):
+    def __init__(self, repo_id, dataset_url):
+        self.msg = f"`{repo_id}` 仓库数据集内未找到任何文件: {dataset_url}"
+
+
+class ModelCreateError(GSnetError):
+    def __init__(self, model_name, msg):
+        self.msg = f" `{model_name}` 模型创建失败: {msg}"
+
+
+class ModelNotFoundError(GSnetError):
+    def __init__(self, model_name, model_list_url):
+        self.msg = f"模型不存在 `{model_name}`; 请在网页端查看正确模型名称: {model_list_url}"
+
+
+class ServerFileExistsError(GSnetError):
+    def __init__(
+        self,
+        filename: str,
+        existing_filename: str,
+        existing_repo_id: str = None,
+    ):
+        if existing_repo_id:
+            self.msg = (
+                f"`{filename}` 该文件已被上传为 {existing_repo_id} 仓库中的 `{existing_filename}` 文件, 无法重复上传"
+            )
+        else:
+            self.msg = f"`{filename}` 文件已上传, 无法重复上传"
+
+
+class ServerFileNotFoundError(GSnetError):
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class EmptyFolderError(GSnetError):
+    def __init__(self, local_dir: Path):
+        self.msg = f"空目录，未能找到任何本地文件: {local_dir.absolute()}"
+
+
+class LocalPathNotFound(GSnetError):
+    def __init__(self, filepath: Path):
+        self.msg = f"本地文件或目录不存在: {filepath.absolute()}"
+
+
+class LocalDirNotFound(GSnetError):
+    def __init__(self, local_dir: Path):
+        self.msg = f"本地目录不存在: {local_dir.absolute()}"
+
+
+class FileTypeError(GSnetError):
+    def __init__(self, filepath: Path):
+        self.msg = f"非压缩文件(`.zip` 或 `.tar.gz`): {filepath.absolute()}"
+
+
+class FileSizeError(GSnetError):
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class PutUploadError(GSnetError):
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class UploadError(GSnetError):
+    def __init__(self, msg):
+        self.msg = msg
+
+
+CallableT = TypeVar("CallableT", bound=Callable)
+
+
+def validate_gsnet_args(fn: CallableT) -> CallableT:
+    """Decorator that retrieves and prints all input arguments of a function.
+
+    Args:
+        fn: The function to be decorated.
+
+    Returns:
+        A wrapper function that prints arguments and calls the original function.
+    """
+
+    @wraps(fn)
+    def _inner_fn(*args, **kwargs):
+        params = signature(fn).parameters
+        arg_names = [param.name for param in params.values()]
+
+        _kwargs = dict()
+
+        for i, arg_name in enumerate(arg_names):
+            if i < len(args):
+                value = args[i]
+            else:
+                value = kwargs.get(arg_name, None)
+            _kwargs.update({arg_name: value})
+
+        repo_id = _kwargs.get("repo_id", None)
+        if not repo_id or not isinstance(repo_id, str):
+            raise ParamsTypeError(
+                arg_name="repo_id",
+                arg_value=repo_id,
+                accepted_type=str,
+            )
+        if not re.match(REPO_ID_REGEX, repo_id):
+            raise RepoIdError(
+                repo_id=repo_id,
+            )
+
+        filename = _kwargs.get("file", None)
+        if filename and not isinstance(filename, (Path, str)):
+            raise ParamsTypeError(
+                arg_name="file",
+                arg_value=filename,
+                accepted_type="either a pathlib.Path() object or str",
+            )
+
+        cluster = _kwargs.get("cluster", None)
+        if cluster and (not isinstance(cluster, str) or cluster.lower() not in ["gpu", "npu"]):
+            raise ParamsTypeError(
+                arg_name="cluster",
+                arg_value=cluster,
+                accepted_type="['gpu', 'npu'] case insensitive",
+            )
+
+        save_path = _kwargs.get("save_path", None)
+        if save_path and not isinstance(save_path, (Path, str)):
+            raise ParamsTypeError(
+                arg_name="save_path",
+                arg_value=save_path,
+                accepted_type="either a pathlib.Path() object or str",
+            )
+
+        force = _kwargs.get("force", None)
+        if force and not isinstance(force, bool):
+            raise ParamsTypeError(
+                arg_name="force",
+                arg_value=force,
+                accepted_type=bool,
+            )
+
+        model_name = _kwargs.get("model_name", None)
+        if model_name and not isinstance(model_name, str):
+            raise ParamsTypeError(
+                arg_name="model_name",
+                arg_value=model_name,
+                accepted_type=str,
+            )
+
+        upload_name = _kwargs.get("upload_name", None)
+        if upload_name and not isinstance(upload_name, str):
+            raise ParamsTypeError(
+                arg_name="upload_name",
+                arg_value=upload_name,
+                accepted_type=str,
+            )
+
+        folder = _kwargs.get("folder", None)
+        if folder and not isinstance(folder, (Path, str)):
+            raise ParamsTypeError(
+                arg_name="folder",
+                arg_value=folder,
+                accepted_type=str,
+            )
+
+        return fn(*args, **kwargs)
+
+    return _inner_fn