modelscope-hub 0.4.2__tar.gz → 0.4.4__tar.gz

{modelscope_hub-0.4.2/src/modelscope_hub.egg-info → modelscope_hub-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelscope-hub
-Version: 0.4.2
+Version: 0.4.4
 Summary: The official Python client to connect with ModelScope Hub.
 License: Apache-2.0
 Keywords: modelscope,hub,sdk,openapi,machine-learning

{modelscope_hub-0.4.2 → modelscope_hub-0.4.4}/src/modelscope_hub/_legacy_api.py

@@ -14,6 +14,7 @@ needed for API calls.
 
 from __future__ import annotations
 
+import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import IO, Any, BinaryIO
@@ -24,18 +25,28 @@ from requests.adapters import HTTPAdapter, Retry
 
 from .constants import (
     API_CONNECT_TIMEOUT,
+    API_CONNECTION_POOL_MAXSIZE,
     API_MAX_RETRIES,
     API_TIMEOUT,
     LEGACY_API_PREFIX,
     REPO_FILES_TRUNCATION_LIMIT,
     REPO_TREE_MAX_REQUESTS,
+    REPO_TREE_PAGE_MAX_ATTEMPTS,
+    REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS,
     REPO_TREE_WALK_WORKERS,
     UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS,
     UPLOAD_BLOB_READ_TIMEOUT_SECONDS,
     UPLOAD_HTTP_RETRY_ALLOWED_METHODS,
     RepoType,
 )
-from .errors import InvalidParameter, NetworkError, RequestTimeoutError, ServerError, raise_for_status
+from .errors import (
+    InvalidParameter,
+    NetworkError,
+    PermissionDeniedError,
+    RequestTimeoutError,
+    ServerError,
+    raise_for_status,
+)
 from .utils.logger import get_logger
 
 logger = get_logger("legacy_api")
@@ -100,6 +111,10 @@ class LegacyClient:
         self._endpoint = endpoint.rstrip("/")
         self._timeout: int | tuple[int, int] = (API_CONNECT_TIMEOUT, timeout)
         self._session_authenticated = False
+        # Set once any file-tree read succeeds. From then on a 403 on a tree
+        # request cannot be an authorization result, so it is retried instead of
+        # aborting an enumeration that spans hundreds of requests.
+        self._tree_reads_ok = False
 
         self._session = requests.Session()
         if user_agent:
@@ -110,7 +125,11 @@ class LegacyClient:
             status_forcelist=[429, 500, 502, 503, 504],
             allowed_methods=UPLOAD_HTTP_RETRY_ALLOWED_METHODS,
         )
-        adapter = HTTPAdapter(max_retries=retry)
+        adapter = HTTPAdapter(
+            max_retries=retry,
+            pool_connections=API_CONNECTION_POOL_MAXSIZE,
+            pool_maxsize=API_CONNECTION_POOL_MAXSIZE,
+        )
         self._session.mount("https://", adapter)
         self._session.mount("http://", adapter)
 
@@ -394,7 +413,12 @@ class LegacyClient:
             params["Root"] = root
 
         suffix = "repo/tree" if _is_dataset(repo_type) else "repo/files"
-        resp = self._request("GET", f"{segment}/{repo_id}/{suffix}", params=params)
+        resp = self._request_repo_tree(
+            f"{segment}/{repo_id}/{suffix}",
+            params,
+            repo_id=repo_id,
+            authorized=self._tree_reads_ok,
+        )
         data = self._json_data(resp)
         if isinstance(data, list):
             return data
@@ -567,6 +591,13 @@ class LegacyClient:
         Datasets can have millions of files, so this method pages through
         ``GET /api/v1/datasets/{repo_id}/repo/tree`` with
         ``PageNumber``/``PageSize`` params.
+
+        A page occasionally answers ``403 无权访问该数据集`` on a repository the
+        caller demonstrably can read. Once any page has succeeded the credential
+        is proven, so a later-page denial is a server-side hiccup rather than an
+        authorization result, and it is retried instead of discarding every page
+        collected so far -- a large listing spans hundreds of pages, which makes
+        hitting it near-certain.
         """
         all_files: list[dict] = []
         page_number = 1
@@ -579,10 +610,11 @@ class LegacyClient:
             }
             if root_path and root_path != "/":
                 params["Root"] = root_path
-            resp = self._request(
-                "GET",
+            resp = self._request_repo_tree(
                 f"datasets/{repo_id}/repo/tree",
-                params=params,
+                params,
+                repo_id=repo_id,
+                authorized=self._tree_reads_ok,
             )
             data = self._json_data(resp)
             if isinstance(data, list):
@@ -598,6 +630,54 @@ class LegacyClient:
             page_number += 1
         return all_files
 
+    def _request_repo_tree(
+        self,
+        path: str,
+        params: dict[str, Any],
+        *,
+        repo_id: str,
+        authorized: bool,
+    ) -> Any:
+        """Fetch one file-tree listing, retrying a spurious denial.
+
+        The server intermittently answers a tree request with ``403 无权访问该数据
+        集`` on a repository the caller has just read successfully -- observed on
+        both ``PageNumber``-paginated and ``Root``-scoped listings. Enumerating a
+        large repository takes hundreds of such requests, so at that scale a
+        single-request failure rate is effectively a guaranteed whole-listing
+        failure, and it discards every entry gathered so far.
+
+        ``authorized`` means some tree read already succeeded on this client, so
+        the credential is proven and a denial cannot be an authorization result.
+        Until then a 403 is taken at face value, keeping a real permission error
+        fast and honest.
+        """
+        last_error: PermissionDeniedError | None = None
+        for attempt in range(REPO_TREE_PAGE_MAX_ATTEMPTS):
+            try:
+                resp = self._request("GET", path, params=params)
+            except PermissionDeniedError as error:
+                if not authorized:
+                    raise
+                last_error = error
+                if attempt < REPO_TREE_PAGE_MAX_ATTEMPTS - 1:
+                    wait = min(2**attempt, REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS)
+                    logger.warning(
+                        "Repo %s: tree listing (%s) denied on an already-authorized repo, retrying in %ds ...",
+                        repo_id,
+                        params.get("Root") or params.get("PageNumber") or "/",
+                        wait,
+                    )
+                    time.sleep(wait)
+                continue
+            self._tree_reads_ok = True
+            return resp
+        raise NetworkError(
+            f"Repo {repo_id}: tree listing ({params.get('Root') or params.get('PageNumber') or '/'}) kept "
+            f"returning a denial after {REPO_TREE_PAGE_MAX_ATTEMPTS} attempts on an already-authorized "
+            f"repo: {last_error}"
+        ) from last_error
+
     # ------------------------------------------------------------------
     # Revisions
     # ------------------------------------------------------------------

{modelscope_hub-0.4.2 → modelscope_hub-0.4.4}/src/modelscope_hub/_openapi.py

@@ -30,9 +30,17 @@ from typing import Any, BinaryIO
 from urllib.parse import urljoin, urlsplit
 
 import requests
+from requests.adapters import HTTPAdapter
 
 from .config import HubConfig, get_default_config
-from .constants import API_CONNECT_TIMEOUT, API_MAX_RETRIES, API_TIMEOUT, OPENAPI_PREFIX, TokenScope
+from .constants import (
+    API_CONNECT_TIMEOUT,
+    API_CONNECTION_POOL_MAXSIZE,
+    API_MAX_RETRIES,
+    API_TIMEOUT,
+    OPENAPI_PREFIX,
+    TokenScope,
+)
 from .errors import (
     APIError,
     AuthenticationError,
@@ -200,6 +208,16 @@ class OpenAPIClient:
     ) -> None:
         self._config = config or get_default_config()
         self._session = session or requests.Session()
+        if session is None:
+            # A bare Session caps the pool at urllib3's default of 10, which is
+            # below the concurrency bulk transfers use; the excess connections
+            # are discarded and pay for a new TLS handshake on next use.
+            adapter = HTTPAdapter(
+                pool_connections=API_CONNECTION_POOL_MAXSIZE,
+                pool_maxsize=API_CONNECTION_POOL_MAXSIZE,
+            )
+            self._session.mount("https://", adapter)
+            self._session.mount("http://", adapter)
         self._timeout: float | tuple[float, float] = (
             float(timeout) if timeout is not None else (float(API_CONNECT_TIMEOUT), float(API_TIMEOUT))
         )
@@ -1084,6 +1102,62 @@ class OpenAPIClient:
                 params.append((key, str(value).lower() if isinstance(value, bool) else str(value)))
         return params
 
+    @staticmethod
+    def _decode_mcp_list_response(response: requests.Response) -> JSON:
+        """Decode MCP list replies, including legacy HTTP-200 error envelopes.
+
+        The service normally signals errors through HTTP status codes. Some
+        deployments instead return ``200`` with ``success: false`` and an
+        OpenAPI-style error code. Treating that dictionary as a list payload
+        silently turned invalid request bodies into empty server lists.
+        """
+        try:
+            payload = response.json()
+        except ValueError:
+            return {"raw_response": response.text}
+        if not isinstance(payload, dict):
+            return payload
+
+        success = payload.get("success") if "success" in payload else payload.get("Success")
+        if success is False:
+            code = payload.get("code") if payload.get("code") is not None else payload.get("Code")
+            message = next(
+                (
+                    value.strip()
+                    for key in ("message", "Message", "msg", "Msg", "detail", "Detail")
+                    if isinstance(value := payload.get(key), str) and value.strip()
+                ),
+                "MCP server list request was rejected.",
+            )
+            request_id = payload.get("request_id") or payload.get("requestId") or payload.get("RequestId")
+            code_text = str(code).strip()
+            error_cls: type[APIError] = APIError
+            status_code = 400
+            if code_text == "InputParameterError":
+                error_cls = InvalidParameter
+            elif code_text == "InvalidAuthentication":
+                error_cls = AuthenticationError
+                status_code = 401
+            elif code_text == "OperationNotAllowed":
+                error_cls = PermissionDeniedError
+                status_code = 403
+            elif code_text == "RateLimitExceed":
+                error_cls = RateLimitError
+                status_code = 429
+            raise error_cls(
+                message,
+                status_code=status_code,
+                request_id=request_id,
+                response_body=payload,
+                url=response.url,
+                method=response.request.method if response.request else "PUT",
+            )
+        if "data" in payload:
+            return payload["data"]
+        if "Data" in payload:
+            return payload["Data"]
+        return payload
+
     @staticmethod
     def _is_method_or_route_unsupported(exc: APIError) -> bool:
         return exc.status_code in (404, 405, 501)
@@ -1307,6 +1381,10 @@ class OpenAPIClient:
         filter : dict, optional
             Nested filter object. Supported keys: ``category``, ``is_hosted``.
         """
+        if isinstance(page_number, bool) or not isinstance(page_number, int) or page_number < 1:
+            raise InvalidParameter("page_number must be an integer >= 1.")
+        if isinstance(page_size, bool) or not isinstance(page_size, int) or page_size < 1:
+            raise InvalidParameter("page_size must be an integer >= 1.")
         if page_number * page_size > 100:
             # The service enforces this itself, answering 403 QuotaLimitExceed with
             # exactly this rule. Checking here spares the round trip and reports it
@@ -1326,14 +1404,16 @@ class OpenAPIClient:
         body = {k: v for k, v in body.items() if v is not None}
 
         def _get() -> JSON:
-            return self._request(
+            response = self._request(
                 "GET",
                 "/mcp/servers",
                 params=self._flatten_mcp_list_params(body),
                 require_token=False,
                 required_scope=TokenScope.READ,
                 anonymous_retry=True,
+                unwrap=False,
             )
+            return self._decode_mcp_list_response(response)
 
         if self._mcp_list_supports_get:
             # This deployment already answered GET and refused PUT, so leading
@@ -1341,14 +1421,16 @@ class OpenAPIClient:
             return _get()
 
         try:
-            return self._request(
+            response = self._request(
                 "PUT",
                 "/mcp/servers",
                 json_body=body,
                 require_token=False,
                 required_scope=TokenScope.READ,
                 anonymous_retry=True,
+                unwrap=False,
             )
+            return self._decode_mcp_list_response(response)
         except APIError as exc:
             if self._mcp_list_supports_get is False or not self._is_method_or_route_unsupported(exc):
                 raise