modelscope 1.34.0__tar.gz → 1.35.1__tar.gz

{modelscope-1.34.0/modelscope.egg-info → modelscope-1.35.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelscope
-Version: 1.34.0
+Version: 1.35.1
 Summary: ModelScope: bring the notion of Model-as-a-Service to life.
 Author: ModelScope team
 Author-email: contact@modelscope.cn
@@ -17,12 +17,14 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: filelock
+Requires-Dist: packaging
 Requires-Dist: requests>=2.25
 Requires-Dist: setuptools
 Requires-Dist: tqdm>=4.64.0
 Requires-Dist: urllib3>=1.26
 Provides-Extra: hub
 Requires-Dist: filelock; extra == "hub"
+Requires-Dist: packaging; extra == "hub"
 Requires-Dist: requests>=2.25; extra == "hub"
 Requires-Dist: setuptools; extra == "hub"
 Requires-Dist: tqdm>=4.64.0; extra == "hub"
@@ -30,7 +32,7 @@ Requires-Dist: urllib3>=1.26; extra == "hub"
 Provides-Extra: datasets
 Requires-Dist: addict; extra == "datasets"
 Requires-Dist: attrs; extra == "datasets"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "datasets"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "datasets"
 Requires-Dist: einops; extra == "datasets"
 Requires-Dist: oss2; extra == "datasets"
 Requires-Dist: Pillow; extra == "datasets"
@@ -43,7 +45,7 @@ Requires-Dist: urllib3>=1.26; extra == "datasets"
 Provides-Extra: framework
 Requires-Dist: addict; extra == "framework"
 Requires-Dist: attrs; extra == "framework"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "framework"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "framework"
 Requires-Dist: einops; extra == "framework"
 Requires-Dist: Pillow; extra == "framework"
 Requires-Dist: python-dateutil>=2.1; extra == "framework"
@@ -75,7 +77,7 @@ Requires-Dist: yapf==0.30.0; extra == "tests"
 Provides-Extra: cv
 Requires-Dist: addict; extra == "cv"
 Requires-Dist: attrs; extra == "cv"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "cv"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "cv"
 Requires-Dist: einops; extra == "cv"
 Requires-Dist: Pillow; extra == "cv"
 Requires-Dist: python-dateutil>=2.1; extra == "cv"
@@ -159,7 +161,7 @@ Requires-Dist: yacs; extra == "cv"
 Provides-Extra: nlp
 Requires-Dist: addict; extra == "nlp"
 Requires-Dist: attrs; extra == "nlp"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "nlp"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "nlp"
 Requires-Dist: einops; extra == "nlp"
 Requires-Dist: Pillow; extra == "nlp"
 Requires-Dist: python-dateutil>=2.1; extra == "nlp"
@@ -198,7 +200,7 @@ Requires-Dist: zhconv; extra == "nlp"
 Provides-Extra: multi-modal
 Requires-Dist: addict; extra == "multi-modal"
 Requires-Dist: attrs; extra == "multi-modal"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "multi-modal"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "multi-modal"
 Requires-Dist: einops; extra == "multi-modal"
 Requires-Dist: Pillow; extra == "multi-modal"
 Requires-Dist: python-dateutil>=2.1; extra == "multi-modal"
@@ -236,7 +238,7 @@ Requires-Dist: zhconv; extra == "multi-modal"
 Provides-Extra: science
 Requires-Dist: addict; extra == "science"
 Requires-Dist: attrs; extra == "science"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "science"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "science"
 Requires-Dist: einops; extra == "science"
 Requires-Dist: Pillow; extra == "science"
 Requires-Dist: python-dateutil>=2.1; extra == "science"
@@ -258,7 +260,7 @@ Requires-Dist: tokenizers; extra == "science"
 Provides-Extra: audio-asr
 Requires-Dist: addict; extra == "audio-asr"
 Requires-Dist: attrs; extra == "audio-asr"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio-asr"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio-asr"
 Requires-Dist: einops; extra == "audio-asr"
 Requires-Dist: Pillow; extra == "audio-asr"
 Requires-Dist: python-dateutil>=2.1; extra == "audio-asr"
@@ -273,7 +275,7 @@ Requires-Dist: funasr>=1.0.0; extra == "audio-asr"
 Provides-Extra: audio-codec
 Requires-Dist: addict; extra == "audio-codec"
 Requires-Dist: attrs; extra == "audio-codec"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio-codec"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio-codec"
 Requires-Dist: einops; extra == "audio-codec"
 Requires-Dist: Pillow; extra == "audio-codec"
 Requires-Dist: python-dateutil>=2.1; extra == "audio-codec"
@@ -288,7 +290,7 @@ Requires-Dist: ms-funcodec>=0.2.0; extra == "audio-codec"
 Provides-Extra: audio-tts
 Requires-Dist: addict; extra == "audio-tts"
 Requires-Dist: attrs; extra == "audio-tts"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio-tts"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio-tts"
 Requires-Dist: einops; extra == "audio-tts"
 Requires-Dist: Pillow; extra == "audio-tts"
 Requires-Dist: python-dateutil>=2.1; extra == "audio-tts"
@@ -328,7 +330,7 @@ Requires-Dist: wcwidth>=0.2.5; extra == "audio-tts"
 Provides-Extra: audio-kws
 Requires-Dist: addict; extra == "audio-kws"
 Requires-Dist: attrs; extra == "audio-kws"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio-kws"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio-kws"
 Requires-Dist: einops; extra == "audio-kws"
 Requires-Dist: Pillow; extra == "audio-kws"
 Requires-Dist: python-dateutil>=2.1; extra == "audio-kws"
@@ -348,7 +350,7 @@ Requires-Dist: tensorboardX; extra == "audio-kws"
 Provides-Extra: audio-signal
 Requires-Dist: addict; extra == "audio-signal"
 Requires-Dist: attrs; extra == "audio-signal"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio-signal"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio-signal"
 Requires-Dist: einops; extra == "audio-signal"
 Requires-Dist: Pillow; extra == "audio-signal"
 Requires-Dist: python-dateutil>=2.1; extra == "audio-signal"
@@ -374,7 +376,7 @@ Requires-Dist: umap-learn; extra == "audio-signal"
 Provides-Extra: audio
 Requires-Dist: addict; extra == "audio"
 Requires-Dist: attrs; extra == "audio"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "audio"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "audio"
 Requires-Dist: einops; extra == "audio"
 Requires-Dist: Pillow; extra == "audio"
 Requires-Dist: python-dateutil>=2.1; extra == "audio"
@@ -433,13 +435,14 @@ Requires-Dist: tqdm; extra == "audio"
 Requires-Dist: umap-learn; extra == "audio"
 Provides-Extra: all
 Requires-Dist: filelock; extra == "all"
+Requires-Dist: packaging; extra == "all"
 Requires-Dist: requests>=2.25; extra == "all"
 Requires-Dist: setuptools; extra == "all"
 Requires-Dist: tqdm>=4.64.0; extra == "all"
 Requires-Dist: urllib3>=1.26; extra == "all"
 Requires-Dist: addict; extra == "all"
 Requires-Dist: attrs; extra == "all"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "all"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "all"
 Requires-Dist: einops; extra == "all"
 Requires-Dist: oss2; extra == "all"
 Requires-Dist: Pillow; extra == "all"
@@ -451,7 +454,7 @@ Requires-Dist: sortedcontainers>=1.5.9; extra == "all"
 Requires-Dist: urllib3>=1.26; extra == "all"
 Requires-Dist: addict; extra == "all"
 Requires-Dist: attrs; extra == "all"
-Requires-Dist: datasets<=3.6.0,>=3.0.0; extra == "all"
+Requires-Dist: datasets<=4.6.1,>=4.0.0; extra == "all"
 Requires-Dist: einops; extra == "all"
 Requires-Dist: Pillow; extra == "all"
 Requires-Dist: python-dateutil>=2.1; extra == "all"

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/hub/api.py

@@ -11,7 +11,6 @@ import platform
 import re
 import shutil
 import tempfile
-import time
 import uuid
 import warnings
 from collections import defaultdict
@@ -1403,6 +1402,8 @@ class HubApi:
         raise_for_http_status(r)
         raise_on_error(r.json())
 
+    _dataset_id_type_cache: dict = {}
+
     def get_dataset_id_and_type(self,
                                 dataset_name: str,
                                 namespace: str,
@@ -1411,6 +1412,10 @@ class HubApi:
         """ Get the dataset id and type. """
         if not endpoint:
             endpoint = self.endpoint
+        cache_key = (namespace, dataset_name, endpoint)
+        cached = HubApi._dataset_id_type_cache.get(cache_key)
+        if cached is not None:
+            return cached
         datahub_url = f'{endpoint}/api/v1/datasets/{namespace}/{dataset_name}'
         cookies = self.get_cookies(access_token=token)
         r = self.session.get(datahub_url, cookies=cookies)
@@ -1418,6 +1423,7 @@ class HubApi:
         datahub_raise_on_error(datahub_url, resp, r)
         dataset_id = resp['Data']['Id']
         dataset_type = resp['Data']['Type']
+        HubApi._dataset_id_type_cache[cache_key] = (dataset_id, dataset_type)
         return dataset_id, dataset_type
 
     def list_repo_tree(self,
@@ -1526,7 +1532,8 @@ class HubApi:
                           page_number: int = 1,
                           page_size: int = 100,
                           endpoint: Optional[str] = None,
-                          token: Optional[str] = None):
+                          token: Optional[str] = None,
+                          dataset_hub_id: Optional[str] = None):
         """
         Get the dataset files.
 
@@ -1539,19 +1546,23 @@ class HubApi:
             page_size (int): The number of items per page. Defaults to 100.
             endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.
             token (Optional[str]): The access token.
+            dataset_hub_id (Optional[str]): Pre-fetched dataset hub id. When provided,
+                skips the internal ``get_dataset_id_and_type`` lookup. Useful in pagination
+                loops to avoid redundant API calls per page.
 
         Returns:
             List: The response containing the dataset repository tree information.
                 e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...]
         """
 
-        if is_relative_path(repo_id) and repo_id.count('/') == 1:
-            _owner, _dataset_name = repo_id.split('/')
-        else:
-            raise ValueError(f'Invalid repo_id: {repo_id} !')
+        if dataset_hub_id is None:
+            if is_relative_path(repo_id) and repo_id.count('/') == 1:
+                _owner, _dataset_name = repo_id.split('/')
+            else:
+                raise ValueError(f'Invalid repo_id: {repo_id} !')
 
-        dataset_hub_id, dataset_type = self.get_dataset_id_and_type(
-            dataset_name=_dataset_name, namespace=_owner, endpoint=endpoint, token=token)
+            dataset_hub_id, _ = self.get_dataset_id_and_type(
+                dataset_name=_dataset_name, namespace=_owner, endpoint=endpoint, token=token)
 
         if not endpoint:
             endpoint = self.endpoint
@@ -1569,7 +1580,10 @@ class HubApi:
         resp = r.json()
         datahub_raise_on_error(datahub_url, resp, r)
 
-        return resp['Data']['Files']
+        data = resp.get('Data')
+        if data is None:
+            return []
+        return data.get('Files') or []
 
     def get_dataset(
         self,
@@ -2100,11 +2114,9 @@ class HubApi:
             repo_type: Optional[str] = REPO_TYPE_MODEL,
             revision: Optional[str] = DEFAULT_REPOSITORY_REVISION,
             endpoint: Optional[str] = None,
-            max_retries: int = 3,
-            timeout: int = 180,
     ) -> CommitInfo:
         """
-        Create a commit on the ModelScope Hub with retry mechanism.
+        Create a commit on the ModelScope Hub.
 
         Args:
             repo_id (str): The repo id in the format of `owner_name/repo_name`.
@@ -2117,14 +2129,14 @@ class HubApi:
             revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
             endpoint (Optional[str]): The endpoint to use.
                 In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
-            max_retries (int): Number of max retry attempts (default: 3).
             timeout (int): Timeout for each request in seconds (default: 180).
 
         Returns:
             CommitInfo: The commit info.
 
         Raises:
-            requests.exceptions.RequestException: If all retry attempts fail.
+            ValueError: If the request fails with a 4xx client error.
+            requests.exceptions.RequestException: If a network-level error occurs.
         """
         if not repo_id:
             raise ValueError('Repo id cannot be empty!')
@@ -2147,66 +2159,29 @@ class HubApi:
             commit_message=commit_message,
         )
 
-        # POST with retry mechanism
-        last_exception = None
-        for attempt in range(max_retries):
-            try:
-                if attempt > 0:
-                    logger.info(f'Attempt {attempt + 1} to create commit for {repo_id}...')
-                response = requests.post(
-                    url,
-                    headers=self.builder_headers(self.headers),
-                    data=json.dumps(payload),
-                    cookies=cookies,
-                    timeout=timeout,
-                )
-
-                if response.status_code != 200:
-                    try:
-                        error_detail = response.json()
-                    except json.JSONDecodeError:
-                        error_detail = response.text
-
-                    error_msg = (
-                        f'HTTP {response.status_code} error from {url}: '
-                        f'{error_detail}'
-                    )
-
-                    # If server error (5xx), we can retry, otherwise (4xx) raise immediately
-                    if 500 <= response.status_code < 600:
-                        logger.warning(
-                            f'Server error on attempt {attempt + 1}: {error_msg}'
-                        )
-                    else:
-                        raise ValueError(f'Client request failed: {error_msg}')
-                else:
-                    resp = response.json()
-
-                    oid = resp.get('Data', {}).get('oid', '')
-                    logger.info(f'Commit succeeded: {url}')
-                    return CommitInfo(
-                        commit_url=url,
-                        commit_message=commit_message,
-                        commit_description=commit_description,
-                        oid=oid,
-                    )
-
-            except requests.exceptions.RequestException as e:
-                last_exception = e
-                logger.warning(f'Request failed on attempt {attempt + 1}: {str(e)}')
-
-            except Exception as e:
-                last_exception = e
-                logger.error(f'Unexpected error on attempt {attempt + 1}: {str(e)}')
-                if attempt == max_retries - 1:
-                    raise
+        response = self.session.post(
+            url,
+            headers=self.builder_headers(self.headers),
+            data=json.dumps(payload),
+            cookies=cookies,
+        )
 
-            if attempt < max_retries - 1:
-                time.sleep(1)
+        if response.status_code != 200:
+            try:
+                error_detail = response.json()
+            except json.JSONDecodeError:
+                error_detail = response.text
+            error_msg = f'HTTP {response.status_code} error from {url}: {error_detail}'
+            raise ValueError(error_msg)
 
-        # All retries exhausted
-        raise requests.exceptions.RequestException(
-            f'Failed to create commit after {max_retries} attempts. Last error: {last_exception}'
+        resp = response.json()
+        oid = resp.get('Data', {}).get('oid', '')
+        logger.info(f'Commit succeeded: {url}')
+        return CommitInfo(
+            commit_url=url,
+            commit_message=commit_message,
+            commit_description=commit_description,
+            oid=oid,
         )
 
     def upload_file(
@@ -2593,21 +2568,21 @@ class HubApi:
 
             if isinstance(data, (str, Path)):
                 with open(data, 'rb') as f:
-                    response = requests.put(
+                    response = self.session.put(
                         upload_object['url'],
                         headers=headers,
                         data=read_in_chunks(f, pbar)
                     )
 
             elif isinstance(data, bytes):
-                response = requests.put(
+                response = self.session.put(
                     upload_object['url'],
                     headers=headers,
                     data=read_in_chunks(io.BytesIO(data), pbar)
                 )
 
             elif isinstance(data, io.BufferedIOBase):
-                response = requests.put(
+                response = self.session.put(
                     upload_object['url'],
                     headers=headers,
                     data=read_in_chunks(data, pbar)
@@ -2664,7 +2639,7 @@ class HubApi:
         }
 
         cookies = self.get_cookies(access_token=token, cookies_required=True)
-        response = requests.post(
+        response = self.session.post(
             url,
             headers=self.builder_headers(self.headers),
             data=json.dumps(payload),
@@ -2907,6 +2882,9 @@ class HubApi:
             file_paths = [f['Path'] for f in files]
         elif repo_type == REPO_TYPE_DATASET:
             file_paths = []
+            _owner, _dataset_name = repo_id.split('/')
+            _hub_id, _ = self.get_dataset_id_and_type(
+                dataset_name=_dataset_name, namespace=_owner, endpoint=endpoint, token=token)
             page_number = 1
             page_size = 100
             while True:
@@ -2919,6 +2897,7 @@ class HubApi:
                         page_size=page_size,
                         endpoint=endpoint,
                         token=token,
+                        dataset_hub_id=_hub_id,
                     )
                 except Exception as e:
                     logger.error(f'Get dataset: {repo_id} file list failed, message: {str(e)}')

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/hub/file_download.py

@@ -253,6 +253,11 @@ def _repo_file_download(
         group_or_owner, name = model_id_to_group_owner_name(repo_id)
         if not revision:
             revision = DEFAULT_DATASET_REVISION
+        _hub_id, _ = _api.get_dataset_id_and_type(
+            dataset_name=name,
+            namespace=group_or_owner,
+            endpoint=endpoint,
+            token=token)
         page_number = 1
         page_size = 100
         while True:
@@ -265,7 +270,8 @@ def _repo_file_download(
                     page_number=page_number,
                     page_size=page_size,
                     endpoint=endpoint,
-                    token=token)
+                    token=token,
+                    dataset_hub_id=_hub_id)
             except Exception as e:
                 logger.error(
                     f'Get dataset: {repo_id} file list failed, error: {e}')

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/hub/snapshot_download.py

@@ -428,6 +428,10 @@ def _snapshot_download(
 
 
 def fetch_repo_files(_api, repo_id, revision, endpoint):
+    _owner, _dataset_name = repo_id.split('/')
+    _hub_id, _ = _api.get_dataset_id_and_type(
+        dataset_name=_dataset_name, namespace=_owner, endpoint=endpoint)
+
     page_number = 1
     page_size = 150
     repo_files = []
@@ -441,7 +445,8 @@ def fetch_repo_files(_api, repo_id, revision, endpoint):
                 recursive=True,
                 page_number=page_number,
                 page_size=page_size,
-                endpoint=endpoint)
+                endpoint=endpoint,
+                dataset_hub_id=_hub_id)
         except Exception as e:
             logger.error(f'Error fetching dataset files: {e}')
             break

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/hub/utils/utils.py

@@ -1,11 +1,11 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import contextlib
+import errno
 import hashlib
 import os
 import sys
 import time
-import zoneinfo
 from datetime import datetime
 from pathlib import Path
 from typing import Generator, List, Optional, Union
@@ -18,7 +18,6 @@ from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DOMAIN,
                                       MODEL_ID_SEPARATOR, MODELSCOPE_DOMAIN,
                                       MODELSCOPE_SDK_DEBUG,
                                       MODELSCOPE_URL_SCHEME)
-from modelscope.hub.errors import FileIntegrityError
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -288,6 +287,16 @@ def weak_file_lock(lock_file: Union[str, Path],
                     lock_file)
                 lock = SoftFileLock(lock_file, timeout=default_interval)
                 continue
+            raise
+        except OSError as e:
+            # Handle NFS stale file handle and similar issues
+            if e.errno in (errno.ESTALE, errno.ENOENT, errno.EREMOTEIO):
+                logger.warning(
+                    'Encountered OSError (errno=%d) on %s, Falling back to SoftFileLock.',
+                    e.errno, lock_file)
+                lock = SoftFileLock(lock_file, timeout=default_interval)
+                continue
+            raise
         else:
             break
 
@@ -312,6 +321,7 @@ def convert_timestamp(time_stamp: Union[int, str, datetime],
     Returns:
         Timezone-aware datetime object or None if input is None
     """
+    import zoneinfo
     if not time_stamp:
         return None
 
@@ -381,6 +391,30 @@ def convert_timestamp(time_stamp: Union[int, str, datetime],
         )
 
 
+# Fallback MIME types for common media formats that may not be registered
+# in the system's MIME database on certain platforms.
+_FALLBACK_MIME_TYPES = {
+    '.webp': 'image/webp',
+    '.avif': 'image/avif',
+    '.heic': 'image/heic',
+    '.heif': 'image/heif',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.png': 'image/png',
+    '.gif': 'image/gif',
+    '.bmp': 'image/bmp',
+    '.svg': 'image/svg+xml',
+    '.tiff': 'image/tiff',
+    '.tif': 'image/tiff',
+    '.ico': 'image/x-icon',
+    '.mp4': 'video/mp4',
+    '.webm': 'video/webm',
+    '.avi': 'video/x-msvideo',
+    '.mov': 'video/quicktime',
+    '.mkv': 'video/x-matroska',
+}
+
+
 def encode_media_to_base64(media_file_path: str) -> str:
     """
     Encode image or video file to base64 string.
@@ -407,8 +441,13 @@ def encode_media_to_base64(media_file_path: str) -> str:
     if not os.path.isfile(media_file_path):
         raise ValueError(f'Path is not a file: {media_file_path}')
 
-    # Get MIME type
+    # Get MIME type, with fallback for formats that may not be registered
+    # in the system's MIME database on some platforms (e.g. Linux servers,
+    # Docker containers).
     mime_type, _ = mimetypes.guess_type(media_file_path)
+    if not mime_type:
+        mime_type = _FALLBACK_MIME_TYPES.get(
+            os.path.splitext(media_file_path)[1].lower())
     if not mime_type:
         raise ValueError(f'File is not a valid format: {media_file_path}')
 

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/models/multi_modal/videocomposer/ops/utils.py

@@ -21,7 +21,6 @@ from multiprocessing.pool import ThreadPool as Pool
 import imageio
 import json
 import numpy as np
-import oss2 as oss
 import requests
 import skvideo.io
 import torch
@@ -169,6 +168,8 @@ def DOWNLOAD_TO_CACHE(oss_key,
 
 
 def parse_oss_url(path):
+    import oss2 as oss
+
     if path.startswith('oss://'):
         path = path[len('oss://'):]
 

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/models/multi_modal/videocomposer/utils/utils.py

@@ -22,7 +22,6 @@ from multiprocessing.pool import ThreadPool as Pool
 import imageio
 import json
 import numpy as np
-import oss2 as oss
 import requests
 import skvideo.io
 import torch
@@ -81,6 +80,8 @@ def setup_seed(seed):
 
 
 def parse_oss_url(path):
+    import oss2 as oss
+
     if path.startswith('oss://'):
         path = path[len('oss://'):]
 

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/models/nlp/glm_130b/kernels/__init__.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2022 Zhipu.AI
 import ctypes
+from importlib.resources import files
 from typing import List
 
-import pkg_resources
 import torch
 from cpm_kernels.kernels.base import (KernelFunction, LazyKernelCModule,
                                       round_up)
@@ -14,12 +14,12 @@ class Kernel:
 
     def __init__(self, filename: str, function_names: List[str]):
         filename = filename + '.fatbin'
-        if not pkg_resources.resource_exists(RESOURCE_PACKAGE_NAME, filename):
+        resource_ref = files(RESOURCE_PACKAGE_NAME).joinpath(filename)
+        if not resource_ref.is_file():
             raise RuntimeError('File `%s` not found in `%s`' %
                                (filename, RESOURCE_PACKAGE_NAME))
         self.filename = filename
-        self.code = pkg_resources.resource_string(RESOURCE_PACKAGE_NAME,
-                                                  filename)
+        self.code = resource_ref.read_bytes()
         self._function_names = function_names
         self._cmodule = LazyKernelCModule(self.code)
 

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/msdatasets/data_loader/data_loader.py

@@ -148,7 +148,6 @@ class OssDownloader(BaseDownloader):
                 data_files=data_files,
                 cache_dir=cache_dir,
                 download_mode=download_mode.value,
-                trust_remote_code=trust_remote_code,
                 **input_kwargs)
         else:
             self.dataset = self.data_files_manager.fetch_data_files(

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/msdatasets/data_loader/data_loader_manager.py

@@ -87,7 +87,6 @@ class LocalDataLoaderManager(DataLoaderManager):
                 cache_dir=cache_root_dir,
                 download_mode=download_mode.value,
                 streaming=use_streaming,
-                trust_remote_code=trust_remote_code,
                 **input_config_kwargs)
         raise f'Expected local data loader type: {LocalDataLoaderType.HF_DATA_LOADER.value}.'
 
@@ -130,7 +129,6 @@ class RemoteDataLoaderManager(DataLoaderManager):
                 data_files=data_files,
                 download_mode=download_mode_val,
                 streaming=use_streaming,
-                trust_remote_code=trust_remote_code,
                 token=token,
                 **input_config_kwargs)
             # download statistics

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/msdatasets/download/dataset_builder.py

@@ -13,8 +13,8 @@ from datasets.filesystems import is_remote_filesystem
 from datasets.info import DatasetInfo
 from datasets.naming import camelcase_to_snakecase
 from datasets.packaged_modules import csv
-from datasets.utils.filelock import FileLock
 from datasets.utils.py_utils import map_nested
+from filelock import FileLock
 
 from modelscope.hub.api import HubApi
 from modelscope.msdatasets.context.dataset_context_config import \

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/msdatasets/meta/data_meta_manager.py

@@ -5,7 +5,7 @@ import shutil
 from collections import defaultdict
 
 import json
-from datasets.utils.filelock import FileLock
+from filelock import FileLock
 
 from modelscope.hub.api import HubApi
 from modelscope.msdatasets.context.dataset_context_config import \

{modelscope-1.34.0 → modelscope-1.35.1}/modelscope/msdatasets/ms_dataset.py

@@ -283,10 +283,15 @@ class MsDataset:
             return load_dataset(
                 dataset_name,
                 name=subset_name,
+                data_dir=data_dir,
+                data_files=data_files,
                 split=split,
-                streaming=use_streaming,
+                cache_dir=cache_dir,
+                features=features,
                 download_mode=download_mode.value,
-                trust_remote_code=trust_remote_code,
+                revision=version,
+                token=token,
+                streaming=use_streaming,
                 **config_kwargs)
 
         # Load from the modelscope hub