modelscope 1.35.3__tar.gz → 1.35.4__tar.gz

{modelscope-1.35.3/modelscope.egg-info → modelscope-1.35.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modelscope
-Version: 1.35.3
+Version: 1.35.4
 Summary: ModelScope: bring the notion of Model-as-a-Service to life.
 Author: ModelScope team
 Author-email: contact@modelscope.cn

{modelscope-1.35.3 → modelscope-1.35.4}/modelscope/cli/download.py

@@ -145,7 +145,8 @@ class DownloadCMD(CLICommand):
                     cache_dir=self.args.cache_dir,
                     local_dir=self.args.local_dir,
                     revision=self.args.revision,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             elif len(
                     self.args.files) > 1:  # download specified multiple files.
                 snapshot_download(
@@ -155,7 +156,8 @@ class DownloadCMD(CLICommand):
                     local_dir=self.args.local_dir,
                     allow_file_pattern=self.args.files,
                     max_workers=self.args.max_workers,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             else:  # download repo
                 snapshot_download(
                     self.args.model,
@@ -165,7 +167,8 @@ class DownloadCMD(CLICommand):
                     allow_file_pattern=convert_patterns(self.args.include),
                     ignore_file_pattern=convert_patterns(self.args.exclude),
                     max_workers=self.args.max_workers,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             print(f'\nSuccessfully Downloaded from model {self.args.model}.\n')
         elif self.args.dataset:
             dataset_revision: str = self.args.revision if self.args.revision else DEFAULT_DATASET_REVISION
@@ -176,7 +179,8 @@ class DownloadCMD(CLICommand):
                     cache_dir=self.args.cache_dir,
                     local_dir=self.args.local_dir,
                     revision=dataset_revision,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             elif len(
                     self.args.files) > 1:  # download specified multiple files.
                 dataset_snapshot_download(
@@ -186,7 +190,8 @@ class DownloadCMD(CLICommand):
                     local_dir=self.args.local_dir,
                     allow_file_pattern=self.args.files,
                     max_workers=self.args.max_workers,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             else:  # download repo
                 dataset_snapshot_download(
                     self.args.dataset,
@@ -196,7 +201,8 @@ class DownloadCMD(CLICommand):
                     allow_file_pattern=convert_patterns(self.args.include),
                     ignore_file_pattern=convert_patterns(self.args.exclude),
                     max_workers=self.args.max_workers,
-                    cookies=cookies)
+                    cookies=cookies,
+                    token=self.args.token)
             print(
                 f'\nSuccessfully Downloaded from dataset {self.args.dataset}.\n'
             )

{modelscope-1.35.3 → modelscope-1.35.4}/modelscope/hub/file_download.py

@@ -446,6 +446,7 @@ def download_part_with_retry(params):
                     headers=get_headers,
                     cookies=cookies,
                     timeout=API_FILE_DOWNLOAD_TIMEOUT)
+                r.raise_for_status()
                 for chunk in r.iter_content(
                         chunk_size=API_FILE_DOWNLOAD_CHUNK_SIZE):
                     if chunk:  # filter out keep-alive new chunks
@@ -738,15 +739,21 @@ def download_file(
     temp_file = os.path.join(temporary_cache_dir, file_meta['Path'])
     if FILE_HASH in file_meta:
         expected_hash = file_meta[FILE_HASH]
-        # if a real-time hash has been computed
         if file_digest is not None:
-            # if real-time hash mismatched, try to compute it again
             if file_digest != expected_hash:
-                print(
-                    'Mismatched real-time digest found, falling back to lump-sum hash computation'
-                )
-                file_integrity_validation(temp_file, expected_hash)
+                logger.warning(
+                    'Mismatched real-time digest for %s, falling back to full hash check',
+                    file_meta['Path'])
+                if not file_integrity_validation(temp_file, expected_hash):
+                    raise FileDownloadError(
+                        'File %s hash validation failed after download, '
+                        'the file may be corrupted. Please retry.'
+                        % file_meta['Path'])
         else:
-            file_integrity_validation(temp_file, expected_hash)
+            if not file_integrity_validation(temp_file, expected_hash):
+                raise FileDownloadError(
+                    'File %s hash validation failed after download, '
+                    'the file may be corrupted. Please retry.'
+                    % file_meta['Path'])
     # put file into to cache
     return cache.put_file(file_meta, temp_file)

{modelscope-1.35.3 → modelscope-1.35.4}/modelscope/hub/snapshot_download.py

@@ -308,7 +308,7 @@ def _snapshot_download(
 
         _api = HubApi(token=token)
         endpoint = _api.get_endpoint_for_read(
-            repo_id=repo_id, repo_type=repo_type)
+            repo_id=repo_id, repo_type=repo_type, token=token)
         if cookies is None:
             cookies = _api.get_cookies()
         if repo_type == REPO_TYPE_MODEL:
@@ -393,8 +393,8 @@ def _snapshot_download(
             revision_detail = revision or DEFAULT_DATASET_REVISION
 
             logger.info('Fetching dataset repo file list...')
-            repo_files = fetch_repo_files(_api, repo_id, revision_detail,
-                                          endpoint)
+            repo_files = fetch_repo_files(
+                _api, repo_id, revision_detail, endpoint, token=token)
 
             if repo_files is None:
                 logger.error(
@@ -427,10 +427,13 @@ def _snapshot_download(
         return cache_root_path
 
 
-def fetch_repo_files(_api, repo_id, revision, endpoint):
+def fetch_repo_files(_api, repo_id, revision, endpoint, token=None):
     _owner, _dataset_name = repo_id.split('/')
     _hub_id, _ = _api.get_dataset_id_and_type(
-        dataset_name=_dataset_name, namespace=_owner, endpoint=endpoint)
+        dataset_name=_dataset_name,
+        namespace=_owner,
+        endpoint=endpoint,
+        token=token)
 
     page_number = 1
     page_size = 150
@@ -446,6 +449,7 @@ def fetch_repo_files(_api, repo_id, revision, endpoint):
                 page_number=page_number,
                 page_size=page_size,
                 endpoint=endpoint,
+                token=token,
                 dataset_hub_id=_hub_id)
         except Exception as e:
             logger.error(f'Error fetching dataset files: {e}')

{modelscope-1.35.3 → modelscope-1.35.4}/modelscope/hub/utils/caching.py

@@ -54,8 +54,47 @@ class FileSystemCache(object):
         cache_keys_file_path = os.path.join(self.cache_root_location,
                                             FileSystemCache.KEY_FILE_NAME)
         if os.path.exists(cache_keys_file_path):
-            with open(cache_keys_file_path, 'rb') as f:
-                self.cached_files = pickle.load(f)
+            try:
+                with open(cache_keys_file_path, 'rb') as f:
+                    data = pickle.load(f)
+                if isinstance(data, list):
+                    self.cached_files = data
+                else:
+                    logger.warning(
+                        'Cache index %s has unexpected type %s, resetting.',
+                        cache_keys_file_path,
+                        type(data).__name__)
+            except Exception as e:
+                logger.warning(
+                    'Failed to load cache index %s: %s. '
+                    'Resetting — already-downloaded files will be re-validated on next run.',
+                    cache_keys_file_path, e)
+                try:
+                    os.replace(cache_keys_file_path,
+                               cache_keys_file_path + '.corrupted')
+                except OSError:
+                    pass
+
+    def _save_cached_files_unlocked(self):
+        """Write .msc atomically. Caller must hold ``_cache_lock``."""
+        cache_keys_file_path = os.path.join(self.cache_root_location,
+                                            FileSystemCache.KEY_FILE_NAME)
+        fd, temp_filename = tempfile.mkstemp(
+            suffix='.tmp', dir=self.cache_root_location)
+        try:
+            with os.fdopen(fd, 'wb') as f:
+                pickle.dump(list(self.cached_files), f)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(temp_filename, cache_keys_file_path)
+        except Exception:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+            if os.path.exists(temp_filename):
+                os.unlink(temp_filename)
+            raise
 
     def save_cached_files(self):
         """
@@ -65,23 +104,7 @@ class FileSystemCache(object):
             [{'Path': 'configuration.json', 'Revision': 'f01dxxx'}, {'Path': 'model.bin', 'Revision': '1159xxx'}, ...]
         """
         with self._cache_lock:
-            cache_keys_file_path = os.path.join(self.cache_root_location,
-                                                FileSystemCache.KEY_FILE_NAME)
-            fd, temp_filename = tempfile.mkstemp(
-                suffix='.tmp', dir=self.cache_root_location)
-
-            try:
-                with os.fdopen(fd, 'wb') as f:
-                    pickle.dump(self.cached_files, f)
-                move(temp_filename, cache_keys_file_path)
-            except Exception:
-                try:
-                    os.close(fd)
-                except OSError:
-                    pass
-                if os.path.exists(temp_filename):
-                    os.unlink(temp_filename)
-                raise
+            self._save_cached_files_unlocked()
 
     def get_file(self, key):
         """Check the key is in the cache, if exist, return the file, otherwise return None.
@@ -170,10 +193,22 @@ class ModelFileSystemCache(FileSystemCache):
         meta_file_path = os.path.join(self.cache_root_location,
                                       FileSystemCache.MODEL_META_FILE_NAME)
         if os.path.exists(meta_file_path):
-            with open(meta_file_path, 'rb') as f:
-                self.model_meta = pickle.load(f)
-        else:
-            self.model_meta = {FileSystemCache.MODEL_META_MODEL_ID: 'unknown'}
+            try:
+                with open(meta_file_path, 'rb') as f:
+                    data = pickle.load(f)
+                if isinstance(data, dict):
+                    self.model_meta = data
+                    return
+                logger.warning('Model meta %s has unexpected type, resetting.',
+                               meta_file_path)
+            except Exception as e:
+                logger.warning('Failed to load model meta %s: %s. Resetting.',
+                               meta_file_path, e)
+                try:
+                    os.replace(meta_file_path, meta_file_path + '.corrupted')
+                except OSError:
+                    pass
+        self.model_meta = {FileSystemCache.MODEL_META_MODEL_ID: 'unknown'}
 
     def load_model_version(self):
         model_version_file_path = os.path.join(
@@ -341,16 +376,15 @@ class ModelFileSystemCache(FileSystemCache):
         Returns:
             str: The location of the cached file.
         """
-        self.remove_if_exists(model_file_info)  # backup old revision
+        self.remove_if_exists(model_file_info)
         cache_key = self.__get_cache_key(model_file_info)
-        cache_full_path = os.path.join(
-            self.cache_root_location,
-            cache_key['Path'])  # Branch and Tag do not have same name.
+        cache_full_path = os.path.join(self.cache_root_location,
+                                       cache_key['Path'])
         cache_file_dir = os.path.dirname(cache_full_path)
         if not os.path.exists(cache_file_dir):
             os.makedirs(cache_file_dir, exist_ok=True)
-        # We can't make operation transaction
         move(model_file_location, cache_full_path)
-        self.cached_files.append(cache_key)
-        self.save_cached_files()
+        with self._cache_lock:
+            self.cached_files.append(cache_key)
+            self._save_cached_files_unlocked()
         return cache_full_path