b10-transfer 0.3.6__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/PKG-INFO +1 -1
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/pyproject.toml +1 -1
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/__init__.py +1 -1
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/cleanup.py +6 -6
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/README.md +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/archive.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/cache.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/cache_cli.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/config.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/constants.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/core.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/environment.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/info.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/logging_utils.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/space_monitor.py +0 -0
- {b10_transfer-0.3.6 → b10_transfer-0.3.7}/src/b10_transfer/utils.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: b10-transfer
|
3
|
-
Version: 0.3.
|
3
|
+
Version: 0.3.7
|
4
4
|
Summary: Distributed PyTorch file transfer for Baseten - Environment-aware, lock-free file transfer management
|
5
5
|
License: MIT
|
6
6
|
Keywords: pytorch,file-transfer,cache,machine-learning,inference
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
4
4
|
|
5
5
|
[tool.poetry]
|
6
6
|
name = "b10-transfer"
|
7
|
-
version = "0.3.
|
7
|
+
version = "0.3.7"
|
8
8
|
description = "Distributed PyTorch file transfer for Baseten - Environment-aware, lock-free file transfer management"
|
9
9
|
authors = ["Shounak Ray <shounak.noreply@baseten.co>", "Fred Liu <fred.liu.noreply@baseten.co>"]
|
10
10
|
maintainers = ["Fred Liu <fred.liu.noreply@baseten.co>", "Shounak Ray <shounak.noreply@baseten.co>"]
|
@@ -13,8 +13,6 @@ from typing import List
|
|
13
13
|
from .config import config
|
14
14
|
from .constants import (
|
15
15
|
CACHE_INCOMPLETE_SUFFIX,
|
16
|
-
CLEANUP_LOCK_TIMEOUT_SECONDS,
|
17
|
-
CLEANUP_INCOMPLETE_TIMEOUT_SECONDS,
|
18
16
|
)
|
19
17
|
from .utils import safe_execute, safe_unlink
|
20
18
|
from .logging_utils import get_b10_logger
|
@@ -108,8 +106,8 @@ def cooperative_cleanup_b10fs() -> None:
|
|
108
106
|
removing files that are likely orphaned due to pod crashes or failures.
|
109
107
|
|
110
108
|
Removes:
|
111
|
-
- Lock files older than CLEANUP_LOCK_TIMEOUT_SECONDS (*.lock)
|
112
|
-
- Incomplete cache files older than CLEANUP_INCOMPLETE_TIMEOUT_SECONDS (*.incomplete*)
|
109
|
+
- Lock files older than config.CLEANUP_LOCK_TIMEOUT_SECONDS (*.lock)
|
110
|
+
- Incomplete cache files older than config.CLEANUP_INCOMPLETE_TIMEOUT_SECONDS (*.incomplete*)
|
113
111
|
|
114
112
|
Does NOT remove:
|
115
113
|
- Final cache files (*.latest.tar.gz) - these are the actual cached results
|
@@ -124,13 +122,15 @@ def cooperative_cleanup_b10fs() -> None:
|
|
124
122
|
return
|
125
123
|
|
126
124
|
# Find and clean stale lock files
|
127
|
-
stale_locks = _find_stale_files(
|
125
|
+
stale_locks = _find_stale_files(
|
126
|
+
b10fs_dir, "*.lock", config.CLEANUP_LOCK_TIMEOUT_SECONDS
|
127
|
+
)
|
128
128
|
cleaned_locks = _cleanup_files(stale_locks, "lock")
|
129
129
|
|
130
130
|
# Find and clean stale incomplete cache files
|
131
131
|
incomplete_pattern = f"*{CACHE_INCOMPLETE_SUFFIX}*"
|
132
132
|
stale_incomplete = _find_stale_files(
|
133
|
-
b10fs_dir, incomplete_pattern, CLEANUP_INCOMPLETE_TIMEOUT_SECONDS
|
133
|
+
b10fs_dir, incomplete_pattern, config.CLEANUP_INCOMPLETE_TIMEOUT_SECONDS
|
134
134
|
)
|
135
135
|
cleaned_incomplete = _cleanup_files(stale_incomplete, "incomplete cache")
|
136
136
|
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|