amzn-sagemaker-checkpointing 1.0.10__tar.gz → 1.0.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amzn_sagemaker_checkpointing-1.0.13/DEVELOPING.md +22 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/PKG-INFO +3 -3
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/README.md +2 -2
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/pyproject.toml +7 -2
- amzn_sagemaker_checkpointing-1.0.13/setup-hatch.sh +19 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/checkpointing/filesystem/filesystem.py +208 -246
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/config/in_memory_client.py +1 -3
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/inmemory/inmemory_client.py +24 -28
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/inmemory/models.py +1 -3
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/s3/s3_client.py +2 -7
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/s3/s3_client_manager.py +5 -15
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/scripts/test_inmemory_client.py +2 -5
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_configuration.py +117 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_helpers.py +100 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_reader.py +483 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_step_extraction.py +88 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_writer.py +347 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_delete_checkpoint.py +145 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_delete_namespace.py +68 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_get_checkpoint.py +212 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_get_latest_checkpoints.py +101 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_get_namespace_config.py +113 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_get_or_create_namespace.py +248 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_put_checkpoint.py +202 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_reset_cluster.py +67 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/utils/__init__.py +0 -0
- amzn_sagemaker_checkpointing-1.0.13/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/utils/test_base.py +113 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/tests/amzn_sagemaker_checkpointing/storage/clients/s3/test_s3_client.py +2 -2
- amzn_sagemaker_checkpointing-1.0.10/DEVELOPING.md +0 -46
- amzn_sagemaker_checkpointing-1.0.10/tests/amzn_sagemaker_checkpointing/checkpointing/filesystem/test_filesystem.py +0 -738
- amzn_sagemaker_checkpointing-1.0.10/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/test_inmemory_client.py +0 -77
- amzn_sagemaker_checkpointing-1.0.10/tests/test_dummy.py +0 -5
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/.crux_dry_run_build +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/.gitignore +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/LICENSE.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/brazil.ion +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements/requirements-build-tools.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements/requirements-hatch-build.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements/requirements-hatch-static-analysis.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements/requirements-hatch-test.py3.11.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements/requirements-hatch-test.py3.12.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/requirements.txt +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/checkpointing/filesystem/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/checkpointing/filesystem/exceptions.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/config/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/config/sagemaker_checkpoint_config.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/py.typed +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/inmemory/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/inmemory/checksum.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/inmemory/exceptions.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/local/disk_fs.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/storage/clients/s3/__init__.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/src/amzn_sagemaker_checkpointing/utils/logging_utils.py +0 -0
- {amzn_sagemaker_checkpointing-1.0.10 → amzn_sagemaker_checkpointing-1.0.13}/tests/amzn_sagemaker_checkpointing/storage/clients/inmemory/checksum_test.py +0 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Developing SageMakerCheckpointing
|
|
2
|
+
|
|
3
|
+
This package uses the [hatch](https://hatch.pypa.io/latest/) build system.
|
|
4
|
+
|
|
5
|
+
### Building
|
|
6
|
+
|
|
7
|
+
A number of scripts and commands exist in `pyproject.toml` under the `scripts` configurations with more
|
|
8
|
+
documentation in the comments of `pyproject.toml`. Running a script for a specific environment is simply running
|
|
9
|
+
`hatch run <env_name>:<script>`. You can omit the `<env_name>` for those under the `default` environment.
|
|
10
|
+
|
|
11
|
+
You need to set up hatch pluging first:
|
|
12
|
+
```
|
|
13
|
+
./setup-hatch.sh
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
### Available Hatch Commands
|
|
17
|
+
|
|
18
|
+
- **`hatch run release`** - Runs typing checks (mypy), tests, and coverage.
|
|
19
|
+
- **`hatch test --cover`** - Runs tests and coverage.
|
|
20
|
+
- **`hatch typing`** - Runs mypy type checking.
|
|
21
|
+
- **`hatch fmt`** - Formats code using ruff.
|
|
22
|
+
- **`hatch build`** - builds both source and wheel distributions in ./build directory.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: amzn-sagemaker-checkpointing
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.13
|
|
4
4
|
Summary: Amazon SageMaker Checkpointing Library
|
|
5
5
|
License: Apache 2.0
|
|
6
6
|
License-File: LICENSE.txt
|
|
@@ -95,12 +95,12 @@ following to your S3 bucket policy
|
|
|
95
95
|
```
|
|
96
96
|
|
|
97
97
|
## Installation
|
|
98
|
-
###
|
|
98
|
+
### Prerequisites
|
|
99
99
|
```bash
|
|
100
100
|
pip install s3torchconnector tenacity torch boto3 botocore
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
###
|
|
103
|
+
### SageMaker Checkpointing Library
|
|
104
104
|
```bash
|
|
105
105
|
pip install amzn-sagemaker-checkpointing
|
|
106
106
|
```
|
|
@@ -82,12 +82,12 @@ following to your S3 bucket policy
|
|
|
82
82
|
```
|
|
83
83
|
|
|
84
84
|
## Installation
|
|
85
|
-
###
|
|
85
|
+
### Prerequisites
|
|
86
86
|
```bash
|
|
87
87
|
pip install s3torchconnector tenacity torch boto3 botocore
|
|
88
88
|
```
|
|
89
89
|
|
|
90
|
-
###
|
|
90
|
+
### SageMaker Checkpointing Library
|
|
91
91
|
```bash
|
|
92
92
|
pip install amzn-sagemaker-checkpointing
|
|
93
93
|
```
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "amzn-sagemaker-checkpointing"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.13"
|
|
8
8
|
description = "Amazon SageMaker Checkpointing Library"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { "text" = "Apache 2.0" }
|
|
@@ -62,9 +62,12 @@ output = "private/brazil-documentation/coverage/coverage.xml"
|
|
|
62
62
|
[tool.coverage.html]
|
|
63
63
|
directory = "private/brazil-documentation/coverage/"
|
|
64
64
|
|
|
65
|
+
[tool.ruff]
|
|
66
|
+
exclude = [ "./build", ".hatch", "private", ".venv" ]
|
|
67
|
+
|
|
65
68
|
[tool.ruff.lint]
|
|
66
69
|
isort.known-first-party = ["amzn_sagemaker_checkpointing"]
|
|
67
|
-
|
|
70
|
+
select = ["E", "W", "F", "I", "N", "RUF"]
|
|
68
71
|
|
|
69
72
|
[tool.ruff.lint.per-file-ignores]
|
|
70
73
|
# Tests can use magic values, assertions, and relative imports
|
|
@@ -72,6 +75,7 @@ exclude = [ "./build", ".hatch", "private" ]
|
|
|
72
75
|
|
|
73
76
|
[tool.hatch.build]
|
|
74
77
|
directory = "./build"
|
|
78
|
+
exclude = ["DEVELOPING_INTERNAL.md"]
|
|
75
79
|
|
|
76
80
|
[tool.hatch.env]
|
|
77
81
|
requires = [ "hatch-pip-compile" ]
|
|
@@ -89,6 +93,7 @@ update = [ "hatch-pip-compile --upgrade --all" ]
|
|
|
89
93
|
generate-version = [ "hatch version | tail -1 > ./build/latest_version" ]
|
|
90
94
|
|
|
91
95
|
release = [
|
|
96
|
+
"hatch fmt",
|
|
92
97
|
"typing",
|
|
93
98
|
"hatch test --all --cover",
|
|
94
99
|
"mkdir -p ./build",
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -e
|
|
3
|
+
|
|
4
|
+
mkdir -p .hatch
|
|
5
|
+
|
|
6
|
+
cat > .hatch/hatch_plugin.py << 'EOF'
|
|
7
|
+
from hatch.env.collectors.plugin.interface import EnvironmentCollectorInterface
|
|
8
|
+
|
|
9
|
+
class CustomEnvironmentCollector(EnvironmentCollectorInterface):
|
|
10
|
+
PLUGIN_NAME = 'custom'
|
|
11
|
+
|
|
12
|
+
def get_initial_config(self):
|
|
13
|
+
return {}
|
|
14
|
+
|
|
15
|
+
def finalize_config(self, config):
|
|
16
|
+
return config
|
|
17
|
+
EOF
|
|
18
|
+
|
|
19
|
+
echo "Hatch plugin created"
|