denpex-sdk 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- denpex_sdk-0.2.3/PKG-INFO +99 -0
- denpex_sdk-0.2.2/PKG-INFO → denpex_sdk-0.2.3/README.md +72 -65
- denpex_sdk-0.2.3/pyproject.toml +41 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/client.py +16 -0
- denpex_sdk-0.2.3/src/denpex_sdk.egg-info/PKG-INFO +99 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_client.py +20 -0
- denpex_sdk-0.2.2/README.md +0 -54
- denpex_sdk-0.2.2/pyproject.toml +0 -21
- denpex_sdk-0.2.2/src/denpex_sdk.egg-info/PKG-INFO +0 -65
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/setup.cfg +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/__init__.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/checkpoint.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/checkpoint_extra.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/evidence.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/frameworks.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/instrument.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/sdc.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk/sdc_runner.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk.egg-info/SOURCES.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk.egg-info/dependency_links.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk.egg-info/requires.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/src/denpex_sdk.egg-info/top_level.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_checkpoint_extra.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_frameworks.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_layer_metrics.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_live_monitor.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_sdc.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.3}/tests/test_sdk.py +0 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: denpex-sdk
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
+
Author: Denpex
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Project-URL: Homepage, https://denpex.com
|
|
8
|
+
Project-URL: Documentation, https://denpex.com/docs/integrations
|
|
9
|
+
Project-URL: API Reference, https://denpex.com/docs/api
|
|
10
|
+
Project-URL: Python Agent & CLI, https://denpex.com/docs/agent
|
|
11
|
+
Project-URL: VS Code Extension, https://denpex.com/vscode
|
|
12
|
+
Project-URL: Open VSX Extension, https://denpex.com/openvsx
|
|
13
|
+
Project-URL: GPU Failure Encyclopedia, https://denpex.com/failures
|
|
14
|
+
Project-URL: Free GPU Debugging Tools, https://denpex.com/tools
|
|
15
|
+
Project-URL: Changelog, https://denpex.com/changelog
|
|
16
|
+
Project-URL: Repository, https://github.com/Denpex/gpu-failure-cheatsheet
|
|
17
|
+
Project-URL: Issue Tracker, https://denpex.com/contact
|
|
18
|
+
Keywords: denpex,gpu,cuda,nccl,pytorch,slurm,vllm,xid,mlops,diagnostics
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Intended Audience :: Developers
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Monitoring
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
Provides-Extra: torch
|
|
26
|
+
Requires-Dist: torch>=2.1; extra == "torch"
|
|
27
|
+
|
|
28
|
+
# denpex-sdk
|
|
29
|
+
|
|
30
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
31
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
32
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
33
|
+
|
|
34
|
+
## Cloud diagnosis client
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from denpex_sdk import DenpexClient
|
|
38
|
+
|
|
39
|
+
client = DenpexClient(api_key="dpx_...")
|
|
40
|
+
result = client.diagnose(
|
|
41
|
+
open("training.log").read(),
|
|
42
|
+
jobName="llama3-405b-pretrain",
|
|
43
|
+
telemetry={"jobId": "slurm-1842"},
|
|
44
|
+
)
|
|
45
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
46
|
+
print(result["fix"]["summary"])
|
|
47
|
+
print(result["fix"]["action"])
|
|
48
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
52
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
53
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
54
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
55
|
+
|
|
56
|
+
## SDC detection
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
60
|
+
|
|
61
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
62
|
+
weight_report = hook.inspect()
|
|
63
|
+
grad_report = hook.inspect_gradients()
|
|
64
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Checkpoint validation
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
71
|
+
|
|
72
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
73
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
74
|
+
handle.write(manifest_to_json(manifest))
|
|
75
|
+
|
|
76
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
77
|
+
expected = manifest_from_json(handle.read())
|
|
78
|
+
|
|
79
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
80
|
+
assert result.valid
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Official Denpex Ecosystem & Documentation
|
|
86
|
+
|
|
87
|
+
| Resource | Description | Link |
|
|
88
|
+
| :--- | :--- | :--- |
|
|
89
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
90
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
91
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
92
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
93
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
94
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
95
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
96
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
97
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
98
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
99
|
+
|
|
@@ -1,65 +1,72 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
##
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
1
|
+
# denpex-sdk
|
|
2
|
+
|
|
3
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
4
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
5
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
6
|
+
|
|
7
|
+
## Cloud diagnosis client
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from denpex_sdk import DenpexClient
|
|
11
|
+
|
|
12
|
+
client = DenpexClient(api_key="dpx_...")
|
|
13
|
+
result = client.diagnose(
|
|
14
|
+
open("training.log").read(),
|
|
15
|
+
jobName="llama3-405b-pretrain",
|
|
16
|
+
telemetry={"jobId": "slurm-1842"},
|
|
17
|
+
)
|
|
18
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
19
|
+
print(result["fix"]["summary"])
|
|
20
|
+
print(result["fix"]["action"])
|
|
21
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
25
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
26
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
27
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
28
|
+
|
|
29
|
+
## SDC detection
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
33
|
+
|
|
34
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
35
|
+
weight_report = hook.inspect()
|
|
36
|
+
grad_report = hook.inspect_gradients()
|
|
37
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Checkpoint validation
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
44
|
+
|
|
45
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
46
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
47
|
+
handle.write(manifest_to_json(manifest))
|
|
48
|
+
|
|
49
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
50
|
+
expected = manifest_from_json(handle.read())
|
|
51
|
+
|
|
52
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
53
|
+
assert result.valid
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Official Denpex Ecosystem & Documentation
|
|
59
|
+
|
|
60
|
+
| Resource | Description | Link |
|
|
61
|
+
| :--- | :--- | :--- |
|
|
62
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
63
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
64
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
65
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
66
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
67
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
68
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
69
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
70
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
71
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
72
|
+
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "denpex-sdk"
|
|
7
|
+
version = "0.2.3"
|
|
8
|
+
description = "Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "Proprietary" }
|
|
12
|
+
authors = [{ name = "Denpex" }]
|
|
13
|
+
keywords = ["denpex", "gpu", "cuda", "nccl", "pytorch", "slurm", "vllm", "xid", "mlops", "diagnostics"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
18
|
+
"Topic :: System :: Monitoring",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
"Homepage" = "https://denpex.com"
|
|
23
|
+
"Documentation" = "https://denpex.com/docs/integrations"
|
|
24
|
+
"API Reference" = "https://denpex.com/docs/api"
|
|
25
|
+
"Python Agent & CLI" = "https://denpex.com/docs/agent"
|
|
26
|
+
"VS Code Extension" = "https://denpex.com/vscode"
|
|
27
|
+
"Open VSX Extension" = "https://denpex.com/openvsx"
|
|
28
|
+
"GPU Failure Encyclopedia" = "https://denpex.com/failures"
|
|
29
|
+
"Free GPU Debugging Tools" = "https://denpex.com/tools"
|
|
30
|
+
"Changelog" = "https://denpex.com/changelog"
|
|
31
|
+
"Repository" = "https://github.com/Denpex/gpu-failure-cheatsheet"
|
|
32
|
+
"Issue Tracker" = "https://denpex.com/contact"
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
torch = ["torch>=2.1"]
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.packages.find]
|
|
38
|
+
where = ["src"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
denpex_sdk = ["tests/*.xml"]
|
|
@@ -87,6 +87,22 @@ class DenpexClient:
|
|
|
87
87
|
selected_logs, _truncated = select_evidence(logs, MAX_LOG_CHARS)
|
|
88
88
|
return self._request("POST", "/api/diagnose", {"logs": selected_logs, **options})
|
|
89
89
|
|
|
90
|
+
def refine(self, logs: str, operator_answers: list, **options: Any) -> Dict[str, Any]:
|
|
91
|
+
"""Submit observed answers with the original logs. Check selection is not a result."""
|
|
92
|
+
return self.diagnose(logs, operatorAnswers=operator_answers, **options)
|
|
93
|
+
|
|
94
|
+
def followup(self, logs: str, diagnosis_id: str, question: str,
|
|
95
|
+
conversation_id: Optional[str] = None) -> Dict[str, Any]:
|
|
96
|
+
"""Continue the same incident and preserve its structured questions."""
|
|
97
|
+
if not isinstance(logs, str) or len(logs) < MIN_LOG_CHARS or not diagnosis_id or not isinstance(question, str) or not question.strip():
|
|
98
|
+
raise DenpexError("Original logs, diagnosis_id and a non-empty question are required", 400)
|
|
99
|
+
selected_logs, _ = select_evidence(logs, MAX_LOG_CHARS)
|
|
100
|
+
body: Dict[str, Any] = {"logs": selected_logs, "diagnosisId": diagnosis_id,
|
|
101
|
+
"history": [{"role": "user", "content": question}]}
|
|
102
|
+
if conversation_id:
|
|
103
|
+
body["conversationId"] = conversation_id
|
|
104
|
+
return self._request("POST", "/api/diagnose/chat", body)
|
|
105
|
+
|
|
90
106
|
def feedback(self, diagnosis_id: str, outcome: str, note: Optional[str] = None) -> Dict[str, Any]:
|
|
91
107
|
"""Report whether a diagnosis worked ("worked" | "did_not_work");
|
|
92
108
|
this is a user-reported outcome, not independent root-cause verification."""
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: denpex-sdk
|
|
3
|
+
Version: 0.2.3
|
|
4
|
+
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
+
Author: Denpex
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Project-URL: Homepage, https://denpex.com
|
|
8
|
+
Project-URL: Documentation, https://denpex.com/docs/integrations
|
|
9
|
+
Project-URL: API Reference, https://denpex.com/docs/api
|
|
10
|
+
Project-URL: Python Agent & CLI, https://denpex.com/docs/agent
|
|
11
|
+
Project-URL: VS Code Extension, https://denpex.com/vscode
|
|
12
|
+
Project-URL: Open VSX Extension, https://denpex.com/openvsx
|
|
13
|
+
Project-URL: GPU Failure Encyclopedia, https://denpex.com/failures
|
|
14
|
+
Project-URL: Free GPU Debugging Tools, https://denpex.com/tools
|
|
15
|
+
Project-URL: Changelog, https://denpex.com/changelog
|
|
16
|
+
Project-URL: Repository, https://github.com/Denpex/gpu-failure-cheatsheet
|
|
17
|
+
Project-URL: Issue Tracker, https://denpex.com/contact
|
|
18
|
+
Keywords: denpex,gpu,cuda,nccl,pytorch,slurm,vllm,xid,mlops,diagnostics
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Intended Audience :: Developers
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Monitoring
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
Provides-Extra: torch
|
|
26
|
+
Requires-Dist: torch>=2.1; extra == "torch"
|
|
27
|
+
|
|
28
|
+
# denpex-sdk
|
|
29
|
+
|
|
30
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
31
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
32
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
33
|
+
|
|
34
|
+
## Cloud diagnosis client
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from denpex_sdk import DenpexClient
|
|
38
|
+
|
|
39
|
+
client = DenpexClient(api_key="dpx_...")
|
|
40
|
+
result = client.diagnose(
|
|
41
|
+
open("training.log").read(),
|
|
42
|
+
jobName="llama3-405b-pretrain",
|
|
43
|
+
telemetry={"jobId": "slurm-1842"},
|
|
44
|
+
)
|
|
45
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
46
|
+
print(result["fix"]["summary"])
|
|
47
|
+
print(result["fix"]["action"])
|
|
48
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
52
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
53
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
54
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
55
|
+
|
|
56
|
+
## SDC detection
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
60
|
+
|
|
61
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
62
|
+
weight_report = hook.inspect()
|
|
63
|
+
grad_report = hook.inspect_gradients()
|
|
64
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Checkpoint validation
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
71
|
+
|
|
72
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
73
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
74
|
+
handle.write(manifest_to_json(manifest))
|
|
75
|
+
|
|
76
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
77
|
+
expected = manifest_from_json(handle.read())
|
|
78
|
+
|
|
79
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
80
|
+
assert result.valid
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Official Denpex Ecosystem & Documentation
|
|
86
|
+
|
|
87
|
+
| Resource | Description | Link |
|
|
88
|
+
| :--- | :--- | :--- |
|
|
89
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
90
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
91
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
92
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
93
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
94
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
95
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
96
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
97
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
98
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
99
|
+
|
|
@@ -186,3 +186,23 @@ class ApiKeyTests(unittest.TestCase):
|
|
|
186
186
|
|
|
187
187
|
if __name__ == "__main__":
|
|
188
188
|
unittest.main()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class ClarificationFollowupTests(unittest.TestCase):
|
|
192
|
+
def test_followup_keeps_incident_and_questions(self):
|
|
193
|
+
captured = []
|
|
194
|
+
payload = {"conversationId": "thread-one", "reply": {"clarifyingChoices": [{"allowOther": True}]}}
|
|
195
|
+
client = DenpexClient(api_key="unit-test-only", opener=_stub_opener(payload, captured))
|
|
196
|
+
self.assertEqual(client.followup(LOG, "diagnosis-one", "First error was I/O", "thread-one"), payload)
|
|
197
|
+
body = json.loads(captured[0].data)
|
|
198
|
+
self.assertEqual(body["logs"], LOG)
|
|
199
|
+
self.assertEqual(body["diagnosisId"], "diagnosis-one")
|
|
200
|
+
self.assertEqual(body["conversationId"], "thread-one")
|
|
201
|
+
self.assertTrue(captured[0].full_url.endswith("/api/diagnose/chat"))
|
|
202
|
+
|
|
203
|
+
def test_refine_preserves_typed_answers(self):
|
|
204
|
+
captured = []
|
|
205
|
+
client = DenpexClient(api_key="unit-test-only", opener=_stub_opener({}, captured))
|
|
206
|
+
answers = [{"questionId": "scope", "optionId": "unsure", "answer": "Not sure"}]
|
|
207
|
+
client.refine(LOG, answers)
|
|
208
|
+
self.assertEqual(json.loads(captured[0].data)["operatorAnswers"], answers)
|
denpex_sdk-0.2.2/README.md
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
# denpex-sdk
|
|
2
|
-
|
|
3
|
-
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
4
|
-
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
5
|
-
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
6
|
-
|
|
7
|
-
## Cloud diagnosis client
|
|
8
|
-
|
|
9
|
-
```python
|
|
10
|
-
from denpex_sdk import DenpexClient
|
|
11
|
-
|
|
12
|
-
client = DenpexClient(api_key="dpx_...")
|
|
13
|
-
result = client.diagnose(
|
|
14
|
-
open("training.log").read(),
|
|
15
|
-
jobName="llama3-405b-pretrain",
|
|
16
|
-
telemetry={"jobId": "slurm-1842"},
|
|
17
|
-
)
|
|
18
|
-
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
19
|
-
print(result["fix"]["summary"])
|
|
20
|
-
print(result["fix"]["action"])
|
|
21
|
-
client.feedback(result["diagnosisId"], "worked")
|
|
22
|
-
```
|
|
23
|
-
|
|
24
|
-
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
25
|
-
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
26
|
-
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
27
|
-
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
28
|
-
|
|
29
|
-
## SDC detection
|
|
30
|
-
|
|
31
|
-
```python
|
|
32
|
-
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
33
|
-
|
|
34
|
-
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
35
|
-
weight_report = hook.inspect()
|
|
36
|
-
grad_report = hook.inspect_gradients()
|
|
37
|
-
opt_report = hook.inspect_optimizer(optimizer)
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
## Checkpoint validation
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
44
|
-
|
|
45
|
-
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
46
|
-
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
47
|
-
handle.write(manifest_to_json(manifest))
|
|
48
|
-
|
|
49
|
-
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
50
|
-
expected = manifest_from_json(handle.read())
|
|
51
|
-
|
|
52
|
-
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
53
|
-
assert result.valid
|
|
54
|
-
```
|
denpex_sdk-0.2.2/pyproject.toml
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=68", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "denpex-sdk"
|
|
7
|
-
version = "0.2.2"
|
|
8
|
-
description = "Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.8"
|
|
11
|
-
license = { text = "Proprietary" }
|
|
12
|
-
authors = [{ name = "Denpex" }]
|
|
13
|
-
|
|
14
|
-
[project.optional-dependencies]
|
|
15
|
-
torch = ["torch>=2.1"]
|
|
16
|
-
|
|
17
|
-
[tool.setuptools.packages.find]
|
|
18
|
-
where = ["src"]
|
|
19
|
-
|
|
20
|
-
[tool.setuptools.package-data]
|
|
21
|
-
denpex_sdk = ["tests/*.xml"]
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: denpex-sdk
|
|
3
|
-
Version: 0.2.2
|
|
4
|
-
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
-
Author: Denpex
|
|
6
|
-
License: Proprietary
|
|
7
|
-
Requires-Python: >=3.8
|
|
8
|
-
Description-Content-Type: text/markdown
|
|
9
|
-
Provides-Extra: torch
|
|
10
|
-
Requires-Dist: torch>=2.1; extra == "torch"
|
|
11
|
-
|
|
12
|
-
# denpex-sdk
|
|
13
|
-
|
|
14
|
-
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
15
|
-
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
16
|
-
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
17
|
-
|
|
18
|
-
## Cloud diagnosis client
|
|
19
|
-
|
|
20
|
-
```python
|
|
21
|
-
from denpex_sdk import DenpexClient
|
|
22
|
-
|
|
23
|
-
client = DenpexClient(api_key="dpx_...")
|
|
24
|
-
result = client.diagnose(
|
|
25
|
-
open("training.log").read(),
|
|
26
|
-
jobName="llama3-405b-pretrain",
|
|
27
|
-
telemetry={"jobId": "slurm-1842"},
|
|
28
|
-
)
|
|
29
|
-
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
30
|
-
print(result["fix"]["summary"])
|
|
31
|
-
print(result["fix"]["action"])
|
|
32
|
-
client.feedback(result["diagnosisId"], "worked")
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
36
|
-
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
37
|
-
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
38
|
-
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
39
|
-
|
|
40
|
-
## SDC detection
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
44
|
-
|
|
45
|
-
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
46
|
-
weight_report = hook.inspect()
|
|
47
|
-
grad_report = hook.inspect_gradients()
|
|
48
|
-
opt_report = hook.inspect_optimizer(optimizer)
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
## Checkpoint validation
|
|
52
|
-
|
|
53
|
-
```python
|
|
54
|
-
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
55
|
-
|
|
56
|
-
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
57
|
-
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
58
|
-
handle.write(manifest_to_json(manifest))
|
|
59
|
-
|
|
60
|
-
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
61
|
-
expected = manifest_from_json(handle.read())
|
|
62
|
-
|
|
63
|
-
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
64
|
-
assert result.valid
|
|
65
|
-
```
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|