denpex-sdk 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- denpex_sdk-0.2.4/PKG-INFO +99 -0
- denpex_sdk-0.2.2/PKG-INFO → denpex_sdk-0.2.4/README.md +72 -65
- denpex_sdk-0.2.4/pyproject.toml +41 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/__init__.py +16 -2
- denpex_sdk-0.2.4/src/denpex_sdk/client.py +333 -0
- denpex_sdk-0.2.4/src/denpex_sdk.egg-info/PKG-INFO +99 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_client.py +74 -1
- denpex_sdk-0.2.2/README.md +0 -54
- denpex_sdk-0.2.2/pyproject.toml +0 -21
- denpex_sdk-0.2.2/src/denpex_sdk/client.py +0 -134
- denpex_sdk-0.2.2/src/denpex_sdk.egg-info/PKG-INFO +0 -65
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/setup.cfg +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/checkpoint.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/checkpoint_extra.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/evidence.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/frameworks.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/instrument.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/sdc.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk/sdc_runner.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk.egg-info/SOURCES.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk.egg-info/dependency_links.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk.egg-info/requires.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/src/denpex_sdk.egg-info/top_level.txt +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_checkpoint_extra.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_frameworks.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_layer_metrics.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_live_monitor.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_sdc.py +0 -0
- {denpex_sdk-0.2.2 → denpex_sdk-0.2.4}/tests/test_sdk.py +0 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: denpex-sdk
|
|
3
|
+
Version: 0.2.4
|
|
4
|
+
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
+
Author: Denpex
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Project-URL: Homepage, https://denpex.com
|
|
8
|
+
Project-URL: Documentation, https://denpex.com/docs/integrations
|
|
9
|
+
Project-URL: API Reference, https://denpex.com/docs/api
|
|
10
|
+
Project-URL: Python Agent & CLI, https://denpex.com/docs/agent
|
|
11
|
+
Project-URL: VS Code Extension, https://denpex.com/vscode
|
|
12
|
+
Project-URL: Open VSX Extension, https://denpex.com/openvsx
|
|
13
|
+
Project-URL: GPU Failure Encyclopedia, https://denpex.com/failures
|
|
14
|
+
Project-URL: Free GPU Debugging Tools, https://denpex.com/tools
|
|
15
|
+
Project-URL: Changelog, https://denpex.com/changelog
|
|
16
|
+
Project-URL: Repository, https://github.com/Denpex/gpu-failure-cheatsheet
|
|
17
|
+
Project-URL: Issue Tracker, https://denpex.com/contact
|
|
18
|
+
Keywords: denpex,gpu,cuda,nccl,pytorch,slurm,vllm,xid,mlops,diagnostics
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Intended Audience :: Developers
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Monitoring
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
Provides-Extra: torch
|
|
26
|
+
Requires-Dist: torch>=2.1; extra == "torch"
|
|
27
|
+
|
|
28
|
+
# denpex-sdk
|
|
29
|
+
|
|
30
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
31
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
32
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
33
|
+
|
|
34
|
+
## Cloud diagnosis client
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from denpex_sdk import DenpexClient
|
|
38
|
+
|
|
39
|
+
client = DenpexClient(api_key="dpx_...")
|
|
40
|
+
result = client.diagnose(
|
|
41
|
+
open("training.log").read(),
|
|
42
|
+
jobName="llama3-405b-pretrain",
|
|
43
|
+
telemetry={"jobId": "slurm-1842"},
|
|
44
|
+
)
|
|
45
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
46
|
+
print(result["fix"]["summary"])
|
|
47
|
+
print(result["fix"]["action"])
|
|
48
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
52
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
53
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
54
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
55
|
+
|
|
56
|
+
## SDC detection
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
60
|
+
|
|
61
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
62
|
+
weight_report = hook.inspect()
|
|
63
|
+
grad_report = hook.inspect_gradients()
|
|
64
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Checkpoint validation
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
71
|
+
|
|
72
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
73
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
74
|
+
handle.write(manifest_to_json(manifest))
|
|
75
|
+
|
|
76
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
77
|
+
expected = manifest_from_json(handle.read())
|
|
78
|
+
|
|
79
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
80
|
+
assert result.valid
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Official Denpex Ecosystem & Documentation
|
|
86
|
+
|
|
87
|
+
| Resource | Description | Link |
|
|
88
|
+
| :--- | :--- | :--- |
|
|
89
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
90
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
91
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
92
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
93
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
94
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
95
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
96
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
97
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
98
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
99
|
+
|
|
@@ -1,65 +1,72 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
##
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
1
|
+
# denpex-sdk
|
|
2
|
+
|
|
3
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
4
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
5
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
6
|
+
|
|
7
|
+
## Cloud diagnosis client
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from denpex_sdk import DenpexClient
|
|
11
|
+
|
|
12
|
+
client = DenpexClient(api_key="dpx_...")
|
|
13
|
+
result = client.diagnose(
|
|
14
|
+
open("training.log").read(),
|
|
15
|
+
jobName="llama3-405b-pretrain",
|
|
16
|
+
telemetry={"jobId": "slurm-1842"},
|
|
17
|
+
)
|
|
18
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
19
|
+
print(result["fix"]["summary"])
|
|
20
|
+
print(result["fix"]["action"])
|
|
21
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
25
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
26
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
27
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
28
|
+
|
|
29
|
+
## SDC detection
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
33
|
+
|
|
34
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
35
|
+
weight_report = hook.inspect()
|
|
36
|
+
grad_report = hook.inspect_gradients()
|
|
37
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Checkpoint validation
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
44
|
+
|
|
45
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
46
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
47
|
+
handle.write(manifest_to_json(manifest))
|
|
48
|
+
|
|
49
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
50
|
+
expected = manifest_from_json(handle.read())
|
|
51
|
+
|
|
52
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
53
|
+
assert result.valid
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Official Denpex Ecosystem & Documentation
|
|
59
|
+
|
|
60
|
+
| Resource | Description | Link |
|
|
61
|
+
| :--- | :--- | :--- |
|
|
62
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
63
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
64
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
65
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
66
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
67
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
68
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
69
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
70
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
71
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
72
|
+
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "denpex-sdk"
|
|
7
|
+
version = "0.2.4"
|
|
8
|
+
description = "Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "Proprietary" }
|
|
12
|
+
authors = [{ name = "Denpex" }]
|
|
13
|
+
keywords = ["denpex", "gpu", "cuda", "nccl", "pytorch", "slurm", "vllm", "xid", "mlops", "diagnostics"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
18
|
+
"Topic :: System :: Monitoring",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
"Homepage" = "https://denpex.com"
|
|
23
|
+
"Documentation" = "https://denpex.com/docs/integrations"
|
|
24
|
+
"API Reference" = "https://denpex.com/docs/api"
|
|
25
|
+
"Python Agent & CLI" = "https://denpex.com/docs/agent"
|
|
26
|
+
"VS Code Extension" = "https://denpex.com/vscode"
|
|
27
|
+
"Open VSX Extension" = "https://denpex.com/openvsx"
|
|
28
|
+
"GPU Failure Encyclopedia" = "https://denpex.com/failures"
|
|
29
|
+
"Free GPU Debugging Tools" = "https://denpex.com/tools"
|
|
30
|
+
"Changelog" = "https://denpex.com/changelog"
|
|
31
|
+
"Repository" = "https://github.com/Denpex/gpu-failure-cheatsheet"
|
|
32
|
+
"Issue Tracker" = "https://denpex.com/contact"
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
torch = ["torch>=2.1"]
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.packages.find]
|
|
38
|
+
where = ["src"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
denpex_sdk = ["tests/*.xml"]
|
|
@@ -53,6 +53,20 @@ __all__ = [
|
|
|
53
53
|
]
|
|
54
54
|
|
|
55
55
|
# Public API client (programmatic diagnosis over https://api.denpex.com).
|
|
56
|
-
from .client import
|
|
56
|
+
from .client import ( # noqa: E402
|
|
57
|
+
DenpexClient,
|
|
58
|
+
DenpexError,
|
|
59
|
+
activate_trial,
|
|
60
|
+
default_credentials_path,
|
|
61
|
+
load_stored_api_key,
|
|
62
|
+
request_trial,
|
|
63
|
+
)
|
|
57
64
|
|
|
58
|
-
__all__ += [
|
|
65
|
+
__all__ += [
|
|
66
|
+
"DenpexClient",
|
|
67
|
+
"DenpexError",
|
|
68
|
+
"request_trial",
|
|
69
|
+
"activate_trial",
|
|
70
|
+
"load_stored_api_key",
|
|
71
|
+
"default_credentials_path",
|
|
72
|
+
]
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"""Denpex API client for programmatic diagnosis over the public API.
|
|
2
|
+
|
|
3
|
+
Mirrors the TypeScript SDK (denpex-sdk-ts) against the same contract
|
|
4
|
+
(worker/api-worker-optimized.ts):
|
|
5
|
+
|
|
6
|
+
POST /api/diagnose {"logs": ..., "jobName": ...} -> diagnosis
|
|
7
|
+
POST /api/diagnose/feedback {"diagnosisId": ..., "outcome": ...}
|
|
8
|
+
GET /api/diagnose/quota
|
|
9
|
+
|
|
10
|
+
Auth: an API key from https://denpex.com/dashboard sent as ``X-Denpex-Key``.
|
|
11
|
+
Zero third-party dependencies (urllib only), matching the SDK's design rule.
|
|
12
|
+
|
|
13
|
+
from denpex_sdk import DenpexClient
|
|
14
|
+
|
|
15
|
+
client = DenpexClient() # reads DENPEX_API_KEY from the environment
|
|
16
|
+
result = client.diagnose(open("training.log").read())
|
|
17
|
+
print(result["meta"]["failureType"], result["meta"].get("engineVersion"))
|
|
18
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import urllib.error
|
|
27
|
+
import urllib.request
|
|
28
|
+
from typing import Any, Callable, Dict, Optional, Tuple
|
|
29
|
+
|
|
30
|
+
from .evidence import select_evidence
|
|
31
|
+
|
|
32
|
+
DEFAULT_BASE_URL = "https://api.denpex.com"
|
|
33
|
+
MIN_LOG_CHARS = 6
|
|
34
|
+
MAX_LOG_CHARS = 256_000
|
|
35
|
+
_DEFAULT_TIMEOUT_S = 320.0 # API deep-path ceiling is 290s; retain network buffer
|
|
36
|
+
_TRIAL_EMAIL_RE = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]{2,}$")
|
|
37
|
+
_TRIAL_CODE_RE = re.compile(r"^DPXV-[0-9A-F]{32}$")
|
|
38
|
+
_TRIAL_CONSUMER_DOMAINS = {
|
|
39
|
+
"gmail.com", "googlemail.com", "yahoo.com", "yahoo.co.uk", "ymail.com",
|
|
40
|
+
"hotmail.com", "hotmail.co.uk", "outlook.com", "live.com", "msn.com",
|
|
41
|
+
"aol.com", "icloud.com", "me.com", "mac.com", "protonmail.com", "proton.me",
|
|
42
|
+
"gmx.com", "web.de", "mail.com", "zoho.com", "yandex.com", "qq.com",
|
|
43
|
+
"163.com", "naver.com", "fastmail.com", "tutanota.com", "duck.com",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def default_credentials_path() -> str:
|
|
48
|
+
override = os.environ.get("DENPEX_CREDENTIALS_PATH", "").strip()
|
|
49
|
+
if override:
|
|
50
|
+
return os.path.expanduser(override)
|
|
51
|
+
home = os.path.expanduser("~")
|
|
52
|
+
if not home or home == "~":
|
|
53
|
+
return ""
|
|
54
|
+
return os.path.join(home, ".denpex", "credentials.json")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def load_stored_api_key(cred_path: Optional[str] = None) -> str:
|
|
58
|
+
"""Read an activated Denpex API key saved by `denpex trial` or `activate_trial()`."""
|
|
59
|
+
path = cred_path or default_credentials_path()
|
|
60
|
+
if not path or not os.path.isfile(path):
|
|
61
|
+
return ""
|
|
62
|
+
try:
|
|
63
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
64
|
+
data = json.load(fh)
|
|
65
|
+
if isinstance(data, dict) and isinstance(data.get("apiKey"), str):
|
|
66
|
+
return data["apiKey"].strip()
|
|
67
|
+
except Exception:
|
|
68
|
+
return ""
|
|
69
|
+
return ""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def validate_trial_work_email(raw: str) -> str:
|
|
73
|
+
"""Validate and normalize a company work email before requesting an in-app trial."""
|
|
74
|
+
email = str(raw or "").strip().lower()
|
|
75
|
+
if not _TRIAL_EMAIL_RE.match(email):
|
|
76
|
+
raise DenpexError("Enter a valid work email address (e.g. alex@yourcompany.com).", 400)
|
|
77
|
+
domain = email.rsplit("@", 1)[-1]
|
|
78
|
+
if domain in _TRIAL_CONSUMER_DOMAINS:
|
|
79
|
+
raise DenpexError(
|
|
80
|
+
f"Team trials require a work email domain (e.g. @company.com, not @{domain}).",
|
|
81
|
+
400,
|
|
82
|
+
)
|
|
83
|
+
return email
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _trial_post_json(
|
|
87
|
+
base_url: str,
|
|
88
|
+
path: str,
|
|
89
|
+
body: Dict[str, Any],
|
|
90
|
+
timeout_s: float = 30.0,
|
|
91
|
+
opener: Optional[Callable[..., Any]] = None,
|
|
92
|
+
) -> Tuple[int, Dict[str, Any]]:
|
|
93
|
+
data = json.dumps(body).encode("utf-8")
|
|
94
|
+
req = urllib.request.Request(
|
|
95
|
+
f"{base_url.rstrip('/')}{path}",
|
|
96
|
+
data=data,
|
|
97
|
+
method="POST",
|
|
98
|
+
headers={
|
|
99
|
+
"Accept": "application/json",
|
|
100
|
+
"Content-Type": "application/json",
|
|
101
|
+
},
|
|
102
|
+
)
|
|
103
|
+
open_fn = opener or urllib.request.urlopen
|
|
104
|
+
try:
|
|
105
|
+
with open_fn(req, timeout=timeout_s) as resp: # type: ignore[call-arg]
|
|
106
|
+
status = getattr(resp, "status", 200)
|
|
107
|
+
raw = resp.read().decode("utf-8", errors="replace") or "{}"
|
|
108
|
+
return int(status), json.loads(raw)
|
|
109
|
+
except urllib.error.HTTPError as exc:
|
|
110
|
+
raw = exc.read().decode("utf-8", errors="replace") or "{}"
|
|
111
|
+
try:
|
|
112
|
+
payload = json.loads(raw)
|
|
113
|
+
except Exception:
|
|
114
|
+
payload = {"message": raw or str(exc)}
|
|
115
|
+
return int(exc.code), payload if isinstance(payload, dict) else {"message": str(payload)}
|
|
116
|
+
except urllib.error.URLError as exc:
|
|
117
|
+
raise DenpexError(f"cannot reach Denpex API: {exc.reason}", 0) from exc
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def request_trial(
|
|
121
|
+
email: str,
|
|
122
|
+
source: str = "python-sdk",
|
|
123
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
124
|
+
timeout_s: float = 30.0,
|
|
125
|
+
opener: Optional[Callable[..., Any]] = None,
|
|
126
|
+
) -> Dict[str, Any]:
|
|
127
|
+
"""Request a 30-day Team trial verification code via email without opening a browser."""
|
|
128
|
+
clean_email = validate_trial_work_email(email)
|
|
129
|
+
status, payload = _trial_post_json(
|
|
130
|
+
base_url,
|
|
131
|
+
"/api/trial/request",
|
|
132
|
+
{"email": clean_email, "source": source},
|
|
133
|
+
timeout_s=timeout_s,
|
|
134
|
+
opener=opener,
|
|
135
|
+
)
|
|
136
|
+
sent = (200 <= status < 300) and (payload.get("ok") is not False) and (payload.get("sent") is not False)
|
|
137
|
+
if not sent:
|
|
138
|
+
msg = payload.get("message") or payload.get("error") or f"Trial request failed (status {status})."
|
|
139
|
+
raise DenpexError(str(msg), status, payload.get("code"))
|
|
140
|
+
return payload
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def activate_trial(
|
|
144
|
+
email: str,
|
|
145
|
+
code: str,
|
|
146
|
+
client: str = "cli",
|
|
147
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
148
|
+
timeout_s: float = 30.0,
|
|
149
|
+
save_credentials: bool = True,
|
|
150
|
+
cred_path: Optional[str] = None,
|
|
151
|
+
opener: Optional[Callable[..., Any]] = None,
|
|
152
|
+
) -> Dict[str, Any]:
|
|
153
|
+
"""Complete a 30-day Team trial inside Python/Jupyter with an emailed DPXV- code."""
|
|
154
|
+
clean_email = validate_trial_work_email(email)
|
|
155
|
+
clean_code = str(code or "").strip().upper()
|
|
156
|
+
if not _TRIAL_CODE_RE.match(clean_code):
|
|
157
|
+
raise DenpexError(
|
|
158
|
+
"Paste the whole code from your email. It starts with DPXV- followed by 32 letters and digits.",
|
|
159
|
+
400,
|
|
160
|
+
)
|
|
161
|
+
key = "dpx_" + os.urandom(16).hex()
|
|
162
|
+
key_hash = hashlib.sha256(key.encode("utf-8")).hexdigest()
|
|
163
|
+
key_display = f"{key[:8]}...{key[-4:]}"
|
|
164
|
+
status, payload = _trial_post_json(
|
|
165
|
+
base_url,
|
|
166
|
+
"/api/trial/activate-in-editor",
|
|
167
|
+
{
|
|
168
|
+
"email": clean_email,
|
|
169
|
+
"code": clean_code,
|
|
170
|
+
"keyHash": key_hash,
|
|
171
|
+
"keyDisplay": key_display,
|
|
172
|
+
"client": client,
|
|
173
|
+
},
|
|
174
|
+
timeout_s=timeout_s,
|
|
175
|
+
opener=opener,
|
|
176
|
+
)
|
|
177
|
+
if not (200 <= status < 300) or payload.get("status") != "active":
|
|
178
|
+
msg = payload.get("message") or payload.get("error") or f"Trial activation failed (status {status})."
|
|
179
|
+
raise DenpexError(str(msg), status, payload.get("status"))
|
|
180
|
+
|
|
181
|
+
saved_to = ""
|
|
182
|
+
target = cred_path or (default_credentials_path() if save_credentials else "")
|
|
183
|
+
if save_credentials and target:
|
|
184
|
+
parent = os.path.dirname(target)
|
|
185
|
+
if parent:
|
|
186
|
+
os.makedirs(parent, mode=0o700, exist_ok=True)
|
|
187
|
+
record = {
|
|
188
|
+
"apiKey": key,
|
|
189
|
+
"keyDisplay": key_display,
|
|
190
|
+
"email": clean_email,
|
|
191
|
+
"plan": payload.get("plan") or "team",
|
|
192
|
+
"apiUrl": base_url.rstrip("/"),
|
|
193
|
+
}
|
|
194
|
+
with open(target, "w", encoding="utf-8") as fh:
|
|
195
|
+
json.dump(record, fh, indent=2)
|
|
196
|
+
fh.write("\n")
|
|
197
|
+
try:
|
|
198
|
+
os.chmod(target, 0o600)
|
|
199
|
+
except OSError:
|
|
200
|
+
pass
|
|
201
|
+
saved_to = target
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
"status": "active",
|
|
205
|
+
"api_key": key,
|
|
206
|
+
"key_display": key_display,
|
|
207
|
+
"plan": payload.get("plan") or "team",
|
|
208
|
+
"daily_cap": payload.get("dailyCap"),
|
|
209
|
+
"expires_at": payload.get("expiresAt"),
|
|
210
|
+
"message": payload.get("message") or "Your 30-day Team trial is active.",
|
|
211
|
+
"credentials_path": saved_to,
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class DenpexError(RuntimeError):
|
|
216
|
+
"""API or validation failure. ``status`` is the HTTP status (0 = client-side),
|
|
217
|
+
``code`` the machine-readable API code when provided (e.g. AUTH_REQUIRED)."""
|
|
218
|
+
|
|
219
|
+
def __init__(
|
|
220
|
+
self,
|
|
221
|
+
message: str,
|
|
222
|
+
status: int = 0,
|
|
223
|
+
code: Optional[str] = None,
|
|
224
|
+
guidance: Optional[str] = None,
|
|
225
|
+
) -> None:
|
|
226
|
+
super().__init__(message)
|
|
227
|
+
self.status = status
|
|
228
|
+
self.code = code
|
|
229
|
+
self.guidance = guidance
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
class DenpexClient:
|
|
233
|
+
"""Minimal, dependency-free client for the Denpex public API."""
|
|
234
|
+
|
|
235
|
+
def __init__(
|
|
236
|
+
self,
|
|
237
|
+
api_key: Optional[str] = None,
|
|
238
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
239
|
+
timeout_s: float = _DEFAULT_TIMEOUT_S,
|
|
240
|
+
opener: Optional[Callable[..., Any]] = None,
|
|
241
|
+
) -> None:
|
|
242
|
+
if api_key is not None:
|
|
243
|
+
resolved_key = api_key
|
|
244
|
+
else:
|
|
245
|
+
resolved_key = os.environ.get("DENPEX_API_KEY", "").strip() or load_stored_api_key()
|
|
246
|
+
self.api_key = resolved_key
|
|
247
|
+
if not self.api_key:
|
|
248
|
+
raise DenpexError(
|
|
249
|
+
"API key required: run `denpex trial` (or `denpex_sdk.request_trial(...)`) with a work email, "
|
|
250
|
+
"set DENPEX_API_KEY, or create a key at https://denpex.com/dashboard"
|
|
251
|
+
)
|
|
252
|
+
self.base_url = base_url.rstrip("/")
|
|
253
|
+
self.timeout_s = timeout_s
|
|
254
|
+
# Injectable for tests; defaults to urllib. Signature: (Request, timeout=...) -> response
|
|
255
|
+
self._opener = opener or urllib.request.urlopen
|
|
256
|
+
|
|
257
|
+
# ── public surface ──────────────────────────────────────────────────────
|
|
258
|
+
|
|
259
|
+
def diagnose(self, logs: str, **options: Any) -> Dict[str, Any]:
|
|
260
|
+
"""Diagnose a training failure from raw logs.
|
|
261
|
+
|
|
262
|
+
Optional keyword args pass through to the API: ``jobName``,
|
|
263
|
+
``runtimeHours``, ``gpuRate``, ``topologyMap``, ``telemetry``,
|
|
264
|
+
``codeContext``, ``signatures``.
|
|
265
|
+
Returns the full API response dict (fix, meta incl. engineVersion,
|
|
266
|
+
forensicReport, advisory, ...; unknown fields are preserved).
|
|
267
|
+
"""
|
|
268
|
+
if not isinstance(logs, str) or len(logs) < MIN_LOG_CHARS:
|
|
269
|
+
raise DenpexError(f"logs must be at least {MIN_LOG_CHARS} characters", 400)
|
|
270
|
+
selected_logs, _truncated = select_evidence(logs, MAX_LOG_CHARS)
|
|
271
|
+
return self._request("POST", "/api/diagnose", {"logs": selected_logs, **options})
|
|
272
|
+
|
|
273
|
+
def refine(self, logs: str, operator_answers: list, **options: Any) -> Dict[str, Any]:
|
|
274
|
+
"""Submit observed answers with the original logs. Check selection is not a result."""
|
|
275
|
+
return self.diagnose(logs, operatorAnswers=operator_answers, **options)
|
|
276
|
+
|
|
277
|
+
def followup(self, logs: str, diagnosis_id: str, question: str,
|
|
278
|
+
conversation_id: Optional[str] = None) -> Dict[str, Any]:
|
|
279
|
+
"""Continue the same incident and preserve its structured questions."""
|
|
280
|
+
if not isinstance(logs, str) or len(logs) < MIN_LOG_CHARS or not diagnosis_id or not isinstance(question, str) or not question.strip():
|
|
281
|
+
raise DenpexError("Original logs, diagnosis_id and a non-empty question are required", 400)
|
|
282
|
+
selected_logs, _ = select_evidence(logs, MAX_LOG_CHARS)
|
|
283
|
+
body: Dict[str, Any] = {"logs": selected_logs, "diagnosisId": diagnosis_id,
|
|
284
|
+
"history": [{"role": "user", "content": question}]}
|
|
285
|
+
if conversation_id:
|
|
286
|
+
body["conversationId"] = conversation_id
|
|
287
|
+
return self._request("POST", "/api/diagnose/chat", body)
|
|
288
|
+
|
|
289
|
+
def feedback(self, diagnosis_id: str, outcome: str, note: Optional[str] = None) -> Dict[str, Any]:
|
|
290
|
+
"""Report whether a diagnosis worked ("worked" | "did_not_work");
|
|
291
|
+
this is a user-reported outcome, not independent root-cause verification."""
|
|
292
|
+
if not diagnosis_id:
|
|
293
|
+
raise DenpexError("diagnosis_id required", 400)
|
|
294
|
+
body: Dict[str, Any] = {"diagnosisId": diagnosis_id, "outcome": outcome}
|
|
295
|
+
if note:
|
|
296
|
+
body["note"] = note
|
|
297
|
+
return self._request("POST", "/api/diagnose/feedback", body)
|
|
298
|
+
|
|
299
|
+
def quota(self) -> Dict[str, Any]:
|
|
300
|
+
"""Remaining diagnosis headroom for this key's plan."""
|
|
301
|
+
return self._request("GET", "/api/diagnose/quota")
|
|
302
|
+
|
|
303
|
+
# ── plumbing ────────────────────────────────────────────────────────────
|
|
304
|
+
|
|
305
|
+
def _request(self, method: str, path: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
306
|
+
data = json.dumps(body).encode("utf-8") if body is not None else None
|
|
307
|
+
req = urllib.request.Request(
|
|
308
|
+
f"{self.base_url}{path}",
|
|
309
|
+
data=data,
|
|
310
|
+
method=method,
|
|
311
|
+
headers={
|
|
312
|
+
"X-Denpex-Key": self.api_key,
|
|
313
|
+
"Accept": "application/json",
|
|
314
|
+
**({"Content-Type": "application/json"} if data is not None else {}),
|
|
315
|
+
},
|
|
316
|
+
)
|
|
317
|
+
try:
|
|
318
|
+
with self._opener(req, timeout=self.timeout_s) as resp: # type: ignore[call-arg]
|
|
319
|
+
return json.loads(resp.read().decode("utf-8") or "{}")
|
|
320
|
+
except urllib.error.HTTPError as exc:
|
|
321
|
+
try:
|
|
322
|
+
payload = json.loads(exc.read().decode("utf-8"))
|
|
323
|
+
except Exception:
|
|
324
|
+
payload = {}
|
|
325
|
+
guidance = payload.get("guidance") if isinstance(payload.get("guidance"), str) else None
|
|
326
|
+
raise DenpexError(
|
|
327
|
+
str(guidance or payload.get("error") or f"Denpex API error (status {exc.code})"),
|
|
328
|
+
exc.code,
|
|
329
|
+
payload.get("code"),
|
|
330
|
+
guidance,
|
|
331
|
+
) from exc
|
|
332
|
+
except urllib.error.URLError as exc:
|
|
333
|
+
raise DenpexError(f"cannot reach Denpex API: {exc.reason}", 0) from exc
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: denpex-sdk
|
|
3
|
+
Version: 0.2.4
|
|
4
|
+
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
+
Author: Denpex
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Project-URL: Homepage, https://denpex.com
|
|
8
|
+
Project-URL: Documentation, https://denpex.com/docs/integrations
|
|
9
|
+
Project-URL: API Reference, https://denpex.com/docs/api
|
|
10
|
+
Project-URL: Python Agent & CLI, https://denpex.com/docs/agent
|
|
11
|
+
Project-URL: VS Code Extension, https://denpex.com/vscode
|
|
12
|
+
Project-URL: Open VSX Extension, https://denpex.com/openvsx
|
|
13
|
+
Project-URL: GPU Failure Encyclopedia, https://denpex.com/failures
|
|
14
|
+
Project-URL: Free GPU Debugging Tools, https://denpex.com/tools
|
|
15
|
+
Project-URL: Changelog, https://denpex.com/changelog
|
|
16
|
+
Project-URL: Repository, https://github.com/Denpex/gpu-failure-cheatsheet
|
|
17
|
+
Project-URL: Issue Tracker, https://denpex.com/contact
|
|
18
|
+
Keywords: denpex,gpu,cuda,nccl,pytorch,slurm,vllm,xid,mlops,diagnostics
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Intended Audience :: Developers
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: System :: Monitoring
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
Provides-Extra: torch
|
|
26
|
+
Requires-Dist: torch>=2.1; extra == "torch"
|
|
27
|
+
|
|
28
|
+
# denpex-sdk
|
|
29
|
+
|
|
30
|
+
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
31
|
+
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
32
|
+
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
33
|
+
|
|
34
|
+
## Cloud diagnosis client
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from denpex_sdk import DenpexClient
|
|
38
|
+
|
|
39
|
+
client = DenpexClient(api_key="dpx_...")
|
|
40
|
+
result = client.diagnose(
|
|
41
|
+
open("training.log").read(),
|
|
42
|
+
jobName="llama3-405b-pretrain",
|
|
43
|
+
telemetry={"jobId": "slurm-1842"},
|
|
44
|
+
)
|
|
45
|
+
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
46
|
+
print(result["fix"]["summary"])
|
|
47
|
+
print(result["fix"]["action"])
|
|
48
|
+
client.feedback(result["diagnosisId"], "worked")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
52
|
+
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
53
|
+
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
54
|
+
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
55
|
+
|
|
56
|
+
## SDC detection
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
60
|
+
|
|
61
|
+
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
62
|
+
weight_report = hook.inspect()
|
|
63
|
+
grad_report = hook.inspect_gradients()
|
|
64
|
+
opt_report = hook.inspect_optimizer(optimizer)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Checkpoint validation
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
71
|
+
|
|
72
|
+
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
73
|
+
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
74
|
+
handle.write(manifest_to_json(manifest))
|
|
75
|
+
|
|
76
|
+
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
77
|
+
expected = manifest_from_json(handle.read())
|
|
78
|
+
|
|
79
|
+
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
80
|
+
assert result.valid
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Official Denpex Ecosystem & Documentation
|
|
86
|
+
|
|
87
|
+
| Resource | Description | Link |
|
|
88
|
+
| :--- | :--- | :--- |
|
|
89
|
+
| **Denpex Platform** | Deterministic GPU and ML training failure diagnostics | [denpex.com](https://denpex.com) |
|
|
90
|
+
| **Documentation Hub** | Quickstart guides, architecture, and operational reference | [denpex.com/docs](https://denpex.com/docs) |
|
|
91
|
+
| **Python Agent & CLI** | On-host monitoring, evidence capture, and zero-egress CLI | [denpex.com/docs/agent](https://denpex.com/docs/agent) |
|
|
92
|
+
| **REST API Reference** | Programmatic `/api/diagnose` and preflight endpoints | [denpex.com/docs/api](https://denpex.com/docs/api) |
|
|
93
|
+
| **VS Code Extension** | Offline GPU crash diagnostics in VS Code | [denpex.com/vscode](https://denpex.com/vscode) |
|
|
94
|
+
| **Open VSX Extension** | Extension for Cursor, Windsurf, VSCodium, and Gitpod | [denpex.com/openvsx](https://denpex.com/openvsx) |
|
|
95
|
+
| **Failure Encyclopedia** | 500+ documented CUDA, NCCL, Xid, Slurm, and vLLM failures | [denpex.com/failures](https://denpex.com/failures) |
|
|
96
|
+
| **Free GPU Tools** | VRAM, NCCL, DeepSpeed ZeRO, and Slurm calculators | [denpex.com/tools](https://denpex.com/tools) |
|
|
97
|
+
| **NVIDIA Xid Reference** | Hardware vs software Xid fault routing table | [denpex.com/xid](https://denpex.com/xid) |
|
|
98
|
+
| **Release Changelog** | Platform, agent, SDK, and extension release notes | [denpex.com/changelog](https://denpex.com/changelog) |
|
|
99
|
+
|
|
@@ -22,13 +22,20 @@ There is no workflow change or new dependency, and they run under pytest too.
|
|
|
22
22
|
"""
|
|
23
23
|
from __future__ import annotations
|
|
24
24
|
|
|
25
|
+
import hashlib
|
|
25
26
|
import io
|
|
26
27
|
import json
|
|
27
28
|
import os
|
|
29
|
+
import tempfile
|
|
28
30
|
import unittest
|
|
29
31
|
import urllib.error
|
|
30
32
|
|
|
31
|
-
from denpex_sdk.client import
|
|
33
|
+
from denpex_sdk.client import (
|
|
34
|
+
DenpexClient,
|
|
35
|
+
DenpexError,
|
|
36
|
+
activate_trial,
|
|
37
|
+
request_trial,
|
|
38
|
+
)
|
|
32
39
|
|
|
33
40
|
LOG = "[E ProcessGroupNCCL.cpp:475] Watchdog caught collective operation timeout"
|
|
34
41
|
|
|
@@ -163,12 +170,20 @@ class ApiKeyTests(unittest.TestCase):
|
|
|
163
170
|
# Replaces pytest's monkeypatch. The env var is restored in tearDown so
|
|
164
171
|
# this cannot leak into another test and make it pass for the wrong reason.
|
|
165
172
|
self._saved = os.environ.pop("DENPEX_API_KEY", None)
|
|
173
|
+
self._saved_cred = os.environ.get("DENPEX_CREDENTIALS_PATH")
|
|
174
|
+
self._tmp_dir = tempfile.TemporaryDirectory()
|
|
175
|
+
os.environ["DENPEX_CREDENTIALS_PATH"] = os.path.join(self._tmp_dir.name, "missing-credentials.json")
|
|
166
176
|
|
|
167
177
|
def tearDown(self):
|
|
168
178
|
if self._saved is not None:
|
|
169
179
|
os.environ["DENPEX_API_KEY"] = self._saved
|
|
170
180
|
else:
|
|
171
181
|
os.environ.pop("DENPEX_API_KEY", None)
|
|
182
|
+
if self._saved_cred is not None:
|
|
183
|
+
os.environ["DENPEX_CREDENTIALS_PATH"] = self._saved_cred
|
|
184
|
+
else:
|
|
185
|
+
os.environ.pop("DENPEX_CREDENTIALS_PATH", None)
|
|
186
|
+
self._tmp_dir.cleanup()
|
|
172
187
|
|
|
173
188
|
def test_missing_api_key_raises(self):
|
|
174
189
|
with self.assertRaises(DenpexError):
|
|
@@ -183,6 +198,64 @@ class ApiKeyTests(unittest.TestCase):
|
|
|
183
198
|
c.quota()
|
|
184
199
|
self.assertEqual(captured[0].get_header("X-denpex-key"), "dpx_from_env")
|
|
185
200
|
|
|
201
|
+
def test_request_and_activate_trial_saves_credentials_for_client(self):
|
|
202
|
+
captured: list = []
|
|
203
|
+
responses = iter([
|
|
204
|
+
{"ok": True, "sent": True, "state": "pending_mailbox"},
|
|
205
|
+
{"status": "active", "plan": "team", "dailyCap": 50, "message": "Team trial active."},
|
|
206
|
+
])
|
|
207
|
+
|
|
208
|
+
def opener(req, timeout=None):
|
|
209
|
+
captured.append(req)
|
|
210
|
+
return _StubResponse(next(responses))
|
|
211
|
+
|
|
212
|
+
with self.assertRaises(DenpexError):
|
|
213
|
+
request_trial("alex@gmail.com", opener=opener)
|
|
214
|
+
self.assertEqual(captured, [])
|
|
215
|
+
|
|
216
|
+
req_res = request_trial("alex@acme-gpu.io", source="jupyterlab", opener=opener)
|
|
217
|
+
self.assertTrue(req_res["sent"])
|
|
218
|
+
|
|
219
|
+
cred_file = os.path.join(self._tmp_dir.name, "credentials.json")
|
|
220
|
+
os.environ["DENPEX_CREDENTIALS_PATH"] = cred_file
|
|
221
|
+
code = "DPXV-" + ("A1B2C3D4" * 4)
|
|
222
|
+
act_res = activate_trial("alex@acme-gpu.io", code, client="cli", opener=opener)
|
|
223
|
+
self.assertEqual(act_res["status"], "active")
|
|
224
|
+
self.assertTrue(act_res["api_key"].startswith("dpx_"))
|
|
225
|
+
self.assertTrue(os.path.isfile(cred_file))
|
|
226
|
+
|
|
227
|
+
sent_req = json.loads(captured[0].data.decode())
|
|
228
|
+
self.assertEqual(sent_req["email"], "alex@acme-gpu.io")
|
|
229
|
+
self.assertEqual(sent_req["source"], "jupyterlab")
|
|
230
|
+
|
|
231
|
+
sent_act = json.loads(captured[1].data.decode())
|
|
232
|
+
self.assertEqual(sent_act["keyHash"], hashlib.sha256(act_res["api_key"].encode()).hexdigest())
|
|
233
|
+
self.assertEqual(sent_act["client"], "cli")
|
|
234
|
+
|
|
235
|
+
loaded = DenpexClient(opener=_stub_opener({"ok": True}, captured))
|
|
236
|
+
loaded.quota()
|
|
237
|
+
self.assertEqual(captured[2].get_header("X-denpex-key"), act_res["api_key"])
|
|
238
|
+
|
|
186
239
|
|
|
187
240
|
if __name__ == "__main__":
|
|
188
241
|
unittest.main()
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class ClarificationFollowupTests(unittest.TestCase):
|
|
245
|
+
def test_followup_keeps_incident_and_questions(self):
|
|
246
|
+
captured = []
|
|
247
|
+
payload = {"conversationId": "thread-one", "reply": {"clarifyingChoices": [{"allowOther": True}]}}
|
|
248
|
+
client = DenpexClient(api_key="unit-test-only", opener=_stub_opener(payload, captured))
|
|
249
|
+
self.assertEqual(client.followup(LOG, "diagnosis-one", "First error was I/O", "thread-one"), payload)
|
|
250
|
+
body = json.loads(captured[0].data)
|
|
251
|
+
self.assertEqual(body["logs"], LOG)
|
|
252
|
+
self.assertEqual(body["diagnosisId"], "diagnosis-one")
|
|
253
|
+
self.assertEqual(body["conversationId"], "thread-one")
|
|
254
|
+
self.assertTrue(captured[0].full_url.endswith("/api/diagnose/chat"))
|
|
255
|
+
|
|
256
|
+
def test_refine_preserves_typed_answers(self):
|
|
257
|
+
captured = []
|
|
258
|
+
client = DenpexClient(api_key="unit-test-only", opener=_stub_opener({}, captured))
|
|
259
|
+
answers = [{"questionId": "scope", "optionId": "unsure", "answer": "Not sure"}]
|
|
260
|
+
client.refine(LOG, answers)
|
|
261
|
+
self.assertEqual(json.loads(captured[0].data)["operatorAnswers"], answers)
|
denpex_sdk-0.2.2/README.md
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
# denpex-sdk
|
|
2
|
-
|
|
3
|
-
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
4
|
-
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
5
|
-
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
6
|
-
|
|
7
|
-
## Cloud diagnosis client
|
|
8
|
-
|
|
9
|
-
```python
|
|
10
|
-
from denpex_sdk import DenpexClient
|
|
11
|
-
|
|
12
|
-
client = DenpexClient(api_key="dpx_...")
|
|
13
|
-
result = client.diagnose(
|
|
14
|
-
open("training.log").read(),
|
|
15
|
-
jobName="llama3-405b-pretrain",
|
|
16
|
-
telemetry={"jobId": "slurm-1842"},
|
|
17
|
-
)
|
|
18
|
-
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
19
|
-
print(result["fix"]["summary"])
|
|
20
|
-
print(result["fix"]["action"])
|
|
21
|
-
client.feedback(result["diagnosisId"], "worked")
|
|
22
|
-
```
|
|
23
|
-
|
|
24
|
-
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
25
|
-
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
26
|
-
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
27
|
-
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
28
|
-
|
|
29
|
-
## SDC detection
|
|
30
|
-
|
|
31
|
-
```python
|
|
32
|
-
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
33
|
-
|
|
34
|
-
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
35
|
-
weight_report = hook.inspect()
|
|
36
|
-
grad_report = hook.inspect_gradients()
|
|
37
|
-
opt_report = hook.inspect_optimizer(optimizer)
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
## Checkpoint validation
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
44
|
-
|
|
45
|
-
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
46
|
-
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
47
|
-
handle.write(manifest_to_json(manifest))
|
|
48
|
-
|
|
49
|
-
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
50
|
-
expected = manifest_from_json(handle.read())
|
|
51
|
-
|
|
52
|
-
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
53
|
-
assert result.valid
|
|
54
|
-
```
|
denpex_sdk-0.2.2/pyproject.toml
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=68", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "denpex-sdk"
|
|
7
|
-
version = "0.2.2"
|
|
8
|
-
description = "Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.8"
|
|
11
|
-
license = { text = "Proprietary" }
|
|
12
|
-
authors = [{ name = "Denpex" }]
|
|
13
|
-
|
|
14
|
-
[project.optional-dependencies]
|
|
15
|
-
torch = ["torch>=2.1"]
|
|
16
|
-
|
|
17
|
-
[tool.setuptools.packages.find]
|
|
18
|
-
where = ["src"]
|
|
19
|
-
|
|
20
|
-
[tool.setuptools.package-data]
|
|
21
|
-
denpex_sdk = ["tests/*.xml"]
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
"""Denpex API client for programmatic diagnosis over the public API.
|
|
2
|
-
|
|
3
|
-
Mirrors the TypeScript SDK (denpex-sdk-ts) against the same contract
|
|
4
|
-
(worker/api-worker-optimized.ts):
|
|
5
|
-
|
|
6
|
-
POST /api/diagnose {"logs": ..., "jobName": ...} -> diagnosis
|
|
7
|
-
POST /api/diagnose/feedback {"diagnosisId": ..., "outcome": ...}
|
|
8
|
-
GET /api/diagnose/quota
|
|
9
|
-
|
|
10
|
-
Auth: an API key from https://denpex.com/dashboard sent as ``X-Denpex-Key``.
|
|
11
|
-
Zero third-party dependencies (urllib only), matching the SDK's design rule.
|
|
12
|
-
|
|
13
|
-
from denpex_sdk import DenpexClient
|
|
14
|
-
|
|
15
|
-
client = DenpexClient() # reads DENPEX_API_KEY from the environment
|
|
16
|
-
result = client.diagnose(open("training.log").read())
|
|
17
|
-
print(result["meta"]["failureType"], result["meta"].get("engineVersion"))
|
|
18
|
-
client.feedback(result["diagnosisId"], "worked")
|
|
19
|
-
"""
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
import json
|
|
23
|
-
import os
|
|
24
|
-
import urllib.error
|
|
25
|
-
import urllib.request
|
|
26
|
-
from typing import Any, Callable, Dict, Optional
|
|
27
|
-
|
|
28
|
-
from .evidence import select_evidence
|
|
29
|
-
|
|
30
|
-
DEFAULT_BASE_URL = "https://api.denpex.com"
|
|
31
|
-
MIN_LOG_CHARS = 6
|
|
32
|
-
MAX_LOG_CHARS = 256_000
|
|
33
|
-
_DEFAULT_TIMEOUT_S = 320.0 # API deep-path ceiling is 290s; retain network buffer
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class DenpexError(RuntimeError):
|
|
37
|
-
"""API or validation failure. ``status`` is the HTTP status (0 = client-side),
|
|
38
|
-
``code`` the machine-readable API code when provided (e.g. AUTH_REQUIRED)."""
|
|
39
|
-
|
|
40
|
-
def __init__(
|
|
41
|
-
self,
|
|
42
|
-
message: str,
|
|
43
|
-
status: int = 0,
|
|
44
|
-
code: Optional[str] = None,
|
|
45
|
-
guidance: Optional[str] = None,
|
|
46
|
-
) -> None:
|
|
47
|
-
super().__init__(message)
|
|
48
|
-
self.status = status
|
|
49
|
-
self.code = code
|
|
50
|
-
self.guidance = guidance
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class DenpexClient:
|
|
54
|
-
"""Minimal, dependency-free client for the Denpex public API."""
|
|
55
|
-
|
|
56
|
-
def __init__(
|
|
57
|
-
self,
|
|
58
|
-
api_key: Optional[str] = None,
|
|
59
|
-
base_url: str = DEFAULT_BASE_URL,
|
|
60
|
-
timeout_s: float = _DEFAULT_TIMEOUT_S,
|
|
61
|
-
opener: Optional[Callable[..., Any]] = None,
|
|
62
|
-
) -> None:
|
|
63
|
-
self.api_key = api_key or os.environ.get("DENPEX_API_KEY", "")
|
|
64
|
-
if not self.api_key:
|
|
65
|
-
raise DenpexError(
|
|
66
|
-
"API key required: pass api_key= or set DENPEX_API_KEY "
|
|
67
|
-
"(create one at https://denpex.com/dashboard)"
|
|
68
|
-
)
|
|
69
|
-
self.base_url = base_url.rstrip("/")
|
|
70
|
-
self.timeout_s = timeout_s
|
|
71
|
-
# Injectable for tests; defaults to urllib. Signature: (Request, timeout=...) -> response
|
|
72
|
-
self._opener = opener or urllib.request.urlopen
|
|
73
|
-
|
|
74
|
-
# ── public surface ──────────────────────────────────────────────────────
|
|
75
|
-
|
|
76
|
-
def diagnose(self, logs: str, **options: Any) -> Dict[str, Any]:
|
|
77
|
-
"""Diagnose a training failure from raw logs.
|
|
78
|
-
|
|
79
|
-
Optional keyword args pass through to the API: ``jobName``,
|
|
80
|
-
``runtimeHours``, ``gpuRate``, ``topologyMap``, ``telemetry``,
|
|
81
|
-
``codeContext``, ``signatures``.
|
|
82
|
-
Returns the full API response dict (fix, meta incl. engineVersion,
|
|
83
|
-
forensicReport, advisory, ...; unknown fields are preserved).
|
|
84
|
-
"""
|
|
85
|
-
if not isinstance(logs, str) or len(logs) < MIN_LOG_CHARS:
|
|
86
|
-
raise DenpexError(f"logs must be at least {MIN_LOG_CHARS} characters", 400)
|
|
87
|
-
selected_logs, _truncated = select_evidence(logs, MAX_LOG_CHARS)
|
|
88
|
-
return self._request("POST", "/api/diagnose", {"logs": selected_logs, **options})
|
|
89
|
-
|
|
90
|
-
def feedback(self, diagnosis_id: str, outcome: str, note: Optional[str] = None) -> Dict[str, Any]:
|
|
91
|
-
"""Report whether a diagnosis worked ("worked" | "did_not_work");
|
|
92
|
-
this is a user-reported outcome, not independent root-cause verification."""
|
|
93
|
-
if not diagnosis_id:
|
|
94
|
-
raise DenpexError("diagnosis_id required", 400)
|
|
95
|
-
body: Dict[str, Any] = {"diagnosisId": diagnosis_id, "outcome": outcome}
|
|
96
|
-
if note:
|
|
97
|
-
body["note"] = note
|
|
98
|
-
return self._request("POST", "/api/diagnose/feedback", body)
|
|
99
|
-
|
|
100
|
-
def quota(self) -> Dict[str, Any]:
|
|
101
|
-
"""Remaining diagnosis headroom for this key's plan."""
|
|
102
|
-
return self._request("GET", "/api/diagnose/quota")
|
|
103
|
-
|
|
104
|
-
# ── plumbing ────────────────────────────────────────────────────────────
|
|
105
|
-
|
|
106
|
-
def _request(self, method: str, path: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
107
|
-
data = json.dumps(body).encode("utf-8") if body is not None else None
|
|
108
|
-
req = urllib.request.Request(
|
|
109
|
-
f"{self.base_url}{path}",
|
|
110
|
-
data=data,
|
|
111
|
-
method=method,
|
|
112
|
-
headers={
|
|
113
|
-
"X-Denpex-Key": self.api_key,
|
|
114
|
-
"Accept": "application/json",
|
|
115
|
-
**({"Content-Type": "application/json"} if data is not None else {}),
|
|
116
|
-
},
|
|
117
|
-
)
|
|
118
|
-
try:
|
|
119
|
-
with self._opener(req, timeout=self.timeout_s) as resp: # type: ignore[call-arg]
|
|
120
|
-
return json.loads(resp.read().decode("utf-8") or "{}")
|
|
121
|
-
except urllib.error.HTTPError as exc:
|
|
122
|
-
try:
|
|
123
|
-
payload = json.loads(exc.read().decode("utf-8"))
|
|
124
|
-
except Exception:
|
|
125
|
-
payload = {}
|
|
126
|
-
guidance = payload.get("guidance") if isinstance(payload.get("guidance"), str) else None
|
|
127
|
-
raise DenpexError(
|
|
128
|
-
str(guidance or payload.get("error") or f"Denpex API error (status {exc.code})"),
|
|
129
|
-
exc.code,
|
|
130
|
-
payload.get("code"),
|
|
131
|
-
guidance,
|
|
132
|
-
) from exc
|
|
133
|
-
except urllib.error.URLError as exc:
|
|
134
|
-
raise DenpexError(f"cannot reach Denpex API: {exc.reason}", 0) from exc
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: denpex-sdk
|
|
3
|
-
Version: 0.2.2
|
|
4
|
-
Summary: Denpex SDK: API client for GPU training-failure diagnosis, plus SDC detection and checkpoint validation.
|
|
5
|
-
Author: Denpex
|
|
6
|
-
License: Proprietary
|
|
7
|
-
Requires-Python: >=3.8
|
|
8
|
-
Description-Content-Type: text/markdown
|
|
9
|
-
Provides-Extra: torch
|
|
10
|
-
Requires-Dist: torch>=2.1; extra == "torch"
|
|
11
|
-
|
|
12
|
-
# denpex-sdk
|
|
13
|
-
|
|
14
|
-
Zero-third-party Python helpers for SDC detection (per-layer weight/gradient
|
|
15
|
-
norms, NaN/Inf checks, optimizer state validation) and asynchronous
|
|
16
|
-
checkpoint validation (manifest integrity + content checksums) for S3/Lustre.
|
|
17
|
-
|
|
18
|
-
## Cloud diagnosis client
|
|
19
|
-
|
|
20
|
-
```python
|
|
21
|
-
from denpex_sdk import DenpexClient
|
|
22
|
-
|
|
23
|
-
client = DenpexClient(api_key="dpx_...")
|
|
24
|
-
result = client.diagnose(
|
|
25
|
-
open("training.log").read(),
|
|
26
|
-
jobName="llama3-405b-pretrain",
|
|
27
|
-
telemetry={"jobId": "slurm-1842"},
|
|
28
|
-
)
|
|
29
|
-
print(result["meta"]["failureType"], result["meta"].get("confidence"))
|
|
30
|
-
print(result["fix"]["summary"])
|
|
31
|
-
print(result["fix"]["action"])
|
|
32
|
-
client.feedback(result["diagnosisId"], "worked")
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
The client reads `DENPEX_API_KEY` when `api_key` is omitted and calls the live
|
|
36
|
-
`/api/diagnose` contract. Its default 320-second timeout covers the API's
|
|
37
|
-
290-second deep-diagnosis window. Optional arguments such as `telemetry`,
|
|
38
|
-
`codeContext`, `topologyMap`, `runtimeHours`, and `gpuRate` pass through.
|
|
39
|
-
|
|
40
|
-
## SDC detection
|
|
41
|
-
|
|
42
|
-
```python
|
|
43
|
-
from denpex_sdk import SdcHook, install_sdc_hooks
|
|
44
|
-
|
|
45
|
-
hook = install_sdc_hooks(model, SdcHook(max_weight_norm=1e4, max_grad_norm=1e4))
|
|
46
|
-
weight_report = hook.inspect()
|
|
47
|
-
grad_report = hook.inspect_gradients()
|
|
48
|
-
opt_report = hook.inspect_optimizer(optimizer)
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
## Checkpoint validation
|
|
52
|
-
|
|
53
|
-
```python
|
|
54
|
-
from denpex_sdk import build_manifest, manifest_to_json, manifest_from_json, validate_checkpoint
|
|
55
|
-
|
|
56
|
-
manifest = build_manifest(["/checkpoints/step-1000/"], metadata={"job_id": "..."})
|
|
57
|
-
with open("/checkpoints/step-1000/manifest.json", "w") as handle:
|
|
58
|
-
handle.write(manifest_to_json(manifest))
|
|
59
|
-
|
|
60
|
-
with open("/checkpoints/step-1000/manifest.json") as handle:
|
|
61
|
-
expected = manifest_from_json(handle.read())
|
|
62
|
-
|
|
63
|
-
result = validate_checkpoint(["/checkpoints/step-1000/"], expected_manifest=expected)
|
|
64
|
-
assert result.valid
|
|
65
|
-
```
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|