silver-adapters 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- silver_adapters/__init__.py +31 -0
- silver_adapters/frameworks.py +72 -0
- silver_adapters/models.py +50 -0
- silver_adapters/notebooks.py +57 -0
- silver_adapters/protocols.py +54 -0
- silver_adapters/remote.py +57 -0
- silver_adapters-0.1.0.dist-info/METADATA +404 -0
- silver_adapters-0.1.0.dist-info/RECORD +11 -0
- silver_adapters-0.1.0.dist-info/WHEEL +5 -0
- silver_adapters-0.1.0.dist-info/licenses/LICENSE +12 -0
- silver_adapters-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from .frameworks import PyTorchBridge, TensorFlowBridge, python_framework
|
|
2
|
+
from .models import (
|
|
3
|
+
CommandBridge,
|
|
4
|
+
DataImportPlan,
|
|
5
|
+
NotebookCell,
|
|
6
|
+
NotebookDocument,
|
|
7
|
+
PythonFramework,
|
|
8
|
+
SilverJsonlEvent,
|
|
9
|
+
)
|
|
10
|
+
from .notebooks import notebook_document, silver_notebook_cells
|
|
11
|
+
from .protocols import decode_jsonl, encode_jsonl, kaggle_import, pandas_import
|
|
12
|
+
from .remote import RemoteTrainingClient
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"PythonFramework",
|
|
16
|
+
"CommandBridge",
|
|
17
|
+
"DataImportPlan",
|
|
18
|
+
"SilverJsonlEvent",
|
|
19
|
+
"NotebookCell",
|
|
20
|
+
"NotebookDocument",
|
|
21
|
+
"python_framework",
|
|
22
|
+
"pandas_import",
|
|
23
|
+
"kaggle_import",
|
|
24
|
+
"encode_jsonl",
|
|
25
|
+
"decode_jsonl",
|
|
26
|
+
"notebook_document",
|
|
27
|
+
"silver_notebook_cells",
|
|
28
|
+
"RemoteTrainingClient",
|
|
29
|
+
"PyTorchBridge",
|
|
30
|
+
"TensorFlowBridge",
|
|
31
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from typing import Any, Dict, List
|
|
2
|
+
|
|
3
|
+
from .models import CommandBridge, PythonFramework
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def python_framework(
|
|
7
|
+
framework: PythonFramework,
|
|
8
|
+
script: str,
|
|
9
|
+
args: List[str] = None,
|
|
10
|
+
environment: Dict[str, str] = None,
|
|
11
|
+
) -> CommandBridge:
|
|
12
|
+
if not script.strip():
|
|
13
|
+
raise ValueError("Python bridge script is required")
|
|
14
|
+
env = {"SILVER_PROTOCOL": "silver-jsonl-v1"}
|
|
15
|
+
if environment:
|
|
16
|
+
env.update(environment)
|
|
17
|
+
return CommandBridge(
|
|
18
|
+
framework=framework,
|
|
19
|
+
command="python",
|
|
20
|
+
args=[script] + (args or []),
|
|
21
|
+
protocol="silver-jsonl-v1",
|
|
22
|
+
environment=env,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PyTorchBridge:
|
|
27
|
+
@staticmethod
|
|
28
|
+
def from_checkpoint(checkpoint_path: str) -> Dict[str, Any]:
|
|
29
|
+
try:
|
|
30
|
+
import torch
|
|
31
|
+
|
|
32
|
+
return torch.load(checkpoint_path, map_location="cpu")
|
|
33
|
+
except ImportError as error:
|
|
34
|
+
raise ImportError("torch package is required for PyTorchBridge") from error
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def to_checkpoint(model: Any, optimizer: Any, epoch: int, path: str) -> None:
|
|
38
|
+
try:
|
|
39
|
+
import torch
|
|
40
|
+
|
|
41
|
+
checkpoint = {"epoch": epoch}
|
|
42
|
+
if hasattr(model, "state_dict"):
|
|
43
|
+
checkpoint["model_state_dict"] = model.state_dict()
|
|
44
|
+
if hasattr(optimizer, "state_dict"):
|
|
45
|
+
checkpoint["optimizer_state_dict"] = optimizer.state_dict()
|
|
46
|
+
torch.save(checkpoint, path)
|
|
47
|
+
except ImportError as error:
|
|
48
|
+
raise ImportError("torch package is required for PyTorchBridge") from error
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TensorFlowBridge:
|
|
52
|
+
@staticmethod
|
|
53
|
+
def from_checkpoint(checkpoint_path: str) -> Any:
|
|
54
|
+
try:
|
|
55
|
+
import tensorflow as tf
|
|
56
|
+
|
|
57
|
+
return tf.keras.models.load_model(checkpoint_path)
|
|
58
|
+
except ImportError as error:
|
|
59
|
+
raise ImportError(
|
|
60
|
+
"tensorflow package is required for TensorFlowBridge"
|
|
61
|
+
) from error
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def to_checkpoint(model: Any, path: str) -> None:
|
|
65
|
+
try:
|
|
66
|
+
import tensorflow as tf
|
|
67
|
+
|
|
68
|
+
model.save(path)
|
|
69
|
+
except ImportError as error:
|
|
70
|
+
raise ImportError(
|
|
71
|
+
"tensorflow package is required for TensorFlowBridge"
|
|
72
|
+
) from error
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import Any, Dict, List, Literal, Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class PythonFramework(Enum):
|
|
7
|
+
PYTORCH = "pytorch"
|
|
8
|
+
KERAS = "keras"
|
|
9
|
+
TENSORFLOW = "tensorflow"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class CommandBridge:
|
|
14
|
+
framework: PythonFramework
|
|
15
|
+
command: str
|
|
16
|
+
args: List[str]
|
|
17
|
+
protocol: str = "silver-jsonl-v1"
|
|
18
|
+
environment: Dict[str, str] = field(default_factory=dict)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DataImportPlan:
|
|
23
|
+
source: Literal["pandas", "kaggle"]
|
|
24
|
+
command: str
|
|
25
|
+
args: List[str]
|
|
26
|
+
output: str = "jsonl"
|
|
27
|
+
options: Dict[str, str] = field(default_factory=dict)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class SilverJsonlEvent:
|
|
32
|
+
kind: str
|
|
33
|
+
data: Dict[str, Any] = field(default_factory=dict)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class NotebookCell:
|
|
38
|
+
cell_type: Literal["markdown", "code"]
|
|
39
|
+
metadata: Dict[str, Any]
|
|
40
|
+
source: List[str]
|
|
41
|
+
outputs: Optional[List[Any]] = None
|
|
42
|
+
execution_count: Optional[int] = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class NotebookDocument:
|
|
47
|
+
cells: List[NotebookCell]
|
|
48
|
+
metadata: Dict[str, Any]
|
|
49
|
+
nbformat: int = 4
|
|
50
|
+
nbformat_minor: int = 5
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
|
|
3
|
+
from .models import NotebookCell, NotebookDocument
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def notebook_document(cells: List[NotebookCell]) -> NotebookDocument:
|
|
7
|
+
return NotebookDocument(
|
|
8
|
+
cells=[
|
|
9
|
+
NotebookCell(
|
|
10
|
+
cell_type=cell.cell_type,
|
|
11
|
+
metadata=dict(cell.metadata),
|
|
12
|
+
source=list(cell.source),
|
|
13
|
+
outputs=None if cell.outputs is None else list(cell.outputs),
|
|
14
|
+
execution_count=cell.execution_count,
|
|
15
|
+
)
|
|
16
|
+
for cell in cells
|
|
17
|
+
],
|
|
18
|
+
metadata={
|
|
19
|
+
"kernelspec": {
|
|
20
|
+
"display_name": "Python 3",
|
|
21
|
+
"language": "python",
|
|
22
|
+
"name": "python3",
|
|
23
|
+
},
|
|
24
|
+
"language_info": {"name": "python", "version": "3"},
|
|
25
|
+
},
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def silver_notebook_cells() -> List[NotebookCell]:
|
|
30
|
+
return [
|
|
31
|
+
NotebookCell(
|
|
32
|
+
cell_type="markdown",
|
|
33
|
+
metadata={"language": "markdown"},
|
|
34
|
+
source=[
|
|
35
|
+
"# Silver training bridge",
|
|
36
|
+
"",
|
|
37
|
+
"Inspect a dataset, launch a backend, and keep the run observable.",
|
|
38
|
+
],
|
|
39
|
+
),
|
|
40
|
+
NotebookCell(
|
|
41
|
+
cell_type="code",
|
|
42
|
+
metadata={"language": "python"},
|
|
43
|
+
source=[
|
|
44
|
+
"from silver_pandas_bridge import load",
|
|
45
|
+
"dataset = load('train.csv')",
|
|
46
|
+
"print(dataset.report())",
|
|
47
|
+
],
|
|
48
|
+
),
|
|
49
|
+
NotebookCell(
|
|
50
|
+
cell_type="code",
|
|
51
|
+
metadata={"language": "python"},
|
|
52
|
+
source=[
|
|
53
|
+
"# Emit silver-jsonl-v1 events from your training loop",
|
|
54
|
+
'print(\'{"kind": "epoch", "epoch": 1, "metrics": {"loss": 0.5}}\')',
|
|
55
|
+
],
|
|
56
|
+
),
|
|
57
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Dict, List
|
|
3
|
+
|
|
4
|
+
from .models import DataImportPlan, SilverJsonlEvent
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def pandas_import(path: str, options: Dict[str, str] = None) -> DataImportPlan:
|
|
8
|
+
if not path.strip():
|
|
9
|
+
raise ValueError("pandas source path is required")
|
|
10
|
+
return DataImportPlan(
|
|
11
|
+
source="pandas",
|
|
12
|
+
command="python",
|
|
13
|
+
args=["-m", "silver_pandas_bridge", path],
|
|
14
|
+
output="jsonl",
|
|
15
|
+
options=options or {},
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def kaggle_import(dataset: str, options: Dict[str, str] = None) -> DataImportPlan:
|
|
20
|
+
if not dataset.strip():
|
|
21
|
+
raise ValueError("Kaggle dataset identifier is required")
|
|
22
|
+
return DataImportPlan(
|
|
23
|
+
source="kaggle",
|
|
24
|
+
command="kaggle",
|
|
25
|
+
args=["datasets", "download", "-d", dataset, "--unzip"],
|
|
26
|
+
output="jsonl",
|
|
27
|
+
options=options or {},
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def encode_jsonl(events: List[SilverJsonlEvent]) -> str:
|
|
32
|
+
return (
|
|
33
|
+
"\n".join(json.dumps({"kind": event.kind, **event.data}) for event in events)
|
|
34
|
+
+ "\n"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def decode_jsonl(text: str) -> List[SilverJsonlEvent]:
|
|
39
|
+
events = []
|
|
40
|
+
for line_number, line in enumerate(text.strip().splitlines(), 1):
|
|
41
|
+
line = line.strip()
|
|
42
|
+
if not line:
|
|
43
|
+
continue
|
|
44
|
+
try:
|
|
45
|
+
data = json.loads(line)
|
|
46
|
+
if not isinstance(data, dict) or "kind" not in data:
|
|
47
|
+
raise ValueError(
|
|
48
|
+
f"Line {line_number}: must be an object with 'kind' field"
|
|
49
|
+
)
|
|
50
|
+
kind = data.pop("kind")
|
|
51
|
+
events.append(SilverJsonlEvent(kind=kind, data=data))
|
|
52
|
+
except json.JSONDecodeError as error:
|
|
53
|
+
raise ValueError(f"Line {line_number}: invalid JSON - {error}") from error
|
|
54
|
+
return events
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import Any, Dict, List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class RemoteTrainingClient:
|
|
6
|
+
def __init__(
|
|
7
|
+
self, base_url: str, headers: Dict[str, str] = None, session: Any = None
|
|
8
|
+
):
|
|
9
|
+
if not base_url.strip():
|
|
10
|
+
raise ValueError("Remote training base URL is required")
|
|
11
|
+
self.base_url = base_url.rstrip("/")
|
|
12
|
+
self.headers = headers or {}
|
|
13
|
+
self._session = session
|
|
14
|
+
try:
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
self._requests = requests
|
|
18
|
+
except ImportError as error:
|
|
19
|
+
raise ImportError(
|
|
20
|
+
"requests package is required for RemoteTrainingClient"
|
|
21
|
+
) from error
|
|
22
|
+
|
|
23
|
+
def _get_session(self):
|
|
24
|
+
if self._session:
|
|
25
|
+
return self._session
|
|
26
|
+
return self._requests.Session()
|
|
27
|
+
|
|
28
|
+
async def start(self, payload: Any) -> Dict[str, Any]:
|
|
29
|
+
return await self._send("/runs", "POST", payload)
|
|
30
|
+
|
|
31
|
+
async def events(self, run_id: str) -> List[Dict[str, Any]]:
|
|
32
|
+
return await self._send(f"/runs/{run_id}/events", "GET")
|
|
33
|
+
|
|
34
|
+
async def stop(
|
|
35
|
+
self, run_id: str, reason: str = "stopped by user"
|
|
36
|
+
) -> Dict[str, Any]:
|
|
37
|
+
return await self._send(f"/runs/{run_id}/stop", "POST", {"reason": reason})
|
|
38
|
+
|
|
39
|
+
async def checkpoint(self, run_id: str) -> Any:
|
|
40
|
+
return await self._send(f"/runs/{run_id}/checkpoint", "GET")
|
|
41
|
+
|
|
42
|
+
async def _send(self, path: str, method: str, body: Any = None) -> Any:
|
|
43
|
+
def sync_request():
|
|
44
|
+
session = self._get_session()
|
|
45
|
+
response = session.request(
|
|
46
|
+
method,
|
|
47
|
+
f"{self.base_url}{path}",
|
|
48
|
+
headers={"content-type": "application/json", **self.headers},
|
|
49
|
+
json=body if body is not None else None,
|
|
50
|
+
)
|
|
51
|
+
if not response.ok:
|
|
52
|
+
raise RuntimeError(
|
|
53
|
+
f"Remote training request failed: {response.status_code} {response.text}"
|
|
54
|
+
)
|
|
55
|
+
return response.json()
|
|
56
|
+
|
|
57
|
+
return await asyncio.to_thread(sync_request)
|
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: silver-adapters
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Backend, dataset, notebook, and remote-training bridge protocols for Silver.
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/adfgdartec/silver-adapters
|
|
7
|
+
Project-URL: Repository, https://github.com/adfgdartec/silver-adapters
|
|
8
|
+
Project-URL: Issues, https://github.com/adfgdartec/silver-adapters/issues
|
|
9
|
+
Keywords: pytorch,keras,tensorflow,pandas,kaggle,jupyter,machine-learning,python
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: pytorch
|
|
24
|
+
Requires-Dist: torch>=1.9.0; extra == "pytorch"
|
|
25
|
+
Provides-Extra: tensorflow
|
|
26
|
+
Requires-Dist: tensorflow>=2.6.0; extra == "tensorflow"
|
|
27
|
+
Provides-Extra: keras
|
|
28
|
+
Requires-Dist: tensorflow>=2.6.0; extra == "keras"
|
|
29
|
+
Provides-Extra: kaggle
|
|
30
|
+
Requires-Dist: kaggle>=1.5.0; extra == "kaggle"
|
|
31
|
+
Provides-Extra: jupyter
|
|
32
|
+
Requires-Dist: jupyter>=1.0.0; extra == "jupyter"
|
|
33
|
+
Requires-Dist: nbformat>=5.0.0; extra == "jupyter"
|
|
34
|
+
Provides-Extra: pandas
|
|
35
|
+
Requires-Dist: pandas>=1.0.0; extra == "pandas"
|
|
36
|
+
Provides-Extra: remote
|
|
37
|
+
Requires-Dist: requests>=2.25.0; extra == "remote"
|
|
38
|
+
Provides-Extra: all
|
|
39
|
+
Requires-Dist: torch>=1.9.0; extra == "all"
|
|
40
|
+
Requires-Dist: tensorflow>=2.6.0; extra == "all"
|
|
41
|
+
Requires-Dist: kaggle>=1.5.0; extra == "all"
|
|
42
|
+
Requires-Dist: jupyter>=1.0.0; extra == "all"
|
|
43
|
+
Requires-Dist: nbformat>=5.0.0; extra == "all"
|
|
44
|
+
Requires-Dist: pandas>=1.0.0; extra == "all"
|
|
45
|
+
Requires-Dist: requests>=2.25.0; extra == "all"
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
48
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
49
|
+
Requires-Dist: flake8>=6.0.0; extra == "dev"
|
|
50
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
51
|
+
Requires-Dist: build>=0.10.0; extra == "dev"
|
|
52
|
+
Requires-Dist: twine>=4.0.0; extra == "dev"
|
|
53
|
+
Dynamic: license-file
|
|
54
|
+
|
|
55
|
+
# silver-adapters
|
|
56
|
+
|
|
57
|
+
[](https://www.python.org/downloads/)
|
|
58
|
+
[](LICENSE)
|
|
59
|
+
[](tests/)
|
|
60
|
+
[](https://flake8.pycqa.org/)
|
|
61
|
+
|
|
62
|
+
Backend, dataset, notebook, and remote-training bridge protocols for Silver. A Python package designed for ML researchers who need seamless integration between different ML frameworks and tools.
|
|
63
|
+
|
|
64
|
+
The core package has no mandatory ML framework, pandas, requests, or Jupyter
|
|
65
|
+
dependency. Install only the extras you need:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install silver-adapters
|
|
69
|
+
pip install 'silver-adapters[pytorch]'
|
|
70
|
+
pip install 'silver-adapters[remote]'
|
|
71
|
+
pip install 'silver-adapters[pandas]'
|
|
72
|
+
pip install 'silver-adapters[jupyter]'
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Installation
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
# Core package
|
|
79
|
+
pip install silver-adapters
|
|
80
|
+
|
|
81
|
+
# With specific framework support
|
|
82
|
+
pip install silver-adapters[pytorch]
|
|
83
|
+
pip install silver-adapters[tensorflow]
|
|
84
|
+
pip install silver-adapters[kaggle]
|
|
85
|
+
pip install silver-adapters[jupyter]
|
|
86
|
+
|
|
87
|
+
# With all optional dependencies
|
|
88
|
+
pip install silver-adapters[all]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Quick Start
|
|
92
|
+
|
|
93
|
+
### Framework Bridges
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from silver_adapters import python_framework, PyTorchBridge, TensorFlowBridge
|
|
97
|
+
|
|
98
|
+
# Create a PyTorch training bridge
|
|
99
|
+
bridge = python_framework(
|
|
100
|
+
framework="pytorch",
|
|
101
|
+
script="train.py",
|
|
102
|
+
args=["--config", "config.yaml"]
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# PyTorch checkpoint utilities
|
|
106
|
+
checkpoint = PyTorchBridge.from_checkpoint("model.pt")
|
|
107
|
+
PyTorchBridge.to_checkpoint(model, optimizer, epoch=10, path="model.pt")
|
|
108
|
+
|
|
109
|
+
# TensorFlow/Keras checkpoint utilities
|
|
110
|
+
model = TensorFlowBridge.from_checkpoint("model.h5")
|
|
111
|
+
TensorFlowBridge.to_checkpoint(model, "model.h5")
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Data Import
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from silver_adapters import pandas_import, kaggle_import
|
|
118
|
+
|
|
119
|
+
# Import from pandas
|
|
120
|
+
plan = pandas_import("data/train.csv")
|
|
121
|
+
|
|
122
|
+
# Import from Kaggle
|
|
123
|
+
plan = kaggle_import("competiton/dataset-name")
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Remote Training
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
import asyncio
|
|
130
|
+
from silver_adapters import RemoteTrainingClient
|
|
131
|
+
|
|
132
|
+
async def main():
|
|
133
|
+
client = RemoteTrainingClient("https://api.example.com")
|
|
134
|
+
|
|
135
|
+
# Start training
|
|
136
|
+
run = await client.start({"config": {...}})
|
|
137
|
+
|
|
138
|
+
# Get events
|
|
139
|
+
events = await client.events(run["run_id"])
|
|
140
|
+
|
|
141
|
+
# Stop training
|
|
142
|
+
await client.stop(run["run_id"], "completed successfully")
|
|
143
|
+
|
|
144
|
+
asyncio.run(main())
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### JSONL Protocol
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from silver_adapters import encode_jsonl, decode_jsonl, SilverJsonlEvent
|
|
151
|
+
|
|
152
|
+
# Encode events
|
|
153
|
+
events = [
|
|
154
|
+
SilverJsonlEvent(kind="epoch", data={"epoch": 1, "loss": 0.5}),
|
|
155
|
+
SilverJsonlEvent(kind="metrics", data={"accuracy": 0.9})
|
|
156
|
+
]
|
|
157
|
+
jsonl = encode_jsonl(events)
|
|
158
|
+
|
|
159
|
+
# Decode events
|
|
160
|
+
decoded = decode_jsonl(jsonl)
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## Features
|
|
164
|
+
|
|
165
|
+
- **Framework Bridges**: Seamless integration with PyTorch, TensorFlow, and Keras
|
|
166
|
+
- **Data Import Plans**: Structured data import from pandas and Kaggle
|
|
167
|
+
- **Remote Training**: Async HTTP client for distributed training
|
|
168
|
+
- **JSONL Protocol**: Event streaming protocol for training observability
|
|
169
|
+
- **Jupyter Integration**: Notebook utilities and template generation
|
|
170
|
+
- **Checkpoint Management**: Unified checkpoint handling across frameworks
|
|
171
|
+
- **Type Safety**: Full type hints for better IDE support and fewer bugs
|
|
172
|
+
|
|
173
|
+
## Use Cases
|
|
174
|
+
|
|
175
|
+
### Multi-Framework Training
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from silver_adapters import PyTorchBridge, TensorFlowBridge
|
|
179
|
+
import torch
|
|
180
|
+
import tensorflow as tf
|
|
181
|
+
|
|
182
|
+
# Train in PyTorch
|
|
183
|
+
pytorch_model = torch.nn.Linear(10, 2)
|
|
184
|
+
optimizer = torch.optim.Adam(pytorch_model.parameters())
|
|
185
|
+
|
|
186
|
+
# Save PyTorch checkpoint
|
|
187
|
+
PyTorchBridge.to_checkpoint(pytorch_model, optimizer, epoch=10, path="pytorch_model.pt")
|
|
188
|
+
|
|
189
|
+
# Load in TensorFlow for inference
|
|
190
|
+
tf_model = tf.keras.Sequential([tf.keras.layers.Dense(2, input_shape=(10,))])
|
|
191
|
+
# Convert weights (framework-specific conversion needed)
|
|
192
|
+
TensorFlowBridge.to_checkpoint(tf_model, "tf_model.h5")
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Distributed Training Setup
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
import asyncio
|
|
199
|
+
from silver_adapters import RemoteTrainingClient
|
|
200
|
+
|
|
201
|
+
async def run_distributed_training():
|
|
202
|
+
# Connect to remote training server
|
|
203
|
+
client = RemoteTrainingClient("https://training-server.example.com")
|
|
204
|
+
|
|
205
|
+
# Start training on remote GPU cluster
|
|
206
|
+
config = {
|
|
207
|
+
"model": "resnet50",
|
|
208
|
+
"dataset": "imagenet",
|
|
209
|
+
"batch_size": 32,
|
|
210
|
+
"epochs": 100
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
run = await client.start(config)
|
|
214
|
+
print(f"Started training run: {run['run_id']}")
|
|
215
|
+
|
|
216
|
+
# Monitor training progress
|
|
217
|
+
while True:
|
|
218
|
+
events = await client.events(run['run_id'])
|
|
219
|
+
latest_events = events[-5:] # Get last 5 events
|
|
220
|
+
|
|
221
|
+
for event in latest_events:
|
|
222
|
+
if event['kind'] == 'epoch':
|
|
223
|
+
print(f"Epoch {event['epoch']}: loss={event.get('loss', 'N/A')}")
|
|
224
|
+
elif event['kind'] == 'completed':
|
|
225
|
+
print("Training completed!")
|
|
226
|
+
return
|
|
227
|
+
|
|
228
|
+
await asyncio.sleep(10) # Check every 10 seconds
|
|
229
|
+
|
|
230
|
+
asyncio.run(run_distributed_training())
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
### Data Pipeline Integration
|
|
234
|
+
|
|
235
|
+
```python
|
|
236
|
+
from silver_adapters import pandas_import, kaggle_import, encode_jsonl
|
|
237
|
+
import pandas as pd
|
|
238
|
+
|
|
239
|
+
# Create data import plans
|
|
240
|
+
csv_plan = pandas_import("data/train.csv", {"sep": ",", "encoding": "utf-8"})
|
|
241
|
+
kaggle_plan = kaggle_import("competiton/titanic", {"unzip": True})
|
|
242
|
+
|
|
243
|
+
# Use plans in your data pipeline
|
|
244
|
+
def execute_import_plan(plan):
|
|
245
|
+
"""Execute a data import plan"""
|
|
246
|
+
if plan.source == "pandas":
|
|
247
|
+
df = pd.read_csv(plan.args[1], **plan.options)
|
|
248
|
+
return df
|
|
249
|
+
elif plan.source == "kaggle":
|
|
250
|
+
# Execute Kaggle download command
|
|
251
|
+
import subprocess
|
|
252
|
+
subprocess.run([plan.command] + plan.args)
|
|
253
|
+
return pd.read_csv("downloaded_file.csv")
|
|
254
|
+
|
|
255
|
+
# Create training events from pandas DataFrame
|
|
256
|
+
def create_training_events(df):
|
|
257
|
+
from silver_adapters import SilverJsonlEvent
|
|
258
|
+
events = []
|
|
259
|
+
|
|
260
|
+
for epoch in range(10):
|
|
261
|
+
# Simulate training metrics
|
|
262
|
+
events.append(SilverJsonlEvent(
|
|
263
|
+
kind="epoch",
|
|
264
|
+
data={"epoch": epoch, "loss": 0.5 - epoch * 0.05}
|
|
265
|
+
))
|
|
266
|
+
|
|
267
|
+
return encode_jsonl(events)
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
### Jupyter Notebook Integration
|
|
271
|
+
|
|
272
|
+
```python
|
|
273
|
+
from silver_adapters import notebook_document, silver_notebook_cells
|
|
274
|
+
|
|
275
|
+
# Create a Silver training notebook
|
|
276
|
+
cells = silver_notebook_cells()
|
|
277
|
+
notebook = notebook_document(cells)
|
|
278
|
+
|
|
279
|
+
# Save as Jupyter notebook
|
|
280
|
+
import json
|
|
281
|
+
with open("silver_training.ipynb", "w") as f:
|
|
282
|
+
json.dump({
|
|
283
|
+
"cells": [
|
|
284
|
+
{
|
|
285
|
+
"cell_type": cell.cell_type,
|
|
286
|
+
"metadata": cell.metadata,
|
|
287
|
+
"source": cell.source,
|
|
288
|
+
"outputs": cell.outputs or [],
|
|
289
|
+
"execution_count": cell.execution_count
|
|
290
|
+
} for cell in notebook.cells
|
|
291
|
+
],
|
|
292
|
+
"metadata": notebook.metadata,
|
|
293
|
+
"nbformat": notebook.nbformat,
|
|
294
|
+
"nbformat_minor": notebook.nbformat_minor
|
|
295
|
+
}, f, indent=2)
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
## Advanced Usage
|
|
299
|
+
|
|
300
|
+
### Custom Training Backend
|
|
301
|
+
|
|
302
|
+
```python
|
|
303
|
+
from silver_adapters import python_framework, PythonFramework
|
|
304
|
+
import subprocess
|
|
305
|
+
|
|
306
|
+
class CustomTrainingBackend:
|
|
307
|
+
def __init__(self, framework, script_path):
|
|
308
|
+
self.bridge = python_framework(
|
|
309
|
+
framework=framework,
|
|
310
|
+
script=script_path,
|
|
311
|
+
environment={"CUDA_VISIBLE_DEVICES": "0"}
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
def launch_training(self, args):
|
|
315
|
+
"""Launch training with given arguments"""
|
|
316
|
+
cmd = [self.bridge.command] + self.bridge.args + args
|
|
317
|
+
env = {**self.bridge.environment}
|
|
318
|
+
|
|
319
|
+
process = subprocess.Popen(
|
|
320
|
+
cmd,
|
|
321
|
+
env={**subprocess.os.environ, **env}
|
|
322
|
+
)
|
|
323
|
+
return process
|
|
324
|
+
|
|
325
|
+
# Usage
|
|
326
|
+
backend = CustomTrainingBackend(
|
|
327
|
+
PythonFramework.PYTORCH,
|
|
328
|
+
"train.py"
|
|
329
|
+
)
|
|
330
|
+
process = backend.launch_training(["--epochs", "100", "--batch-size", "32"])
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
### Event Streaming
|
|
334
|
+
|
|
335
|
+
```python
|
|
336
|
+
from silver_adapters import encode_jsonl, decode_jsonl, SilverJsonlEvent
|
|
337
|
+
import asyncio
|
|
338
|
+
|
|
339
|
+
async def stream_training_events(writer):
|
|
340
|
+
"""Stream training events to a writer"""
|
|
341
|
+
for epoch in range(10):
|
|
342
|
+
event = SilverJsonlEvent(
|
|
343
|
+
kind="epoch",
|
|
344
|
+
data={"epoch": epoch, "loss": 0.5 - epoch * 0.05}
|
|
345
|
+
)
|
|
346
|
+
writer.write(encode_jsonl([event]))
|
|
347
|
+
await asyncio.sleep(0.1)
|
|
348
|
+
|
|
349
|
+
async def consume_training_events(reader):
|
|
350
|
+
"""Consume training events from a reader"""
|
|
351
|
+
buffer = ""
|
|
352
|
+
async for chunk in reader:
|
|
353
|
+
buffer += chunk
|
|
354
|
+
events = decode_jsonl(buffer)
|
|
355
|
+
for event in events:
|
|
356
|
+
print(f"Received: {event.kind} - {event.data}")
|
|
357
|
+
buffer = ""
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
## Requirements
|
|
361
|
+
|
|
362
|
+
- Python 3.8+
|
|
363
|
+
- pandas>=1.0.0
|
|
364
|
+
- requests>=2.25.0
|
|
365
|
+
|
|
366
|
+
### Optional Dependencies
|
|
367
|
+
- torch>=1.9.0 (for PyTorch support)
|
|
368
|
+
- tensorflow>=2.6.0 (for TensorFlow/Keras support)
|
|
369
|
+
- kaggle>=1.5.0 (for Kaggle integration)
|
|
370
|
+
- jupyter>=1.0.0 (for notebook utilities)
|
|
371
|
+
|
|
372
|
+
## Development
|
|
373
|
+
|
|
374
|
+
```bash
|
|
375
|
+
# Install development dependencies
|
|
376
|
+
pip install -e ".[dev]"
|
|
377
|
+
|
|
378
|
+
# Install all optional dependencies for testing
|
|
379
|
+
pip install -e ".[all]"
|
|
380
|
+
|
|
381
|
+
# Run tests
|
|
382
|
+
pytest
|
|
383
|
+
|
|
384
|
+
# Run tests with coverage
|
|
385
|
+
pytest --cov=silver_adapters --cov-report=html
|
|
386
|
+
|
|
387
|
+
# Run linting
|
|
388
|
+
flake8 src/ tests/
|
|
389
|
+
mypy src/
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
## Contributing
|
|
393
|
+
|
|
394
|
+
Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
395
|
+
|
|
396
|
+
## License
|
|
397
|
+
|
|
398
|
+
Apache-2.0 - see [LICENSE](LICENSE) file for details.
|
|
399
|
+
|
|
400
|
+
## Related Packages
|
|
401
|
+
|
|
402
|
+
- [silver-data](https://github.com/adfgdartec/silver-data) - Dataset handling
|
|
403
|
+
- [silver-run](https://github.com/adfgdartec/silver-run) - Training lifecycle
|
|
404
|
+
- [silver-diagnostics](https://github.com/adfgdartec/silver-diagnostics) - ML diagnostics
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
silver_adapters/__init__.py,sha256=7ydaH2U1vsgo4MWOc31Eu4q3-LcFqV5Li4Y7Ee5AhzE,787
|
|
2
|
+
silver_adapters/frameworks.py,sha256=ktY6LMONDoaHg81X6SQAKAIMLv_-bDXX_gGhc4XgT4A,2235
|
|
3
|
+
silver_adapters/models.py,sha256=E5aoTLALy9bYMN_Tosum1BNEX0UDBwC1JZEgrlH2-9I,1146
|
|
4
|
+
silver_adapters/notebooks.py,sha256=n0g01ktm62uQPALC8CpxN3AraeZIouin0T1c3HHKIXM,1762
|
|
5
|
+
silver_adapters/protocols.py,sha256=eAsIOLTM-rPoFZw0xJhsXecqeG1LPZNJ2pvaZJkEauM,1725
|
|
6
|
+
silver_adapters/remote.py,sha256=rsor9oRReihFqyixi5AplQMrzQnN-7XLZ6WO2MHcqkw,2006
|
|
7
|
+
silver_adapters-0.1.0.dist-info/licenses/LICENSE,sha256=9uEgZddcCAZ2jYIKny4rlbkQMZFhKRvSGbxctAg4wjI,517
|
|
8
|
+
silver_adapters-0.1.0.dist-info/METADATA,sha256=wfJSnMajcws3z6S4DVdD9rFOPA02J-TX7mkHydOeHaY,11893
|
|
9
|
+
silver_adapters-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
10
|
+
silver_adapters-0.1.0.dist-info/top_level.txt,sha256=zUC4f5RuALkGGwse0bqLsDX4J9NmhfKbqHXvr6VfOiQ,16
|
|
11
|
+
silver_adapters-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
|
|
4
|
+
Copyright 2026 Silver Contributors
|
|
5
|
+
|
|
6
|
+
Licensed under the Apache License, Version 2.0. You may obtain a copy of the
|
|
7
|
+
License at https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
10
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
11
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
12
|
+
specific language governing permissions and limitations under the License.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
silver_adapters
|