sharedots 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sharedots/__init__.py ADDED
@@ -0,0 +1,30 @@
1
+ """Sharedots Python SDK — Universal Distributed Compute & Remote Execution."""
2
+
3
+ from .client import Client, connect
4
+ from .exceptions import (
5
+ ArtifactError,
6
+ AuthenticationError,
7
+ JobTimeoutError,
8
+ RemoteExecutionError,
9
+ SharedotsError,
10
+ )
11
+ from .future import SharedotsFuture, gather
12
+ from .remote import RemoteFunction, remote
13
+ from .room import Room
14
+
15
+ __version__ = "0.1.0"
16
+
17
+ __all__ = [
18
+ "Client",
19
+ "connect",
20
+ "Room",
21
+ "remote",
22
+ "RemoteFunction",
23
+ "SharedotsFuture",
24
+ "gather",
25
+ "SharedotsError",
26
+ "RemoteExecutionError",
27
+ "AuthenticationError",
28
+ "JobTimeoutError",
29
+ "ArtifactError",
30
+ ]
sharedots/client.py ADDED
@@ -0,0 +1,155 @@
1
+ """Top-level Sharedots Client for managing rooms and remote compute decorators."""
2
+
3
+ import json
4
+ from typing import Any, Callable, Dict, Optional, Union
5
+ import urllib.error
6
+ import urllib.parse
7
+ import urllib.request
8
+
9
+ from .exceptions import AuthenticationError, SharedotsError
10
+ from .remote import RemoteFunction
11
+ from .room import Room
12
+
13
+
14
+ class Client:
15
+ """Entry point for connecting to a Sharedots coordinator cluster."""
16
+
17
+ def __init__(
18
+ self,
19
+ coordinator: str = "http://127.0.0.1:4000",
20
+ room_id: Optional[str] = None,
21
+ token: Optional[str] = None,
22
+ ):
23
+ self.coordinator_url = coordinator.rstrip("/")
24
+ self.token = token
25
+ self.active_room: Optional[Room] = None
26
+
27
+ if room_id and token:
28
+ self.active_room = Room(self.coordinator_url, room_id, token)
29
+
30
+ def _check_render_notice(self, reason: Any) -> str:
31
+ if "onrender.com" in self.coordinator_url:
32
+ return (
33
+ f"\nšŸ’” [Render Free Tier Notice] The coordinator at {self.coordinator_url} appears to be sleeping.\n"
34
+ " Render's free tier spins down after 15 min of inactivity. Please give it ~30-50 seconds to wake up and retry!"
35
+ )
36
+ return ""
37
+
38
+ def health(self) -> Dict[str, Any]:
39
+ """Queries coordinator health status and protocol version."""
40
+ url = f"{self.coordinator_url}/api/health"
41
+ req = urllib.request.Request(url, method="GET")
42
+ try:
43
+ with urllib.request.urlopen(req) as resp:
44
+ return json.loads(resp.read().decode("utf-8"))
45
+ except urllib.error.URLError as e:
46
+ notice = self._check_render_notice(e.reason)
47
+ raise SharedotsError(f"Failed to reach coordinator at {self.coordinator_url}: {e.reason}{notice}")
48
+
49
+ def overview(self) -> Dict[str, Any]:
50
+ """Queries overall cluster telemetry (total workers, cores, memory)."""
51
+ url = f"{self.coordinator_url}/api/overview"
52
+ req = urllib.request.Request(url, method="GET")
53
+ try:
54
+ with urllib.request.urlopen(req) as resp:
55
+ return json.loads(resp.read().decode("utf-8"))
56
+ except urllib.error.URLError as e:
57
+ notice = self._check_render_notice(e.reason)
58
+ raise SharedotsError(f"Failed to reach coordinator at {self.coordinator_url}: {e.reason}{notice}")
59
+
60
+ def create_room(self, name: Optional[str] = None) -> Room:
61
+ """Creates a new authenticated room on the coordinator."""
62
+ url = f"{self.coordinator_url}/api/rooms"
63
+ payload = json.dumps({"name": name or "mesh-room"}).encode("utf-8")
64
+ headers = {"content-type": "application/json"}
65
+ req = urllib.request.Request(url, data=payload, headers=headers, method="POST")
66
+
67
+ try:
68
+ with urllib.request.urlopen(req) as resp:
69
+ data = json.loads(resp.read().decode("utf-8"))
70
+ except urllib.error.URLError as e:
71
+ raise SharedotsError(f"Failed to reach coordinator at {self.coordinator_url}: {e.reason}")
72
+
73
+ room = Room(
74
+ coordinator_url=self.coordinator_url,
75
+ room_id=data["roomId"],
76
+ token=data["secret"],
77
+ )
78
+ self.active_room = room
79
+ if not self.token:
80
+ self.token = data["secret"]
81
+ return room
82
+
83
+ def connect(self, room_id: str, token: Optional[str] = None) -> Room:
84
+ """Connects to an existing room using its room ID and authentication token."""
85
+ effective_token = token or self.token
86
+ if not effective_token:
87
+ raise AuthenticationError("A room token (secret) is required to connect to a room")
88
+
89
+ # Verify room exists on coordinator
90
+ url = f"{self.coordinator_url}/api/rooms/{urllib.parse.quote(room_id)}"
91
+ try:
92
+ req = urllib.request.Request(url, method="GET")
93
+ with urllib.request.urlopen(req) as resp:
94
+ if resp.status != 200:
95
+ raise SharedotsError(f"Room {room_id} not found on coordinator")
96
+ except urllib.error.HTTPError as e:
97
+ if e.code == 404:
98
+ raise SharedotsError(f"Room {room_id} not found on coordinator")
99
+ raise
100
+
101
+ room = Room(self.coordinator_url, room_id, effective_token)
102
+ self.active_room = room
103
+ return room
104
+
105
+ def remote(
106
+ self,
107
+ fn: Optional[Callable[..., Any]] = None,
108
+ *,
109
+ cpu: float = 1.0,
110
+ memory: str = "512MB",
111
+ gpu: bool = False,
112
+ timeout_sec: int = 600,
113
+ max_attempts: int = 3,
114
+ name: Optional[str] = None,
115
+ ) -> Union[RemoteFunction, Callable[[Callable[..., Any]], RemoteFunction]]:
116
+ """Decorator binding a function to this Client's active room for distributed execution.
117
+
118
+ Usage:
119
+ client = sd.Client(coordinator="...", room_id="...", token="...")
120
+
121
+ @client.remote(cpu=2, memory="1GB", gpu=True)
122
+ def process_tensor(data):
123
+ return data * 2
124
+
125
+ future = process_tensor(my_data)
126
+ result = future.result()
127
+ """
128
+ def decorator(target_fn: Callable[..., Any]) -> RemoteFunction:
129
+ if not self.active_room:
130
+ raise SharedotsError(
131
+ "No active room connected. Call client.create_room() or client.connect(...) first."
132
+ )
133
+ return RemoteFunction(
134
+ fn=target_fn,
135
+ room=self.active_room,
136
+ cpu=cpu,
137
+ memory=memory,
138
+ gpu=gpu,
139
+ timeout_sec=timeout_sec,
140
+ max_attempts=max_attempts,
141
+ name=name,
142
+ )
143
+
144
+ if fn is not None:
145
+ return decorator(fn)
146
+ return decorator
147
+
148
+
149
+ def connect(
150
+ room: str,
151
+ token: str,
152
+ coordinator: str = "http://127.0.0.1:4000",
153
+ ) -> Client:
154
+ """Connects directly to an existing room and returns an authenticated Client instance."""
155
+ return Client(coordinator=coordinator, room_id=room, token=token)
sharedots/crypto.py ADDED
@@ -0,0 +1,44 @@
1
+ """Cryptographic and HMAC authentication utilities matching @sharedots/protocol."""
2
+
3
+ import hashlib
4
+ import hmac
5
+ import time
6
+ from typing import Dict, Union
7
+
8
+
9
+ def compute_request_signature(
10
+ room_secret: str,
11
+ method: str,
12
+ path: str,
13
+ timestamp_sec: Union[int, str],
14
+ room_id: str,
15
+ ) -> str:
16
+ """Computes HMAC-SHA256 signature for authenticating HTTP requests to Coordinator."""
17
+ canonical = f"{method.upper()}\n{path}\n{str(timestamp_sec)}\n{room_id}"
18
+ return hmac.new(
19
+ room_secret.encode("utf-8"),
20
+ canonical.encode("utf-8"),
21
+ hashlib.sha256,
22
+ ).hexdigest()
23
+
24
+
25
+ def auth_headers(
26
+ room_secret: str,
27
+ room_id: str,
28
+ method: str,
29
+ path: str,
30
+ timestamp_sec: Union[int, None] = None,
31
+ ) -> Dict[str, str]:
32
+ """Generates standard Sharedots room authentication HTTP headers."""
33
+ ts = timestamp_sec if timestamp_sec is not None else int(time.time())
34
+ sig = compute_request_signature(room_secret, method, path, ts, room_id)
35
+ return {
36
+ "x-sharedots-room": room_id,
37
+ "x-sharedots-ts": str(ts),
38
+ "x-sharedots-sig": sig,
39
+ }
40
+
41
+
42
+ def sha256_bytes(data: bytes) -> str:
43
+ """Computes hexadecimal SHA-256 digest of binary payload."""
44
+ return hashlib.sha256(data).hexdigest()
@@ -0,0 +1,48 @@
1
+ """Sharedots Python SDK Exceptions."""
2
+
3
+ from typing import Optional
4
+
5
+
6
+ class SharedotsError(Exception):
7
+ """Base exception for all Sharedots errors."""
8
+ pass
9
+
10
+
11
+ class AuthenticationError(SharedotsError):
12
+ """Raised when authentication with the Coordinator fails."""
13
+ pass
14
+
15
+
16
+ class ArtifactError(SharedotsError):
17
+ """Raised when artifact upload, download, or verification fails."""
18
+ pass
19
+
20
+
21
+ class JobTimeoutError(SharedotsError):
22
+ """Raised when waiting for a job or task times out."""
23
+ pass
24
+
25
+
26
+ class RemoteExecutionError(SharedotsError):
27
+ """Raised when a remote worker task raises an unhandled exception.
28
+
29
+ Attributes:
30
+ error_type: Name of the remote exception class.
31
+ error_message: String message of the exception.
32
+ remote_traceback: Full traceback captured on the remote worker node.
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ error_message: str,
38
+ error_type: Optional[str] = None,
39
+ remote_traceback: Optional[str] = None,
40
+ ):
41
+ self.error_type = error_type or "Exception"
42
+ self.error_message = error_message
43
+ self.remote_traceback = remote_traceback
44
+
45
+ formatted = f"Remote task failed with {self.error_type}: {self.error_message}"
46
+ if self.remote_traceback:
47
+ formatted += f"\n--- Remote Worker Traceback ---\n{self.remote_traceback}"
48
+ super().__init__(formatted)
sharedots/future.py ADDED
@@ -0,0 +1,156 @@
1
+ """Future and asynchronous completion abstractions for Sharedots remote executions."""
2
+
3
+ import time
4
+ from typing import Any, List, Optional
5
+
6
+ try:
7
+ import cloudpickle as serializer
8
+ except ImportError:
9
+ import pickle as serializer
10
+
11
+ from .exceptions import JobTimeoutError, RemoteExecutionError, SharedotsError
12
+ from .room import Room
13
+
14
+
15
+ class SharedotsFuture:
16
+ """A Future representing the eventual result of a distributed compute task."""
17
+
18
+ def __init__(
19
+ self,
20
+ room: Room,
21
+ job_id: str,
22
+ task_index: int = 0,
23
+ task_id: Optional[str] = None,
24
+ ):
25
+ self.room = room
26
+ self.job_id = job_id
27
+ self.task_index = task_index
28
+ self.task_id = task_id
29
+ self._cached_result: Optional[Any] = None
30
+ self._is_resolved = False
31
+ self._last_snapshot: Optional[dict] = None
32
+
33
+ def done(self) -> bool:
34
+ """Returns True if the remote task has completed, failed, or was cancelled."""
35
+ if self._is_resolved:
36
+ return True
37
+ detail = self.room.get_job(self.job_id)
38
+ self._last_snapshot = detail
39
+ status = detail.get("status")
40
+ return status in ("completed", "failed", "cancelled")
41
+
42
+ @property
43
+ def status(self) -> str:
44
+ """Returns the current execution status (queued, running, completed, failed, cancelled)."""
45
+ if self._is_resolved:
46
+ return "completed"
47
+ if not self._last_snapshot:
48
+ self._last_snapshot = self.room.get_job(self.job_id)
49
+ return self._last_snapshot.get("status", "unknown")
50
+
51
+ @property
52
+ def progress(self) -> float:
53
+ """Returns execution progress percentage (0.0 to 100.0)."""
54
+ if self._is_resolved:
55
+ return 100.0
56
+ if not self._last_snapshot:
57
+ self._last_snapshot = self.room.get_job(self.job_id)
58
+ prog = self._last_snapshot.get("progress", {})
59
+ return float(prog.get("percent", 0.0))
60
+
61
+ def result(self, timeout: Optional[float] = None, poll_interval: float = 0.2) -> Any:
62
+ """Waits for task completion and returns the unpickled result.
63
+
64
+ Raises:
65
+ JobTimeoutError: If the task exceeds timeout seconds.
66
+ RemoteExecutionError: If the remote task raised an exception on the worker.
67
+ SharedotsError: If the job was cancelled or encountered a system fault.
68
+ """
69
+ if self._is_resolved:
70
+ return self._cached_result
71
+
72
+ start_time = time.time()
73
+ while True:
74
+ detail = self.room.get_job(self.job_id)
75
+ self._last_snapshot = detail
76
+ status = detail.get("status")
77
+
78
+ if status == "completed":
79
+ break
80
+ elif status == "failed":
81
+ err_msg = detail.get("errorMessage") or "Job execution failed"
82
+ err_code = detail.get("errorCode") or "EXECUTION_FAILED"
83
+ raise RemoteExecutionError(f"{err_code}: {err_msg}")
84
+ elif status == "cancelled":
85
+ raise SharedotsError(f"Job {self.job_id} was cancelled")
86
+
87
+ if timeout is not None and (time.time() - start_time) > timeout:
88
+ raise JobTimeoutError(f"Job {self.job_id} timed out after {timeout} seconds")
89
+
90
+ time.sleep(poll_interval)
91
+
92
+ # Job completed — resolve the specific task output artifact
93
+ tasks = detail.get("tasks", [])
94
+ output_artifact_id = None
95
+
96
+ task_info = None
97
+ if self.task_id:
98
+ task_info = next((t for t in tasks if t.get("taskId") == self.task_id), None)
99
+ if not task_info:
100
+ task_info = next((t for t in tasks if t.get("taskIndex") == self.task_index), None)
101
+ if not task_info and self.task_index < len(tasks):
102
+ task_info = tasks[self.task_index]
103
+
104
+ if task_info:
105
+ out = task_info.get("output")
106
+ if out and isinstance(out, dict):
107
+ output_artifact_id = out.get("artifactId")
108
+
109
+ # Fallback to result artifact if tasks list output was empty
110
+ if not output_artifact_id:
111
+ res_obj = detail.get("result") or {}
112
+ output_artifact_id = res_obj.get("resultArtifactId")
113
+
114
+ if not output_artifact_id:
115
+ raise SharedotsError(f"Job {self.job_id} task {self.task_index} completed without an output artifact")
116
+
117
+ # Download artifact and unpickle
118
+ artifact_bytes = self.room.download_artifact(output_artifact_id)
119
+ out_data = serializer.loads(artifact_bytes)
120
+
121
+ if not isinstance(out_data, dict) or "success" not in out_data:
122
+ # Direct raw return
123
+ self._cached_result = out_data
124
+ self._is_resolved = True
125
+ return out_data
126
+
127
+ if not out_data.get("success"):
128
+ err = out_data.get("error") or {}
129
+ raise RemoteExecutionError(
130
+ error_message=err.get("message", "Remote task execution failed"),
131
+ error_type=err.get("type"),
132
+ remote_traceback=err.get("traceback"),
133
+ )
134
+
135
+ self._cached_result = out_data.get("result")
136
+ self._is_resolved = True
137
+ return self._cached_result
138
+
139
+ def cancel(self, reason: str = "Cancelled by client") -> dict:
140
+ """Cancels this job on the coordinator."""
141
+ return self.room.cancel_job(self.job_id, reason=reason)
142
+
143
+
144
+ def gather(futures: List[SharedotsFuture], timeout: Optional[float] = None) -> List[Any]:
145
+ """Waits for a list of SharedotsFutures and returns their results in order."""
146
+ start_time = time.time()
147
+ results = []
148
+ for f in futures:
149
+ remaining_timeout = None
150
+ if timeout is not None:
151
+ elapsed = time.time() - start_time
152
+ if elapsed >= timeout:
153
+ raise JobTimeoutError(f"gather() timed out after {timeout} seconds")
154
+ remaining_timeout = timeout - elapsed
155
+ results.append(f.result(timeout=remaining_timeout))
156
+ return results
sharedots/remote.py ADDED
@@ -0,0 +1,229 @@
1
+ """Remote function decorator and distributed task dispatch abstractions."""
2
+
3
+ import inspect
4
+ from pathlib import Path
5
+ from typing import Any, Callable, Iterable, List, Optional, Union
6
+
7
+ try:
8
+ import cloudpickle as serializer
9
+ except ImportError:
10
+ import pickle as serializer
11
+
12
+ from .future import SharedotsFuture, gather
13
+ from .room import Room
14
+
15
+
16
+ def _get_runner_source() -> bytes:
17
+ """Returns the raw source bytes of runner.py for worker execution."""
18
+ runner_file = Path(__file__).parent / "runner.py"
19
+ if runner_file.exists():
20
+ return runner_file.read_bytes()
21
+ # Inline fallback if packaged as zip
22
+ import inspect
23
+ from . import runner
24
+ return inspect.getsource(runner).encode("utf-8")
25
+
26
+
27
+ class RemoteFunction:
28
+ """Wraps a Python callable to enable distributed execution across the Sharedots mesh."""
29
+
30
+ def __init__(
31
+ self,
32
+ fn: Callable[..., Any],
33
+ room: Room,
34
+ cpu: float = 1.0,
35
+ memory: str = "512MB",
36
+ gpu: bool = False,
37
+ timeout_sec: int = 600,
38
+ max_attempts: int = 3,
39
+ name: Optional[str] = None,
40
+ ):
41
+ self.fn = fn
42
+ self.room = room
43
+ self.cpu = max(1.0, float(cpu))
44
+ self.memory = memory
45
+ self.gpu = bool(gpu)
46
+ self.timeout_sec = int(timeout_sec)
47
+ self.max_attempts = int(max_attempts)
48
+ self.name = name or getattr(fn, "__name__", "remote_task")
49
+
50
+ # Cache uploaded runner artifact per room instance
51
+ self._runner_artifact: Optional[dict] = None
52
+
53
+ def _ensure_runner_artifact(self) -> dict:
54
+ if self._runner_artifact is None:
55
+ runner_bytes = _get_runner_source()
56
+ self._runner_artifact = self.room.upload_artifact(
57
+ source=runner_bytes,
58
+ filename="runner.py",
59
+ shared=True,
60
+ )
61
+ return self._runner_artifact
62
+
63
+ def local(self, *args: Any, **kwargs: Any) -> Any:
64
+ """Executes the function locally in the current process (for testing & debugging)."""
65
+ return self.fn(*args, **kwargs)
66
+
67
+ def submit(self, *args: Any, **kwargs: Any) -> SharedotsFuture:
68
+ """Submits the function for remote execution and returns a SharedotsFuture."""
69
+ runner_ref = self._ensure_runner_artifact()
70
+
71
+ # Serialize function + arguments
72
+ task_payload = {
73
+ "fn": self.fn,
74
+ "args": args,
75
+ "kwargs": kwargs,
76
+ }
77
+ serialized_input = serializer.dumps(task_payload)
78
+
79
+ input_artifact = self.room.upload_artifact(
80
+ source=serialized_input,
81
+ filename="task_input.pkl",
82
+ shared=False,
83
+ )
84
+
85
+ job_name = f"{self.name}-job"
86
+ spec = {
87
+ "specVersion": 1,
88
+ "name": job_name,
89
+ "runtime": "python",
90
+ "execution": {
91
+ "strategy": "task-parallel",
92
+ "workers": "auto",
93
+ "partitions": 1,
94
+ },
95
+ "resources": {
96
+ "cpu": self.cpu,
97
+ "memory": self.memory,
98
+ "gpu": "required" if self.gpu else "optional",
99
+ },
100
+ "timeoutSec": self.timeout_sec,
101
+ "retry": {"maxAttempts": self.max_attempts},
102
+ "sharedArtifacts": [runner_ref],
103
+ "tasks": [
104
+ {
105
+ "index": 0,
106
+ "name": f"{self.name}-task-0",
107
+ "entrypoint": "runner.py",
108
+ "args": ["task_input.pkl"],
109
+ "inputs": [input_artifact],
110
+ "outputFilename": "task_output.pkl",
111
+ }
112
+ ],
113
+ }
114
+
115
+ job_resp = self.room.submit(spec)
116
+ job_id = job_resp.get("jobId") or job_resp.get("id")
117
+ return SharedotsFuture(self.room, job_id, task_index=0)
118
+
119
+ def __call__(self, *args: Any, **kwargs: Any) -> SharedotsFuture:
120
+ """Default invocation dispatches execution to cluster and returns a Future."""
121
+ return self.submit(*args, **kwargs)
122
+
123
+ def map(self, iterable: Iterable[Any], timeout: Optional[float] = None) -> List[Any]:
124
+ """Dispatches multiple calls as a batched parallel job across cluster nodes.
125
+
126
+ Accepts an iterable of arguments:
127
+ - If items are tuples, they are expanded as positional args: `fn(*item)`
128
+ - If items are dicts, they are expanded as keyword args: `fn(**item)`
129
+ - Otherwise, items are passed as single positional arg: `fn(item)`
130
+ """
131
+ runner_ref = self._ensure_runner_artifact()
132
+ items = list(iterable)
133
+ if not items:
134
+ return []
135
+
136
+ tasks = []
137
+ for idx, item in enumerate(items):
138
+ if isinstance(item, tuple):
139
+ args = item
140
+ kwargs = {}
141
+ elif isinstance(item, dict):
142
+ args = ()
143
+ kwargs = item
144
+ else:
145
+ args = (item,)
146
+ kwargs = {}
147
+
148
+ task_payload = {
149
+ "fn": self.fn,
150
+ "args": args,
151
+ "kwargs": kwargs,
152
+ }
153
+ serialized_input = serializer.dumps(task_payload)
154
+ input_artifact = self.room.upload_artifact(
155
+ source=serialized_input,
156
+ filename=f"task_input_{idx}.pkl",
157
+ shared=False,
158
+ )
159
+
160
+ tasks.append({
161
+ "index": idx,
162
+ "name": f"{self.name}-shard-{idx}",
163
+ "entrypoint": "runner.py",
164
+ "args": [f"task_input_{idx}.pkl"],
165
+ "inputs": [input_artifact],
166
+ "outputFilename": f"task_output_{idx}.pkl",
167
+ })
168
+
169
+ job_name = f"{self.name}-batch-{len(items)}"
170
+ spec = {
171
+ "specVersion": 1,
172
+ "name": job_name,
173
+ "runtime": "python",
174
+ "execution": {
175
+ "strategy": "task-parallel",
176
+ "workers": "auto",
177
+ "partitions": len(tasks),
178
+ },
179
+ "resources": {
180
+ "cpu": self.cpu,
181
+ "memory": self.memory,
182
+ "gpu": "required" if self.gpu else "optional",
183
+ },
184
+ "timeoutSec": self.timeout_sec,
185
+ "retry": {"maxAttempts": self.max_attempts},
186
+ "sharedArtifacts": [runner_ref],
187
+ "tasks": tasks,
188
+ }
189
+
190
+ job_resp = self.room.submit(spec)
191
+ job_id = job_resp.get("jobId") or job_resp.get("id")
192
+
193
+ # Create futures for every shard in this batched job
194
+ futures = [SharedotsFuture(self.room, job_id, task_index=i) for i in range(len(tasks))]
195
+ return gather(futures, timeout=timeout)
196
+
197
+
198
+ def remote(
199
+ fn: Optional[Callable[..., Any]] = None,
200
+ *,
201
+ room: Optional[Room] = None,
202
+ cpu: float = 1.0,
203
+ memory: str = "512MB",
204
+ gpu: bool = False,
205
+ timeout_sec: int = 600,
206
+ max_attempts: int = 3,
207
+ name: Optional[str] = None,
208
+ ) -> Union[RemoteFunction, Callable[[Callable[..., Any]], RemoteFunction]]:
209
+ """Decorator to convert a Python function into a distributed RemoteFunction."""
210
+ def decorator(target_fn: Callable[..., Any]) -> RemoteFunction:
211
+ if room is None:
212
+ raise ValueError(
213
+ "A Room instance is required to decorate with @remote. "
214
+ "Use @client.remote(...) or pass room=room to @remote."
215
+ )
216
+ return RemoteFunction(
217
+ fn=target_fn,
218
+ room=room,
219
+ cpu=cpu,
220
+ memory=memory,
221
+ gpu=gpu,
222
+ timeout_sec=timeout_sec,
223
+ max_attempts=max_attempts,
224
+ name=name,
225
+ )
226
+
227
+ if fn is not None:
228
+ return decorator(fn)
229
+ return decorator
sharedots/room.py ADDED
@@ -0,0 +1,152 @@
1
+ """Room client handling authenticated artifact transfer and job management."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Dict, List, Optional, Union
6
+ import urllib.error
7
+ import urllib.parse
8
+ import urllib.request
9
+
10
+ from .crypto import auth_headers, sha256_bytes
11
+ from .exceptions import ArtifactError, AuthenticationError, SharedotsError
12
+
13
+
14
+ class Room:
15
+ """Represents an active connection to a Sharedots compute room."""
16
+
17
+ def __init__(self, coordinator_url: str, room_id: str, token: str):
18
+ self.coordinator_url = coordinator_url.rstrip("/")
19
+ self.room_id = room_id
20
+ self.token = token
21
+
22
+ def _request(
23
+ self,
24
+ method: str,
25
+ path: str,
26
+ data: Optional[bytes] = None,
27
+ headers: Optional[Dict[str, str]] = None,
28
+ ) -> urllib.request.Request:
29
+ url = f"{self.coordinator_url}{path}"
30
+ canonical_path = path.split("?")[0]
31
+ req_headers = auth_headers(self.token, self.room_id, method, canonical_path)
32
+ if headers:
33
+ req_headers.update(headers)
34
+ return urllib.request.Request(url, data=data, headers=req_headers, method=method)
35
+
36
+ def _open(self, req: urllib.request.Request) -> tuple[int, bytes, Dict[str, str]]:
37
+ try:
38
+ with urllib.request.urlopen(req) as resp:
39
+ status = resp.status
40
+ body = resp.read()
41
+ resp_headers = {k.lower(): v for k, v in resp.headers.items()}
42
+ return status, body, resp_headers
43
+ except urllib.error.HTTPError as e:
44
+ err_body = e.read().decode("utf-8", errors="replace")
45
+ if e.code == 401:
46
+ raise AuthenticationError(f"Authentication failed for room {self.room_id}: {err_body}")
47
+ raise SharedotsError(f"HTTP {e.code} for {req.get_full_url()}: {err_body}")
48
+ except urllib.error.URLError as e:
49
+ notice = ""
50
+ if "onrender.com" in self.coordinator_url:
51
+ notice = (
52
+ f"\nšŸ’” [Render Free Tier Notice] The coordinator at {self.coordinator_url} appears to be sleeping.\n"
53
+ " Render's free tier spins down after 15 min of inactivity. Please give it ~30-50 seconds to wake up and retry!"
54
+ )
55
+ raise SharedotsError(f"Failed to reach coordinator at {self.coordinator_url}: {e.reason}{notice}")
56
+
57
+ def upload_artifact(
58
+ self,
59
+ source: Union[bytes, str, Path],
60
+ filename: Optional[str] = None,
61
+ shared: bool = False,
62
+ job_id: Optional[str] = None,
63
+ ) -> Dict[str, Any]:
64
+ """Uploads a data payload or file as a content-addressed artifact."""
65
+ if isinstance(source, (str, Path)):
66
+ file_path = Path(source).resolve()
67
+ if not file_path.exists():
68
+ raise ArtifactError(f"Source file not found: {file_path}")
69
+ data = file_path.read_bytes()
70
+ resolved_name = filename or file_path.name
71
+ else:
72
+ data = source
73
+ resolved_name = filename or "artifact.bin"
74
+
75
+ sha256 = sha256_bytes(data)
76
+ qs_params = {"filename": resolved_name}
77
+ if shared:
78
+ qs_params["shared"] = "true"
79
+ if job_id:
80
+ qs_params["jobId"] = job_id
81
+
82
+ path_with_qs = f"/api/artifacts/upload?{urllib.parse.urlencode(qs_params)}"
83
+ headers = {
84
+ "content-type": "application/octet-stream",
85
+ "x-sha256": sha256,
86
+ }
87
+
88
+ req = self._request("POST", path_with_qs, data=data, headers=headers)
89
+ status, body, _ = self._open(req)
90
+
91
+ if status not in (200, 201):
92
+ raise ArtifactError(f"Artifact upload failed ({status}): {body.decode('utf-8', errors='replace')}")
93
+
94
+ result = json.loads(body.decode("utf-8"))
95
+ result["shared"] = shared
96
+ return result
97
+
98
+ def download_artifact(self, artifact_id: str) -> bytes:
99
+ """Downloads and SHA-256 verifies an artifact by ID."""
100
+ path = f"/api/artifacts/{urllib.parse.quote(artifact_id)}"
101
+ req = self._request("GET", path)
102
+ status, body, headers = self._open(req)
103
+
104
+ if status != 200:
105
+ raise ArtifactError(f"Artifact download failed ({status}): {body.decode('utf-8', errors='replace')}")
106
+
107
+ expected_sha = headers.get("x-artifact-sha256")
108
+ if expected_sha:
109
+ actual_sha = sha256_bytes(body)
110
+ if actual_sha.lower() != expected_sha.lower():
111
+ raise ArtifactError(
112
+ f"Artifact {artifact_id} hash mismatch: expected {expected_sha}, got {actual_sha}"
113
+ )
114
+
115
+ return body
116
+
117
+ def submit(self, spec: Dict[str, Any]) -> Dict[str, Any]:
118
+ """Submits a JobSpec to the coordinator scheduler."""
119
+ path = "/api/jobs"
120
+ payload = json.dumps({"spec": spec}).encode("utf-8")
121
+ headers = {"content-type": "application/json"}
122
+ req = self._request("POST", path, data=payload, headers=headers)
123
+ status, body, _ = self._open(req)
124
+
125
+ if status not in (200, 201):
126
+ raise SharedotsError(f"Job submission failed ({status}): {body.decode('utf-8', errors='replace')}")
127
+
128
+ return json.loads(body.decode("utf-8"))
129
+
130
+ def get_job(self, job_id: str) -> Dict[str, Any]:
131
+ """Fetches the latest job status and task execution details."""
132
+ path = f"/api/jobs/{urllib.parse.quote(job_id)}"
133
+ req = self._request("GET", path)
134
+ _, body, _ = self._open(req)
135
+ return json.loads(body.decode("utf-8"))
136
+
137
+ def cancel_job(self, job_id: str, reason: str = "Cancelled by client") -> Dict[str, Any]:
138
+ """Cancels a queued or running job."""
139
+ path = f"/api/jobs/{urllib.parse.quote(job_id)}/cancel"
140
+ payload = json.dumps({"reason": reason}).encode("utf-8")
141
+ headers = {"content-type": "application/json"}
142
+ req = self._request("POST", path, data=payload, headers=headers)
143
+ _, body, _ = self._open(req)
144
+ return json.loads(body.decode("utf-8"))
145
+
146
+ def list_workers(self) -> List[Dict[str, Any]]:
147
+ """Lists all registered workers connected to this room."""
148
+ path = f"/api/workers?roomId={urllib.parse.quote(self.room_id)}"
149
+ req = self._request("GET", path)
150
+ _, body, _ = self._open(req)
151
+ data = json.loads(body.decode("utf-8"))
152
+ return data.get("workers", [])
sharedots/runner.py ADDED
@@ -0,0 +1,111 @@
1
+ """Standalone worker task runner executed inside the Sharedots daemon Python environment.
2
+
3
+ This module is uploaded as a shared artifact and executed directly on worker nodes:
4
+ python sharedots/runner.py [task_input_file]
5
+ """
6
+
7
+ import os
8
+ import sys
9
+ import traceback
10
+
11
+ # Prefer cloudpickle for rich closure and function serialization, fallback to pickle
12
+ try:
13
+ import cloudpickle as serializer
14
+ except ImportError:
15
+ import pickle as serializer
16
+
17
+
18
+ def execute_task(input_path: str, output_path: str) -> bool:
19
+ """Executes a serialized task and writes the result payload to output_path."""
20
+ if not os.path.exists(input_path):
21
+ sys.stderr.write(f"Sharedots runner: task input file not found: {input_path}\n")
22
+ return False
23
+
24
+ try:
25
+ with open(input_path, "rb") as f:
26
+ payload = serializer.load(f)
27
+ except Exception as e:
28
+ sys.stderr.write(f"Sharedots runner: failed to deserialize task payload: {e}\n")
29
+ tb = traceback.format_exc()
30
+ out_data = {
31
+ "success": False,
32
+ "result": None,
33
+ "error": {
34
+ "type": type(e).__name__,
35
+ "message": f"Payload deserialization failed: {e}",
36
+ "traceback": tb,
37
+ },
38
+ }
39
+ write_output(output_path, out_data)
40
+ return False
41
+
42
+ fn = payload.get("fn")
43
+ args = payload.get("args", ())
44
+ kwargs = payload.get("kwargs", {})
45
+
46
+ if not callable(fn):
47
+ msg = f"Task payload 'fn' is not callable (got {type(fn).__name__})"
48
+ sys.stderr.write(f"Sharedots runner: {msg}\n")
49
+ out_data = {
50
+ "success": False,
51
+ "result": None,
52
+ "error": {
53
+ "type": "TypeError",
54
+ "message": msg,
55
+ "traceback": None,
56
+ },
57
+ }
58
+ write_output(output_path, out_data)
59
+ return False
60
+
61
+ success = True
62
+ result = None
63
+ error = None
64
+
65
+ try:
66
+ result = fn(*args, **kwargs)
67
+ except Exception as e:
68
+ success = False
69
+ tb = traceback.format_exc()
70
+ error = {
71
+ "type": type(e).__name__,
72
+ "message": str(e),
73
+ "traceback": tb,
74
+ }
75
+ sys.stderr.write(f"Sharedots runner: Task execution failed with {error['type']}: {error['message']}\n")
76
+ sys.stderr.write(tb + "\n")
77
+
78
+ out_data = {
79
+ "success": success,
80
+ "result": result,
81
+ "error": error,
82
+ }
83
+
84
+ write_output(output_path, out_data)
85
+ return success
86
+
87
+
88
+ def write_output(output_path: str, data: dict) -> None:
89
+ """Safely serializes output data to the destination path."""
90
+ out_dir = os.path.dirname(os.path.abspath(output_path))
91
+ if out_dir:
92
+ os.makedirs(out_dir, exist_ok=True)
93
+
94
+ with open(output_path, "wb") as f:
95
+ serializer.dump(data, f)
96
+
97
+
98
+ def main() -> None:
99
+ input_dir = os.environ.get("SHAREDOTS_INPUT_DIR", ".")
100
+ output_path = os.environ.get("SHAREDOTS_OUTPUT_PATH", "task_output.pkl")
101
+
102
+ task_file = sys.argv[1] if len(sys.argv) > 1 else "task_input.pkl"
103
+ if not os.path.isabs(task_file):
104
+ task_file = os.path.join(input_dir, task_file)
105
+
106
+ success = execute_task(task_file, output_path)
107
+ sys.exit(0 if success else 1)
108
+
109
+
110
+ if __name__ == "__main__":
111
+ main()
@@ -0,0 +1,107 @@
1
+ Metadata-Version: 2.4
2
+ Name: sharedots
3
+ Version: 0.1.0
4
+ Summary: Universal Distributed Compute Network — Python SDK & Remote Execution
5
+ Author: Sharedots Team
6
+ License: MIT
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Topic :: System :: Distributed Computing
9
+ Requires-Python: >=3.9
10
+ Description-Content-Type: text/markdown
11
+ Provides-Extra: dev
12
+ Requires-Dist: cloudpickle>=3.0.0; extra == "dev"
13
+ Requires-Dist: numpy>=1.24.0; extra == "dev"
14
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
15
+ Requires-Dist: pytest-xdist>=3.5.0; extra == "dev"
16
+ Dynamic: requires-python
17
+
18
+ # Sharedots Python SDK (`sharedots`)
19
+
20
+ Universal Distributed Compute Network — Python SDK & Remote Execution Engine.
21
+
22
+ The Sharedots Python SDK provides an intuitive, high-performance distributed computing interface with zero-boilerplate remote function execution. It seamlessly connects with the native Sharedots Coordinator and Rust worker daemons over Tailscale mesh networks or local subnets.
23
+
24
+ ---
25
+
26
+ ## šŸš€ Quickstart
27
+
28
+ ### 1. Connect to a Compute Room
29
+
30
+ ```python
31
+ import sharedots as sd
32
+
33
+ # Connect to an existing room (e.g. over Tailscale or localhost)
34
+ client = sd.Client(
35
+ coordinator="http://100.82.14.30:4000",
36
+ room_id="room-alpha-123",
37
+ token="secret-token-xyz"
38
+ )
39
+
40
+ # Or create a new room directly:
41
+ room = client.create_room(name="my-mesh-room")
42
+ ```
43
+
44
+ ### 2. Distribute Functions with `@client.remote`
45
+
46
+ Decorate any Python function to execute across remote cluster nodes:
47
+
48
+ ```python
49
+ import numpy as np
50
+ import sharedots as sd
51
+
52
+ client = sd.Client(coordinator="http://localhost:4000", room_id="room-1", token="secret")
53
+
54
+ @client.remote(cpu=2, memory="1GB", gpu=True)
55
+ def matrix_dot(a, b):
56
+ # Runs remotely on a cluster worker node
57
+ return np.dot(a, b)
58
+
59
+ # Dispatches task across the cluster and immediately returns a Future
60
+ future = matrix_dot(np.eye(100), np.eye(100))
61
+
62
+ # Await result
63
+ result = future.result(timeout=30)
64
+ print("Computed result:", result)
65
+ ```
66
+
67
+ ### 3. Parallel Sharded Batch Processing with `.map()`
68
+
69
+ Distribute array slices or parameter sweeps across all available worker nodes in a single job:
70
+
71
+ ```python
72
+ @client.remote(cpu=1, memory="512MB")
73
+ def evaluate_trial(trial_id, parameter):
74
+ import time
75
+ # Simulate simulation / training trial
76
+ return {"trial": trial_id, "score": parameter ** 2}
77
+
78
+ # Automatically shards tasks across all cluster workers in parallel:
79
+ trials = [(i, i * 0.1) for i in range(20)]
80
+ results = evaluate_trial.map(trials)
81
+
82
+ print("All trials completed:", results)
83
+ ```
84
+
85
+ ### 4. Gather Multiple Futures
86
+
87
+ ```python
88
+ futures = [matrix_dot(A_i, B) for A_i in slices]
89
+ results = sd.gather(futures)
90
+ ```
91
+
92
+ ### 5. Local Debugging with `.local()`
93
+
94
+ Test functions locally in your process without sending them to the network:
95
+
96
+ ```python
97
+ local_result = matrix_dot.local(np.eye(10), np.eye(10))
98
+ ```
99
+
100
+ ---
101
+
102
+ ## šŸ”’ Security & Sandboxing
103
+
104
+ Unlike naive pickle RPC scripts, Sharedots executes remote functions inside:
105
+ 1. Signed manifests verified by the Coordinator using **HMAC-SHA256**.
106
+ 2. Sandboxed worker processes supervised by the **Rust Daemon** (`sharedots-daemon`) with scrubbed environment variables and process-tree watchdogs.
107
+ 3. Content-addressed, SHA-256 verified chunked HTTP Range artifact caching.
@@ -0,0 +1,13 @@
1
+ sharedots/__init__.py,sha256=L5jmGVsrWuzfb-YGD1n1q4qAocMX61sJzB4RR2qhQYs,640
2
+ sharedots/client.py,sha256=XA_zFdRdECGaZLsiwa-J5Y25QF7JkKfE6755xQg-a9E,6182
3
+ sharedots/crypto.py,sha256=jB52FFExjZzPUKjRLhQJhKVeQjixS-mbTC3nsXrBhEI,1260
4
+ sharedots/exceptions.py,sha256=bDtvoiqZTBbOmJQJHRPhSTZuzTQ6BxG4i1NZ4gjH0Po,1389
5
+ sharedots/future.py,sha256=o9e062Z6lfRjcC45jBi2qKgSO0BaN-wNPUNB1lAuAW8,5940
6
+ sharedots/remote.py,sha256=B6BYP9dam1QdZcBTC1yU1iJB3398VlI_P89-IyJ-C2c,7708
7
+ sharedots/room.py,sha256=9yxdqEj11Zhb81PfVSuJzQpiqcgSLr9s71GGzEX9YBA,6519
8
+ sharedots/runner.py,sha256=XN4JCTtBjT7vWM1h73KjJxtNG69rRZZhMMBW6pRahuY,3266
9
+ sharedots-0.1.0.dist-info/METADATA,sha256=N9Ol6_itlF_KSJGuApSvyEEfaQXKQmNutTs3JuqV5i4,3328
10
+ sharedots-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
11
+ sharedots-0.1.0.dist-info/entry_points.txt,sha256=O5MXZHHAsvwrqx7dncKdMkKe5MbtjCemskDVZdzvdlU,59
12
+ sharedots-0.1.0.dist-info/top_level.txt,sha256=zzYOtUTovFUHEhGleF-7DSA7JtwfbSTEDeNTBEpZWvM,10
13
+ sharedots-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ sharedots-runner = sharedots.runner:main
@@ -0,0 +1 @@
1
+ sharedots