firecloud-devnet 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fc_mlops/__init__.py +3 -0
- fc_mlops/__main__.py +5 -0
- fc_mlops/anomaly.py +112 -0
- fc_mlops/artifact_store.py +111 -0
- fc_mlops/cli.py +190 -0
- fc_mlops/simulate_failure.py +100 -0
- fc_mlops/telemetry.py +72 -0
- fc_rag/__init__.py +3 -0
- fc_rag/cli.py +51 -0
- fc_rag/config.py +24 -0
- fc_rag/embedder.py +62 -0
- fc_rag/indexer.py +121 -0
- fc_rag/query_engine.py +79 -0
- fc_rag/requirements.txt +6 -0
- fc_rag/retriever.py +46 -0
- firecloud/__init__.py +17 -0
- firecloud/chunker.py +122 -0
- firecloud/cli.py +540 -0
- firecloud/crypto.py +269 -0
- firecloud/discovery.py +164 -0
- firecloud/distributor.py +269 -0
- firecloud/exceptions.py +41 -0
- firecloud/fec.py +87 -0
- firecloud/manifest.py +263 -0
- firecloud/network.py +90 -0
- firecloud/node.py +562 -0
- firecloud/storage.py +146 -0
- firecloud/sync.py +277 -0
- firecloud/transport.py +387 -0
- firecloud_devnet-0.1.0.dist-info/METADATA +158 -0
- firecloud_devnet-0.1.0.dist-info/RECORD +34 -0
- firecloud_devnet-0.1.0.dist-info/WHEEL +4 -0
- firecloud_devnet-0.1.0.dist-info/entry_points.txt +4 -0
- firecloud_devnet-0.1.0.dist-info/licenses/LICENSE +21 -0
firecloud/transport.py
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""FireCloud Transport Engine.
|
|
2
|
+
|
|
3
|
+
Handles secure peer-to-peer communication using asyncio TCP over TLS,
|
|
4
|
+
implementing a custom binary protocol, handshake, multiplexed chunk transfer,
|
|
5
|
+
manifest synchronization, and heartbeat monitoring.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
from datetime import datetime, timezone, timedelta
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import ssl
|
|
14
|
+
import struct
|
|
15
|
+
import tempfile
|
|
16
|
+
|
|
17
|
+
from cryptography import x509
|
|
18
|
+
from cryptography.x509.oid import NameOID
|
|
19
|
+
from cryptography.hazmat.primitives import hashes
|
|
20
|
+
from cryptography.hazmat.primitives.asymmetric import rsa
|
|
21
|
+
from cryptography.hazmat.primitives import serialization
|
|
22
|
+
|
|
23
|
+
from firecloud.exceptions import TransportError, NodeAuthError, ChunkNotFoundError
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("firecloud.transport")
|
|
26
|
+
|
|
27
|
+
# Protocol constants
|
|
28
|
+
MSG_AUTH = 0x01
|
|
29
|
+
MSG_AUTH_OK = 0x02
|
|
30
|
+
MSG_STORE_CHUNK = 0x10
|
|
31
|
+
MSG_RETRIEVE_CHUNK = 0x11
|
|
32
|
+
MSG_CHUNK_DATA = 0x12
|
|
33
|
+
MSG_CHUNK_NOT_FOUND = 0x13
|
|
34
|
+
MSG_SYNC_MANIFEST = 0x20
|
|
35
|
+
MSG_HEARTBEAT = 0x30
|
|
36
|
+
MSG_PEER_LIST = 0x40
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# TLS & Certificate Helpers
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def generate_self_signed_cert() -> tuple[bytes, bytes]:
|
|
44
|
+
"""Generate a self-signed cert and key for TLS node authentication."""
|
|
45
|
+
private_key = rsa.generate_private_key(
|
|
46
|
+
public_exponent=65537,
|
|
47
|
+
key_size=2048,
|
|
48
|
+
)
|
|
49
|
+
subject = issuer = x509.Name([
|
|
50
|
+
x509.NameAttribute(NameOID.COMMON_NAME, "firecloud-node"),
|
|
51
|
+
])
|
|
52
|
+
cert = (
|
|
53
|
+
x509.CertificateBuilder()
|
|
54
|
+
.subject_name(subject)
|
|
55
|
+
.issuer_name(issuer)
|
|
56
|
+
.public_key(private_key.public_key())
|
|
57
|
+
.serial_number(x509.random_serial_number())
|
|
58
|
+
.not_valid_before(datetime.now(timezone.utc) - timedelta(days=1))
|
|
59
|
+
.not_valid_after(datetime.now(timezone.utc) + timedelta(days=365))
|
|
60
|
+
.sign(private_key, hashes.SHA256())
|
|
61
|
+
)
|
|
62
|
+
cert_bytes = cert.public_bytes(serialization.Encoding.PEM)
|
|
63
|
+
key_bytes = private_key.private_bytes(
|
|
64
|
+
encoding=serialization.Encoding.PEM,
|
|
65
|
+
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
|
66
|
+
encryption_algorithm=serialization.NoEncryption(),
|
|
67
|
+
)
|
|
68
|
+
return cert_bytes, key_bytes
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def get_ssl_context(is_server: bool, node_dir: Path | None = None) -> ssl.SSLContext:
|
|
72
|
+
"""Get or create the SSLContext for secure connections."""
|
|
73
|
+
if node_dir is None:
|
|
74
|
+
node_dir = Path(tempfile.gettempdir()) / "firecloud_ssl"
|
|
75
|
+
node_dir.mkdir(parents=True, exist_ok=True)
|
|
76
|
+
|
|
77
|
+
cert_path = node_dir / "node.crt"
|
|
78
|
+
key_path = node_dir / "node.key"
|
|
79
|
+
|
|
80
|
+
if not cert_path.exists() or not key_path.exists():
|
|
81
|
+
cert_bytes, key_bytes = generate_self_signed_cert()
|
|
82
|
+
cert_path.write_bytes(cert_bytes)
|
|
83
|
+
key_path.write_bytes(key_bytes)
|
|
84
|
+
|
|
85
|
+
if is_server:
|
|
86
|
+
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
|
87
|
+
context.load_cert_chain(certfile=str(cert_path), keyfile=str(key_path))
|
|
88
|
+
else:
|
|
89
|
+
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
|
|
90
|
+
context.check_hostname = False
|
|
91
|
+
context.verify_mode = ssl.CERT_NONE
|
|
92
|
+
|
|
93
|
+
return context
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
# Binary framing helpers
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def read_msg(reader: asyncio.StreamReader) -> tuple[int, bytes]:
|
|
102
|
+
"""Read a structured message from the stream.
|
|
103
|
+
|
|
104
|
+
Format: [4 bytes length][1 byte type][payload]
|
|
105
|
+
"""
|
|
106
|
+
try:
|
|
107
|
+
header = await reader.readexactly(4)
|
|
108
|
+
length = struct.unpack("!I", header)[0]
|
|
109
|
+
msg_type_byte = await reader.readexactly(1)
|
|
110
|
+
msg_type = msg_type_byte[0]
|
|
111
|
+
payload = await reader.readexactly(length)
|
|
112
|
+
return msg_type, payload
|
|
113
|
+
except asyncio.IncompleteReadError as exc:
|
|
114
|
+
raise TransportError("Connection closed prematurely during read") from exc
|
|
115
|
+
except Exception as exc:
|
|
116
|
+
raise TransportError(f"Error reading message from socket: {exc}") from exc
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
async def write_msg(writer: asyncio.StreamWriter, msg_type: int, payload: bytes) -> None:
|
|
120
|
+
"""Write a structured message to the stream."""
|
|
121
|
+
try:
|
|
122
|
+
header = struct.pack("!IB", len(payload), msg_type)
|
|
123
|
+
writer.write(header + payload)
|
|
124
|
+
await writer.drain()
|
|
125
|
+
except Exception as exc:
|
|
126
|
+
raise TransportError(f"Error writing message to socket: {exc}") from exc
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
# Peer Connection Handler
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class PeerConnection:
|
|
135
|
+
"""Handles an active bidirectional connection with a remote peer."""
|
|
136
|
+
|
|
137
|
+
def __init__(
|
|
138
|
+
self,
|
|
139
|
+
reader: asyncio.StreamReader,
|
|
140
|
+
writer: asyncio.StreamWriter,
|
|
141
|
+
peer_node_id: str,
|
|
142
|
+
node,
|
|
143
|
+
) -> None:
|
|
144
|
+
self.reader = reader
|
|
145
|
+
self.writer = writer
|
|
146
|
+
self.peer_node_id = peer_node_id
|
|
147
|
+
self.node = node
|
|
148
|
+
self.write_lock = asyncio.Lock()
|
|
149
|
+
self.retrieve_lock = asyncio.Lock()
|
|
150
|
+
self.pending_retrieve: asyncio.Future[bytes | None] | None = None
|
|
151
|
+
self.last_seen = datetime.now(timezone.utc)
|
|
152
|
+
self.read_task = asyncio.create_task(self._read_loop())
|
|
153
|
+
|
|
154
|
+
async def send_message(self, msg_type: int, payload: bytes) -> None:
|
|
155
|
+
"""Send a message to the peer."""
|
|
156
|
+
async with self.write_lock:
|
|
157
|
+
await write_msg(self.writer, msg_type, payload)
|
|
158
|
+
|
|
159
|
+
async def retrieve_chunk(self, chunk_id: str) -> bytes | None:
|
|
160
|
+
"""Request and retrieve a chunk from this peer."""
|
|
161
|
+
async with self.retrieve_lock:
|
|
162
|
+
loop = asyncio.get_running_loop()
|
|
163
|
+
self.pending_retrieve = loop.create_future()
|
|
164
|
+
try:
|
|
165
|
+
await self.send_message(MSG_RETRIEVE_CHUNK, chunk_id.encode("utf-8"))
|
|
166
|
+
# Wait for the background loop to resolve the future
|
|
167
|
+
return await self.pending_retrieve
|
|
168
|
+
finally:
|
|
169
|
+
self.pending_retrieve = None
|
|
170
|
+
|
|
171
|
+
async def _read_loop(self) -> None:
|
|
172
|
+
"""Background loop that processes incoming messages from the peer."""
|
|
173
|
+
try:
|
|
174
|
+
while True:
|
|
175
|
+
msg_type, payload = await read_msg(self.reader)
|
|
176
|
+
self.last_seen = datetime.now(timezone.utc)
|
|
177
|
+
|
|
178
|
+
if msg_type == MSG_HEARTBEAT:
|
|
179
|
+
# Heartbeat received, last_seen is updated.
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
elif msg_type == MSG_CHUNK_DATA:
|
|
183
|
+
if self.pending_retrieve and not self.pending_retrieve.done():
|
|
184
|
+
self.pending_retrieve.set_result(payload)
|
|
185
|
+
|
|
186
|
+
elif msg_type == MSG_CHUNK_NOT_FOUND:
|
|
187
|
+
if self.pending_retrieve and not self.pending_retrieve.done():
|
|
188
|
+
self.pending_retrieve.set_result(None)
|
|
189
|
+
|
|
190
|
+
elif msg_type == MSG_STORE_CHUNK:
|
|
191
|
+
if len(payload) < 64:
|
|
192
|
+
continue
|
|
193
|
+
chunk_id = payload[:64].decode("utf-8")
|
|
194
|
+
chunk_data = payload[64:]
|
|
195
|
+
try:
|
|
196
|
+
self.node.chunk_store.store(chunk_id, chunk_data)
|
|
197
|
+
except Exception as e:
|
|
198
|
+
logger.error(f"Failed to store chunk {chunk_id}: {e}")
|
|
199
|
+
|
|
200
|
+
elif msg_type == MSG_RETRIEVE_CHUNK:
|
|
201
|
+
chunk_id = payload.decode("utf-8")
|
|
202
|
+
try:
|
|
203
|
+
chunk_data = self.node.chunk_store.retrieve(chunk_id)
|
|
204
|
+
await self.send_message(MSG_CHUNK_DATA, chunk_data)
|
|
205
|
+
except ChunkNotFoundError:
|
|
206
|
+
await self.send_message(
|
|
207
|
+
MSG_CHUNK_NOT_FOUND, chunk_id.encode("utf-8")
|
|
208
|
+
)
|
|
209
|
+
except Exception as e:
|
|
210
|
+
logger.error(f"Failed to retrieve chunk {chunk_id}: {e}")
|
|
211
|
+
await self.send_message(
|
|
212
|
+
MSG_CHUNK_NOT_FOUND, chunk_id.encode("utf-8")
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
elif msg_type == MSG_SYNC_MANIFEST:
|
|
216
|
+
try:
|
|
217
|
+
remote_entries_dicts = json.loads(payload.decode("utf-8"))
|
|
218
|
+
from firecloud.manifest import FileEntry, ChunkInfo
|
|
219
|
+
remote_entries = []
|
|
220
|
+
for edict in remote_entries_dicts:
|
|
221
|
+
d = dict(edict)
|
|
222
|
+
chunks = [ChunkInfo(**ci) for ci in d.pop("chunks", [])]
|
|
223
|
+
remote_entries.append(FileEntry(**d, chunks=chunks))
|
|
224
|
+
self.node.manifest.merge(remote_entries)
|
|
225
|
+
except Exception as e:
|
|
226
|
+
logger.error(f"Failed to merge remote manifest: {e}")
|
|
227
|
+
|
|
228
|
+
elif msg_type == MSG_PEER_LIST:
|
|
229
|
+
try:
|
|
230
|
+
peers = json.loads(payload.decode("utf-8"))
|
|
231
|
+
for p in peers:
|
|
232
|
+
if p["node_id"] != self.node.node_id:
|
|
233
|
+
self.node.add_peer_discovered(
|
|
234
|
+
p["node_id"], p["host"], p["port"]
|
|
235
|
+
)
|
|
236
|
+
except Exception as e:
|
|
237
|
+
logger.error(f"Failed to process peer list: {e}")
|
|
238
|
+
|
|
239
|
+
except asyncio.CancelledError:
|
|
240
|
+
pass
|
|
241
|
+
except Exception as e:
|
|
242
|
+
logger.debug(f"Connection with peer {self.peer_node_id} dropped: {e}")
|
|
243
|
+
finally:
|
|
244
|
+
await self.close()
|
|
245
|
+
|
|
246
|
+
async def close(self) -> None:
|
|
247
|
+
"""Close the connection."""
|
|
248
|
+
self.read_task.cancel()
|
|
249
|
+
if self.pending_retrieve and not self.pending_retrieve.done():
|
|
250
|
+
self.pending_retrieve.set_exception(
|
|
251
|
+
TransportError("Connection closed while waiting for chunk retrieval")
|
|
252
|
+
)
|
|
253
|
+
try:
|
|
254
|
+
self.writer.close()
|
|
255
|
+
await asyncio.wait_for(self.writer.wait_closed(), timeout=0.5)
|
|
256
|
+
except (Exception, asyncio.CancelledError):
|
|
257
|
+
pass
|
|
258
|
+
self.node.on_connection_closed(self.peer_node_id)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# ---------------------------------------------------------------------------
|
|
262
|
+
# Node Server
|
|
263
|
+
# ---------------------------------------------------------------------------
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class NodeServer:
|
|
267
|
+
"""Listens for secure TCP connections from remote peers."""
|
|
268
|
+
|
|
269
|
+
def __init__(self, node, host: str, port: int) -> None:
|
|
270
|
+
self.node = node
|
|
271
|
+
self.host = host
|
|
272
|
+
self.port = port
|
|
273
|
+
self.server: asyncio.AbstractServer | None = None
|
|
274
|
+
|
|
275
|
+
async def start(self) -> None:
|
|
276
|
+
"""Start the TCP server."""
|
|
277
|
+
ssl_context = get_ssl_context(is_server=True, node_dir=self.node.storage_path / "ssl")
|
|
278
|
+
self.server = await asyncio.start_server(
|
|
279
|
+
self._handle_connection,
|
|
280
|
+
self.host,
|
|
281
|
+
self.port,
|
|
282
|
+
ssl=ssl_context,
|
|
283
|
+
)
|
|
284
|
+
logger.info(f"Node server listening on {self.host}:{self.port}")
|
|
285
|
+
|
|
286
|
+
async def stop(self) -> None:
|
|
287
|
+
"""Stop the TCP server."""
|
|
288
|
+
if self.server:
|
|
289
|
+
self.server.close()
|
|
290
|
+
# Close all active connections on the node to unblock wait_closed
|
|
291
|
+
if hasattr(self.node, "connections"):
|
|
292
|
+
for conn in list(self.node.connections.values()):
|
|
293
|
+
await conn.close()
|
|
294
|
+
try:
|
|
295
|
+
await asyncio.wait_for(self.server.wait_closed(), timeout=1.0)
|
|
296
|
+
except Exception:
|
|
297
|
+
pass
|
|
298
|
+
self.server = None
|
|
299
|
+
logger.info("Node server stopped")
|
|
300
|
+
|
|
301
|
+
async def _handle_connection(
|
|
302
|
+
self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
|
303
|
+
) -> None:
|
|
304
|
+
"""Handle an incoming connection from a peer."""
|
|
305
|
+
try:
|
|
306
|
+
# Perform server handshake
|
|
307
|
+
msg_type, payload = await read_msg(reader)
|
|
308
|
+
if msg_type != MSG_AUTH:
|
|
309
|
+
writer.close()
|
|
310
|
+
await writer.wait_closed()
|
|
311
|
+
return
|
|
312
|
+
|
|
313
|
+
if len(payload) < 32:
|
|
314
|
+
writer.close()
|
|
315
|
+
await writer.wait_closed()
|
|
316
|
+
return
|
|
317
|
+
|
|
318
|
+
token = payload[:32]
|
|
319
|
+
peer_node_id = payload[32:].decode("utf-8")
|
|
320
|
+
|
|
321
|
+
if token != self.node.network.auth_token:
|
|
322
|
+
writer.close()
|
|
323
|
+
await writer.wait_closed()
|
|
324
|
+
raise NodeAuthError("Peer authentication failed: invalid token")
|
|
325
|
+
|
|
326
|
+
# Authentication successful. Send AUTH_OK with our node ID.
|
|
327
|
+
await write_msg(writer, MSG_AUTH_OK, self.node.node_id.encode("utf-8"))
|
|
328
|
+
|
|
329
|
+
conn = PeerConnection(reader, writer, peer_node_id, self.node)
|
|
330
|
+
self.node.register_connection(peer_node_id, conn)
|
|
331
|
+
|
|
332
|
+
except Exception as e:
|
|
333
|
+
logger.debug(f"Error handling incoming connection: {e}")
|
|
334
|
+
try:
|
|
335
|
+
writer.close()
|
|
336
|
+
await writer.wait_closed()
|
|
337
|
+
except Exception:
|
|
338
|
+
pass
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# ---------------------------------------------------------------------------
|
|
342
|
+
# Node Client
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class NodeClient:
|
|
347
|
+
"""Establishes secure TCP connections to remote peers."""
|
|
348
|
+
|
|
349
|
+
def __init__(self, node) -> None:
|
|
350
|
+
self.node = node
|
|
351
|
+
|
|
352
|
+
async def connect(self, host: str, port: int) -> str:
|
|
353
|
+
"""Connect to a peer, authenticate, and register the connection."""
|
|
354
|
+
ssl_context = get_ssl_context(is_server=False, node_dir=self.node.storage_path / "ssl")
|
|
355
|
+
try:
|
|
356
|
+
reader, writer = await asyncio.open_connection(host, port, ssl=ssl_context)
|
|
357
|
+
except Exception as exc:
|
|
358
|
+
raise TransportError(f"Failed to connect to {host}:{port}: {exc}") from exc
|
|
359
|
+
|
|
360
|
+
try:
|
|
361
|
+
# Send AUTH message: auth_token (32 bytes) + our node_id (UTF-8 bytes)
|
|
362
|
+
auth_payload = self.node.network.auth_token + self.node.node_id.encode("utf-8")
|
|
363
|
+
await write_msg(writer, MSG_AUTH, auth_payload)
|
|
364
|
+
|
|
365
|
+
# Receive AUTH_OK
|
|
366
|
+
msg_type, payload = await read_msg(reader)
|
|
367
|
+
if msg_type != MSG_AUTH_OK:
|
|
368
|
+
writer.close()
|
|
369
|
+
await writer.wait_closed()
|
|
370
|
+
raise NodeAuthError("Handshake failed: expected AUTH_OK")
|
|
371
|
+
|
|
372
|
+
peer_node_id = payload.decode("utf-8")
|
|
373
|
+
conn = PeerConnection(reader, writer, peer_node_id, self.node)
|
|
374
|
+
self.node.register_connection(peer_node_id, conn)
|
|
375
|
+
return peer_node_id
|
|
376
|
+
|
|
377
|
+
except Exception as e:
|
|
378
|
+
writer.close()
|
|
379
|
+
try:
|
|
380
|
+
await writer.wait_closed()
|
|
381
|
+
except Exception:
|
|
382
|
+
pass
|
|
383
|
+
if isinstance(e, NodeAuthError):
|
|
384
|
+
raise e
|
|
385
|
+
if isinstance(e, TransportError):
|
|
386
|
+
raise NodeAuthError("Handshake failed: peer rejected connection or auth token mismatch") from e
|
|
387
|
+
raise TransportError(f"Handshake failed with {host}:{port}: {e}") from e
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: firecloud-devnet
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Private, encrypted, distributed storage across your own machines
|
|
5
|
+
Project-URL: Homepage, https://github.com/rajashekharsunkara/firecloud
|
|
6
|
+
Project-URL: Repository, https://github.com/rajashekharsunkara/firecloud
|
|
7
|
+
Project-URL: Issues, https://github.com/rajashekharsunkara/firecloud/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/rajashekharsunkara/firecloud/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Rajashekhar Sunkara
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: chunking,distributed,encryption,p2p,storage
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Security :: Cryptography
|
|
22
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: aiofiles>=23.0.0
|
|
25
|
+
Requires-Dist: click>=8.0.0
|
|
26
|
+
Requires-Dist: cryptography>=41.0.0
|
|
27
|
+
Requires-Dist: fastcdc>=1.5.0
|
|
28
|
+
Requires-Dist: pycryptodome>=3.20.0
|
|
29
|
+
Requires-Dist: watchdog>=3.0.0
|
|
30
|
+
Requires-Dist: zeroconf>=0.80.0
|
|
31
|
+
Requires-Dist: zfec>=1.5.0
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
36
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
37
|
+
Provides-Extra: mlops
|
|
38
|
+
Requires-Dist: fastapi>=0.100.0; extra == 'mlops'
|
|
39
|
+
Requires-Dist: numpy>=1.24.0; extra == 'mlops'
|
|
40
|
+
Requires-Dist: psutil>=5.9.0; extra == 'mlops'
|
|
41
|
+
Requires-Dist: pydantic>=2.0; extra == 'mlops'
|
|
42
|
+
Requires-Dist: rich>=13.0.0; extra == 'mlops'
|
|
43
|
+
Requires-Dist: scikit-learn>=1.3.0; extra == 'mlops'
|
|
44
|
+
Requires-Dist: uvicorn>=0.20.0; extra == 'mlops'
|
|
45
|
+
Provides-Extra: rag
|
|
46
|
+
Requires-Dist: fastembed>=0.2.0; extra == 'rag'
|
|
47
|
+
Requires-Dist: ollama>=0.1.0; extra == 'rag'
|
|
48
|
+
Requires-Dist: pydantic>=2.0; extra == 'rag'
|
|
49
|
+
Requires-Dist: qdrant-client>=1.8.0; extra == 'rag'
|
|
50
|
+
Requires-Dist: rich>=13.0.0; extra == 'rag'
|
|
51
|
+
Description-Content-Type: text/markdown
|
|
52
|
+
|
|
53
|
+

|
|
54
|
+
|
|
55
|
+
# FireCloud
|
|
56
|
+
|
|
57
|
+
Private, encrypted, distributed storage across machines you own.
|
|
58
|
+
|
|
59
|
+
Unlike S3 (vendor lock-in), Syncthing (no erasure coding), or IPFS (public DHT), FireCloud gives you zero-knowledge peer-to-peer storage where data is encrypted locally before it leaves your machine. Every chunk stored on the network is ciphertext — nodes can't read it.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Install
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
# from GitHub (recommended for now)
|
|
67
|
+
pip install git+https://github.com/rajashekharsunkara/firecloud.git
|
|
68
|
+
|
|
69
|
+
# with RAG extensions
|
|
70
|
+
pip install "firecloud-devnet[rag]"
|
|
71
|
+
|
|
72
|
+
# with MLOps extensions
|
|
73
|
+
pip install "firecloud-devnet[mlops]"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Quickstart
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# 1. Start a 4-node network via Docker Compose
|
|
80
|
+
git clone https://github.com/rajashekharsunkara/firecloud.git
|
|
81
|
+
cd firecloud
|
|
82
|
+
cp .env.example .env # set FIRECLOUD_PASSPHRASE in .env
|
|
83
|
+
docker compose up -d # starts bootstrap + 3 storage nodes
|
|
84
|
+
|
|
85
|
+
# 2. Upload a file
|
|
86
|
+
docker exec firecloud-bootstrap firecloud upload /data/my-file.zip
|
|
87
|
+
|
|
88
|
+
# 3. Download from any node
|
|
89
|
+
docker exec firecloud-node-1 firecloud download <file_id> /data/restored.zip
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Architecture
|
|
95
|
+
|
|
96
|
+
```
|
|
97
|
+
┌─────────────────────────────────────────┐
|
|
98
|
+
│ fc-rag (Private RAG — opt-in) │ LLMOps
|
|
99
|
+
│ fc-mlops (Artifact Store — opt-in) │ MLOps
|
|
100
|
+
│ Docker + GitHub Actions │ DevOps
|
|
101
|
+
│ FireCloud Core (storage, crypto, P2P) │ Distributed Systems
|
|
102
|
+
└─────────────────────────────────────────┘
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**Distributed Systems** — XChaCha20-Poly1305 encryption, FastCDC content-defined chunking, zfec erasure coding, mDNS peer discovery. Manifest consistency uses Lamport timestamps with last-writer-wins semantics. Node communication runs over TLS-protected binary RPC.
|
|
106
|
+
|
|
107
|
+
**DevOps** — Multi-node Docker Compose setup with health checks. GitHub Actions CI pipeline (lint → test → build) gates every merge.
|
|
108
|
+
|
|
109
|
+
**MLOps** — `fc-mlops` provides version-tracked ML artifact storage via FireCloud's `Node` API, a FastAPI telemetry endpoint with psutil system metrics, and IsolationForest-based anomaly detection on telemetry readings.
|
|
110
|
+
|
|
111
|
+
**LLMOps** — `fc-rag` is a fully local RAG pipeline using fastembed for embeddings, Qdrant (embedded mode) for vector search, and Ollama for local LLM inference — no text ever leaves your machine.
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## Security
|
|
116
|
+
|
|
117
|
+
FireCloud uses **HMAC-SHA-256 with a network-derived key** for chunk addressing instead of plain SHA-256. This raises the cost of confirmation-of-file attacks — an attacker who suspects a specific file is stored cannot verify its presence by computing chunk hashes from the plaintext, because valid chunk IDs require the network key. This protection holds as long as the network key remains confidential.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## AI/ML Extensions
|
|
122
|
+
|
|
123
|
+
FireCloud stores and retrieves encrypted content. The RAG and artifact layers run entirely on the client — nothing in plaintext crosses the server boundary.
|
|
124
|
+
|
|
125
|
+
### Private RAG (`fc-rag`)
|
|
126
|
+
|
|
127
|
+
Index your docs locally and query with a private LLM — no data leaves your machine.
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
pip install "firecloud-devnet[rag]"
|
|
131
|
+
fc-rag index ./docs
|
|
132
|
+
fc-rag query "How does FireCloud handle node departure?"
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### MLOps Artifact Store (`fc-mlops`)
|
|
136
|
+
|
|
137
|
+
Version-track ML models, datasets, and checkpoints using FireCloud as the storage backend.
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
pip install "firecloud-devnet[mlops]"
|
|
141
|
+
fc-ml save ./model.pt --name resnet --version 1.0.0 --type model --metric accuracy=0.94
|
|
142
|
+
fc-ml simulate-failure
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Development
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
git clone https://github.com/rajashekharsunkara/firecloud.git
|
|
151
|
+
cd firecloud
|
|
152
|
+
pip install -e ".[dev]"
|
|
153
|
+
pytest tests/ -v
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## License
|
|
157
|
+
|
|
158
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
fc_mlops/__init__.py,sha256=BaZAx9-tFMUWeBR-ltdL9a3XfxyDbbONKbhTMPURiN4,61
|
|
2
|
+
fc_mlops/__main__.py,sha256=90NkmknSUTGhdGhiSVswLJJEkDNS0G5pmc6eyS_DTRQ,128
|
|
3
|
+
fc_mlops/anomaly.py,sha256=bej4RTVjWneaCB3ckLm5dBe3njetPv6C7PPn2JCPNE0,3237
|
|
4
|
+
fc_mlops/artifact_store.py,sha256=3qgynHVB28H8yF-iLmQl3n7JllWI3YipZSjdlF0mM2Y,3006
|
|
5
|
+
fc_mlops/cli.py,sha256=waWnKRqu1HXbFQxCmWLuiTI263tLDZCjVFyAbNkxjtc,5732
|
|
6
|
+
fc_mlops/simulate_failure.py,sha256=716JhAFBiS4GdmTyT7AP7mnCtTEP9lz-M5zU5oXXQ4o,3480
|
|
7
|
+
fc_mlops/telemetry.py,sha256=znynDV9UJqQf3JpaNuTJsTmG0uo_nbnV2xw1sByr1XI,2146
|
|
8
|
+
fc_rag/__init__.py,sha256=1tLueyQGaIgv7WPvIrji_RxUr9wa3arAwpj6nTiDCZc,97
|
|
9
|
+
fc_rag/cli.py,sha256=uzvh7ccMGtj6iKAqoY3XEqqzy2r1vdFOCJI3rda4ePU,1278
|
|
10
|
+
fc_rag/config.py,sha256=-SFjQE2Z0SuC2znorrv7GpctKZ33exlStlz6Ue8Yodw,675
|
|
11
|
+
fc_rag/embedder.py,sha256=9l25UcKaCHJPbV6r6Bf3mvaiGs_lNl7-o3M6SO9QppM,1845
|
|
12
|
+
fc_rag/indexer.py,sha256=pkRuDwu8-xjwQY5q410o-KWTY3mGCltLYnrSetwL5pk,4041
|
|
13
|
+
fc_rag/query_engine.py,sha256=U31UBS6O-4nRibtJX1ZXWy9CslIzzEmUK4WyAD2iPL0,2377
|
|
14
|
+
fc_rag/requirements.txt,sha256=PjVjcp3ovoo_XNJ6o0v2nJHQDhkcB9MtUKCQ4TgQBTo,56
|
|
15
|
+
fc_rag/retriever.py,sha256=yUR9Kdq38I9q9co3emi12r5POwNyWBlGOYcRW5NBoek,1225
|
|
16
|
+
firecloud/__init__.py,sha256=VUgE4uVUyHObBHQYG5rDjcBp3Sdsp9XfV43N16yeVCs,415
|
|
17
|
+
firecloud/chunker.py,sha256=fuVfnabHzPuzQy9VhHMJzIWZlTOWnLsSgB5cR4Gzhno,3213
|
|
18
|
+
firecloud/cli.py,sha256=NdMevf1LnMAKJ09ryUwgQ5VeYy0U5J4d0PsWSKR3wkY,18774
|
|
19
|
+
firecloud/crypto.py,sha256=9-M2MxH-xhHWHrVm_tSYKURUOKVG25K3bpeCthi8Z4c,8301
|
|
20
|
+
firecloud/discovery.py,sha256=rs0zg-bFZwBVJ6FJNENA68gX3EHHUyEISNELLtSj2eU,5638
|
|
21
|
+
firecloud/distributor.py,sha256=QGeq6vS8C3aR_8OL9QsTc5ayJqiohNj-tFWaeltK3i8,10224
|
|
22
|
+
firecloud/exceptions.py,sha256=otDFDr-AbSHCpTt8eb40kMRYkoFh6ngpO7H92LpcO1M,1170
|
|
23
|
+
firecloud/fec.py,sha256=q30tH2p4R1vTgtHVR_ZBEh6Nq5BuSW2sp3KJLglfpvU,2713
|
|
24
|
+
firecloud/manifest.py,sha256=Tw2sxY08ITF1QfFUcW42X7lQ2fx5LWTjRy80o60krPY,8893
|
|
25
|
+
firecloud/network.py,sha256=-C6naE3ZcDEHoDCMw9T-03rqRUvyFkxdWtJCF58OcRU,2951
|
|
26
|
+
firecloud/node.py,sha256=V2OzqOkbd7cqQysCH3nH6LSH5vaSxHsTthIscUI0Fvw,21728
|
|
27
|
+
firecloud/storage.py,sha256=ZZwR5ZOWHLvK0aVj6-tKyT9D2t7ZMrqTgdLzYAWjFzs,4971
|
|
28
|
+
firecloud/sync.py,sha256=-oOCxvkkXVWZXBoVbv-1RW4RVaGCHV8yfjCd9V_ra6w,10875
|
|
29
|
+
firecloud/transport.py,sha256=qSCy_P3rywko-qNATlXwuRMEB2JWnKXvboIaX-DN-Ig,14986
|
|
30
|
+
firecloud_devnet-0.1.0.dist-info/METADATA,sha256=wgAi-YLWC2E16jW_MVD7Q9sPGHTS3nx94DnAhts37fg,6091
|
|
31
|
+
firecloud_devnet-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
32
|
+
firecloud_devnet-0.1.0.dist-info/entry_points.txt,sha256=aNR4TWzUB6IWq1CppyIVGmqrANwmQA9LKr4CQsVVeio,97
|
|
33
|
+
firecloud_devnet-0.1.0.dist-info/licenses/LICENSE,sha256=enPSqS9BWWItID5gIsaicRl5K2YdWhaq9WnoQnWynMM,1081
|
|
34
|
+
firecloud_devnet-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024-2025 Rajashekhar Sunkara
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|