fleetmesh-node 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleetmesh_node-0.2.0.dist-info/METADATA +57 -0
- fleetmesh_node-0.2.0.dist-info/RECORD +338 -0
- fleetmesh_node-0.2.0.dist-info/WHEEL +5 -0
- fleetmesh_node-0.2.0.dist-info/entry_points.txt +2 -0
- fleetmesh_node-0.2.0.dist-info/licenses/LICENSE +121 -0
- fleetmesh_node-0.2.0.dist-info/top_level.txt +2 -0
- node/RELEASE +1 -0
- node/__init__.py +35 -0
- node/app/__init__.py +49 -0
- node/app/commands.py +265 -0
- node/app/config.py +131 -0
- node/app/control.py +670 -0
- node/app/core.py +867 -0
- node/app/plugins.py +76 -0
- node/app/router.py +83 -0
- node/app/socket.py +431 -0
- node/app/systemd.py +36 -0
- node/app/views.py +204 -0
- node/channels/__init__.py +378 -0
- node/channels/bus.py +331 -0
- node/channels/journal.py +46 -0
- node/commands/__init__.py +140 -0
- node/commands/adapter.py +210 -0
- node/commands/base.py +278 -0
- node/commands/cluster_cmds.py +183 -0
- node/commands/compute_cmds.py +401 -0
- node/commands/core.py +474 -0
- node/commands/gossip_cmds.py +106 -0
- node/commands/nsite_cmds.py +141 -0
- node/commands/peering.py +402 -0
- node/commands/plugin_cmds.py +135 -0
- node/commands/sidecar_cmds.py +120 -0
- node/commands/sync.py +388 -0
- node/commands/transport_cmds.py +451 -0
- node/commands/video_cmds.py +99 -0
- node/compartment/__init__.py +124 -0
- node/compartment/base.py +868 -0
- node/compartment/command.py +458 -0
- node/compartment/context.py +163 -0
- node/compartment/every.py +108 -0
- node/compartment/handler.py +131 -0
- node/compartment/lifecycle.py +460 -0
- node/compartment/lint.py +388 -0
- node/compartment/need.py +158 -0
- node/compartment/problem.py +521 -0
- node/compartment/refusals.py +120 -0
- node/compartment/registry.py +790 -0
- node/compartment/schema.py +290 -0
- node/compartment/sidecar.py +499 -0
- node/compartment/stores.py +662 -0
- node/compartment/wire.py +225 -0
- node/compartment/worker.py +61 -0
- node/config/__init__.py +43 -0
- node/config/base.py +116 -0
- node/config/change.py +441 -0
- node/config/changestore.py +201 -0
- node/config/compartments.py +1144 -0
- node/config/node.py +318 -0
- node/config/parameter.py +563 -0
- node/config/protocols.py +274 -0
- node/config/records.py +171 -0
- node/examples/__init__.py +1 -0
- node/examples/demo.py +226 -0
- node/examples/follower.py +46 -0
- node/interfaces.py +235 -0
- node/keystore/__init__.py +27 -0
- node/keystore/core.py +103 -0
- node/keystore/enclave.py +688 -0
- node/meshnode.py +111 -0
- node/metrics/__init__.py +396 -0
- node/metrics/adapters.py +102 -0
- node/metrics/face.py +110 -0
- node/planes/__init__.py +21 -0
- node/planes/identity/__init__.py +20 -0
- node/planes/identity/announce/__init__.py +9 -0
- node/planes/identity/announce/compartment.py +447 -0
- node/planes/identity/announce/types.py +26 -0
- node/planes/identity/discovery/__init__.py +9 -0
- node/planes/identity/discovery/compartment.py +522 -0
- node/planes/identity/discovery/types.py +13 -0
- node/planes/identity/linked/__init__.py +1 -0
- node/planes/identity/linked/compartment.py +270 -0
- node/planes/identity/linked/types.py +25 -0
- node/planes/identity/presence/__init__.py +7 -0
- node/planes/identity/presence/compartment.py +409 -0
- node/planes/identity/presence/types.py +14 -0
- node/planes/operations/__init__.py +13 -0
- node/planes/operations/scheduler/__init__.py +9 -0
- node/planes/operations/scheduler/compartment.py +353 -0
- node/planes/operations/scheduler/types.py +21 -0
- node/planes/operations/updates/__init__.py +17 -0
- node/planes/operations/updates/core.py +310 -0
- node/planes/operations/updates/store.py +79 -0
- node/planes/operations/updates/types.py +20 -0
- node/planes/peering/__init__.py +28 -0
- node/planes/peering/budget/__init__.py +43 -0
- node/planes/peering/budget/chain.py +43 -0
- node/planes/peering/budget/core.py +537 -0
- node/planes/peering/budget/evaluation.py +80 -0
- node/planes/peering/budget/meter.py +100 -0
- node/planes/peering/budget/receipts.py +439 -0
- node/planes/peering/budget/shape.py +101 -0
- node/planes/peering/budget/tick.py +110 -0
- node/planes/peering/budget/types.py +67 -0
- node/planes/peering/budget/windows.py +346 -0
- node/planes/peering/capacity/__init__.py +7 -0
- node/planes/peering/capacity/compartment.py +342 -0
- node/planes/peering/capacity/types.py +24 -0
- node/planes/peering/peering/__init__.py +25 -0
- node/planes/peering/peering/core.py +464 -0
- node/planes/peering/peering/envelope.py +142 -0
- node/planes/peering/peering/handshake.py +198 -0
- node/planes/peering/peering/shape.py +80 -0
- node/planes/peering/peering/types.py +39 -0
- node/planes/peering/reporter/__init__.py +42 -0
- node/planes/peering/reporter/constants.py +29 -0
- node/planes/peering/reporter/core.py +299 -0
- node/planes/peering/reporter/finding.py +74 -0
- node/planes/peering/reporter/findings.py +171 -0
- node/planes/peering/reporter/publish.py +285 -0
- node/planes/peering/reporter/reading.py +193 -0
- node/planes/peering/reporter/shape.py +86 -0
- node/planes/peering/reporter/types.py +50 -0
- node/planes/peering/strain/__init__.py +9 -0
- node/planes/peering/strain/compartment.py +204 -0
- node/planes/peering/strain/types.py +21 -0
- node/planes/peering/trust/__init__.py +9 -0
- node/planes/peering/trust/compartment.py +239 -0
- node/planes/peering/trust/types.py +30 -0
- node/planes/services/__init__.py +33 -0
- node/planes/services/compute/__init__.py +160 -0
- node/planes/services/compute/common.py +134 -0
- node/planes/services/compute/compartment.py +681 -0
- node/planes/services/compute/executors/__init__.py +121 -0
- node/planes/services/compute/executors/agent.py +840 -0
- node/planes/services/compute/executors/bacalhau.py +605 -0
- node/planes/services/compute/executors/base.py +127 -0
- node/planes/services/compute/executors/harness.py +439 -0
- node/planes/services/compute/executors/mcp.py +454 -0
- node/planes/services/compute/executors/media.py +898 -0
- node/planes/services/compute/executors/nondet.py +283 -0
- node/planes/services/compute/executors/providers.py +499 -0
- node/planes/services/compute/executors/registry.py +140 -0
- node/planes/services/compute/executors/wasm.py +244 -0
- node/planes/services/compute/executors/zk.py +604 -0
- node/planes/services/compute/jobs.py +227 -0
- node/planes/services/compute/load.py +180 -0
- node/planes/services/compute/market.py +433 -0
- node/planes/services/compute/sandbox.py +282 -0
- node/planes/services/compute/types.py +84 -0
- node/planes/services/compute_wire/__init__.py +58 -0
- node/planes/services/compute_wire/compartment.py +256 -0
- node/planes/services/compute_wire/messages.py +320 -0
- node/planes/services/compute_wire/sealing.py +147 -0
- node/planes/services/compute_wire/session.py +572 -0
- node/planes/services/compute_wire/types.py +49 -0
- node/planes/services/compute_wire/verify.py +331 -0
- node/planes/services/dvm/__init__.py +24 -0
- node/planes/services/dvm/gateway.py +526 -0
- node/planes/services/dvm/protocol.py +35 -0
- node/planes/services/dvm/translate.py +15 -0
- node/planes/services/dvm/types.py +25 -0
- node/planes/services/gateway/__init__.py +40 -0
- node/planes/services/gateway/settle.py +279 -0
- node/planes/services/gateway/translate.py +644 -0
- node/planes/services/gossip/__init__.py +22 -0
- node/planes/services/gossip/core.py +929 -0
- node/planes/services/gossip/envelope.py +154 -0
- node/planes/services/gossip/mesh.py +127 -0
- node/planes/services/gossip/types.py +50 -0
- node/planes/services/nsite/__init__.py +7 -0
- node/planes/services/nsite/client.py +62 -0
- node/planes/services/nsite/compartment.py +752 -0
- node/planes/services/nsite/types.py +44 -0
- node/planes/services/paidfetch/__init__.py +7 -0
- node/planes/services/paidfetch/compartment.py +862 -0
- node/planes/services/paidfetch/types.py +99 -0
- node/planes/services/settle/__init__.py +27 -0
- node/planes/services/settle/compartment.py +384 -0
- node/planes/services/settle/engine.py +456 -0
- node/planes/services/settle/gateways.py +541 -0
- node/planes/services/settle/payment.py +512 -0
- node/planes/services/settle/store.py +74 -0
- node/planes/services/settle/types.py +58 -0
- node/planes/services/sidecars/__init__.py +8 -0
- node/planes/services/sidecars/compartment.py +461 -0
- node/planes/services/sidecars/manifest.py +304 -0
- node/planes/services/sidecars/manifests/10-grasp.json +37 -0
- node/planes/services/sidecars/manifests/20-ncontainer.json +31 -0
- node/planes/services/sidecars/manifests/30-magpie.json +28 -0
- node/planes/services/sidecars/types.py +43 -0
- node/planes/services/video/__init__.py +106 -0
- node/planes/services/video/barter.py +407 -0
- node/planes/services/video/cli.py +158 -0
- node/planes/services/video/compartment.py +932 -0
- node/planes/services/video/ingest.py +524 -0
- node/planes/services/video/ledger.py +705 -0
- node/planes/services/video/packager.py +847 -0
- node/planes/services/video/player.py +456 -0
- node/planes/services/video/timed_segments.py +124 -0
- node/planes/services/video/transit.py +478 -0
- node/planes/store/__init__.py +24 -0
- node/planes/store/blob_replication/__init__.py +7 -0
- node/planes/store/blob_replication/compartment.py +496 -0
- node/planes/store/blob_replication/types.py +35 -0
- node/planes/store/blob_wire/__init__.py +48 -0
- node/planes/store/blob_wire/client.py +57 -0
- node/planes/store/blob_wire/core.py +359 -0
- node/planes/store/blob_wire/intake.py +74 -0
- node/planes/store/blob_wire/messages.py +85 -0
- node/planes/store/projected/__init__.py +7 -0
- node/planes/store/projected/compartment.py +258 -0
- node/planes/store/projected/types.py +29 -0
- node/planes/store/retention/__init__.py +7 -0
- node/planes/store/retention/compartment.py +214 -0
- node/planes/store/retention/types.py +12 -0
- node/planes/store/sync/__init__.py +35 -0
- node/planes/store/sync/core.py +311 -0
- node/planes/store/sync/envelope.py +106 -0
- node/planes/store/sync/handover.py +206 -0
- node/planes/store/sync/initiator.py +128 -0
- node/planes/store/sync/responder.py +114 -0
- node/planes/store/sync/rules.py +179 -0
- node/planes/store/sync/shape.py +80 -0
- node/planes/store/sync/types.py +51 -0
- node/planes/transport/__init__.py +23 -0
- node/planes/transport/bridge/__init__.py +13 -0
- node/planes/transport/bridge/compartment.py +785 -0
- node/planes/transport/bridge/ratelimit.py +44 -0
- node/planes/transport/bridge/types.py +43 -0
- node/planes/transport/lan/__init__.py +9 -0
- node/planes/transport/lan/compartment.py +446 -0
- node/planes/transport/lan/types.py +25 -0
- node/planes/transport/link/__init__.py +7 -0
- node/planes/transport/link/compartment.py +257 -0
- node/planes/transport/link/types.py +27 -0
- node/planes/transport/mediation/__init__.py +45 -0
- node/planes/transport/mediation/compartment.py +238 -0
- node/planes/transport/mediation/leaf.py +107 -0
- node/planes/transport/mediation/messages.py +302 -0
- node/planes/transport/mediation/service.py +233 -0
- node/planes/transport/mediation/store.py +37 -0
- node/planes/transport/mediation/types.py +20 -0
- node/planes/transport/p2p/__init__.py +7 -0
- node/planes/transport/p2p/compartment.py +349 -0
- node/planes/transport/p2p/types.py +22 -0
- node/protocols/__init__.py +277 -0
- node/protocols/activitypub/__init__.py +56 -0
- node/protocols/activitypub/actor.py +230 -0
- node/protocols/activitypub/federation.py +490 -0
- node/protocols/activitypub/media.py +145 -0
- node/protocols/activitypub/models.py +286 -0
- node/protocols/activitypub/package.py +271 -0
- node/protocols/activitypub/store.py +564 -0
- node/protocols/arcade/__init__.py +26 -0
- node/protocols/atproto/__init__.py +13 -0
- node/protocols/atproto/firehose.py +126 -0
- node/protocols/atproto/metering.py +165 -0
- node/protocols/atproto/package.py +155 -0
- node/protocols/atproto/repo.py +250 -0
- node/protocols/atproto/xrpc.py +285 -0
- node/protocols/base.py +109 -0
- node/protocols/circles/__init__.py +29 -0
- node/protocols/dm/__init__.py +50 -0
- node/protocols/grasp/__init__.py +22 -0
- node/protocols/isotime.py +15 -0
- node/protocols/nostr/__init__.py +51 -0
- node/protocols/nostr/blobstore.py +383 -0
- node/protocols/nostr/blossom.py +627 -0
- node/protocols/nostr/client.py +377 -0
- node/protocols/nostr/descriptor.py +211 -0
- node/protocols/nostr/eventstore.py +304 -0
- node/protocols/nostr/package.py +182 -0
- node/protocols/nostr/peers.py +109 -0
- node/protocols/nostr/relayserver.py +656 -0
- node/protocols/p2panda/__init__.py +22 -0
- node/protocols/p2panda/identity.py +19 -0
- node/protocols/p2panda/package.py +236 -0
- node/protocols/p2panda/rung.py +152 -0
- node/protocols/p2panda/store.py +273 -0
- node/protocols/rhio/__init__.py +33 -0
- node/protocols/rhio/package.py +275 -0
- node/protocols/rhio/rung.py +136 -0
- node/protocols/rhio/store.py +1423 -0
- node/protocols/ssb/muxrpc.py +256 -0
- node/protocols/ssb/package.py +267 -0
- node/protocols/textile/__init__.py +111 -0
- node/protocols/textile/transducers.py +259 -0
- node/release.py +119 -0
- node/storage/__init__.py +88 -0
- node/storage/accounts.py +305 -0
- node/storage/adapter.py +161 -0
- node/storage/base.py +385 -0
- node/storage/dialect.py +209 -0
- node/storage/grants.py +317 -0
- node/storage/locator.py +150 -0
- node/storage/postgres.py +294 -0
- node/storage/projected.py +953 -0
- node/storage/sharding.py +448 -0
- node/storage/standing.py +568 -0
- node/testnet/__init__.py +9 -0
- node/testnet/cluster.py +731 -0
- node/testnet/fips_daemon.py +230 -0
- node/transport/__init__.py +94 -0
- node/transport/const.py +110 -0
- node/transport/didcomm_inbound.py +251 -0
- node/transport/dnstxt.py +169 -0
- node/transport/fips_rung/__init__.py +67 -0
- node/transport/fips_rung/core.py +666 -0
- node/transport/fips_rung/listener.py +96 -0
- node/transport/fips_rung/types.py +12 -0
- node/transport/host.py +334 -0
- node/transport/iroh_rung/__init__.py +29 -0
- node/transport/iroh_rung/core.py +477 -0
- node/transport/iroh_rung/quic.py +942 -0
- node/transport/iroh_rung/types.py +22 -0
- node/transport/ladder.py +145 -0
- node/transport/lineproto.py +49 -0
- node/transport/mdns.py +303 -0
- node/transport/netfetch.py +283 -0
- node/transport/relay_rung/__init__.py +23 -0
- node/transport/relay_rung/cooldown.py +87 -0
- node/transport/relay_rung/core.py +138 -0
- node/transport/relay_rung/inbound.py +161 -0
- node/transport/relay_rung/ledger.py +104 -0
- node/transport/relay_rung/listen.py +121 -0
- node/transport/relay_rung/outbound.py +181 -0
- node/transport/relay_rung/shape.py +71 -0
- node/transport/relay_rung/types.py +58 -0
- node/transport/relay_rung/wire.py +63 -0
- node/transport/router.py +761 -0
- node/transport/rung.py +276 -0
- node/transport/ssb_rung.py +166 -0
- node/transport/ws.py +502 -0
- sidecar/fleetmesh_announce.py +338 -0
- sidecar/fleetmesh_p2p.py +373 -0
- sidecar/fleetmesh_rhio.py +352 -0
- sidecar/fleetmesh_ssb.py +603 -0
|
@@ -0,0 +1,1423 @@
|
|
|
1
|
+
"""Rhio blob storage and Bao-compatible chunked tree hashing.
|
|
2
|
+
|
|
3
|
+
Standard library only. Provides:
|
|
4
|
+
- Bao chunked Merkle tree verification (1024-byte chunks).
|
|
5
|
+
- Local CAS blob repository and SQLite synchronization metadata.
|
|
6
|
+
- S3 / MinIO synchronization state tracking.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import binascii
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import shutil
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import urllib.parse
|
|
20
|
+
from typing import Any, Callable, Mapping, Optional, Sequence, TypedDict
|
|
21
|
+
|
|
22
|
+
from fleetmesh_core.conformance.errors import FleetmeshError
|
|
23
|
+
from fleetmesh_core.textile import bao as _bao
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
'FIBER_KINDS',
|
|
27
|
+
'FiberAccessError',
|
|
28
|
+
'INTERACTION_KINDS',
|
|
29
|
+
'MAX_FIBER_BYTES',
|
|
30
|
+
'MAX_OWNER_BYTES',
|
|
31
|
+
'MAX_UPLOAD_BODY_BYTES',
|
|
32
|
+
'PAYLOAD_KINDS',
|
|
33
|
+
'RhioBaoTree',
|
|
34
|
+
'RhioBlobRecord',
|
|
35
|
+
'RhioBlobSync',
|
|
36
|
+
'RhioStoreError',
|
|
37
|
+
'VISIBILITIES',
|
|
38
|
+
'binary_http_response',
|
|
39
|
+
'handle_fiber_http',
|
|
40
|
+
'is_fiber_route',
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
from node.compartment.schema import Column, Index, Table
|
|
45
|
+
from node.storage import SqliteStore
|
|
46
|
+
|
|
47
|
+
MAX_FIBER_BYTES: int = int(
|
|
48
|
+
os.environ.get(
|
|
49
|
+
'MESHTUBE_MAX_FIBER_BYTES',
|
|
50
|
+
os.environ.get('MESHTUBE_ARCADE_MAX_BINARY_BYTES', str(16 * 1024 * 1024)),
|
|
51
|
+
)
|
|
52
|
+
)
|
|
53
|
+
MAX_OWNER_BYTES: int = int(
|
|
54
|
+
os.environ.get(
|
|
55
|
+
'MESHTUBE_MAX_OWNER_FIBER_BYTES',
|
|
56
|
+
os.environ.get('MESHTUBE_ARCADE_MAX_OWNER_BYTES', str(256 * 1024 * 1024)),
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
MAX_UPLOAD_BODY_BYTES: int = (MAX_FIBER_BYTES * 4) // 3 + 256 * 1024
|
|
60
|
+
|
|
61
|
+
PAYLOAD_KINDS: tuple[str, ...] = (
|
|
62
|
+
'video',
|
|
63
|
+
'audio',
|
|
64
|
+
'image',
|
|
65
|
+
'text',
|
|
66
|
+
'captions',
|
|
67
|
+
'subtitle',
|
|
68
|
+
'code',
|
|
69
|
+
'binary',
|
|
70
|
+
'metadata',
|
|
71
|
+
'log',
|
|
72
|
+
)
|
|
73
|
+
INTERACTION_KINDS: tuple[str, ...] = ('like', 'hide', 'reaction', 'scrobble', 'view')
|
|
74
|
+
FIBER_KINDS: tuple[str, ...] = PAYLOAD_KINDS + INTERACTION_KINDS + ('identity',)
|
|
75
|
+
VISIBILITIES: tuple[str, ...] = ('public', 'vault_only')
|
|
76
|
+
|
|
77
|
+
_SAFE_BINARY_MIME: frozenset[str] = frozenset({
|
|
78
|
+
'application/octet-stream',
|
|
79
|
+
'application/wasm',
|
|
80
|
+
'application/x-gameboy-rom',
|
|
81
|
+
'application/x-gameboy-color-rom',
|
|
82
|
+
'audio/mpeg',
|
|
83
|
+
'audio/ogg',
|
|
84
|
+
'audio/flac',
|
|
85
|
+
'audio/wav',
|
|
86
|
+
'video/mp4',
|
|
87
|
+
'video/webm',
|
|
88
|
+
'image/png',
|
|
89
|
+
'image/jpeg',
|
|
90
|
+
'image/webp',
|
|
91
|
+
'image/gif',
|
|
92
|
+
'text/vtt',
|
|
93
|
+
})
|
|
94
|
+
_ROOT_RE = re.compile(r'^[0-9a-f]{64}$')
|
|
95
|
+
_RANGE_HEADER_RE = re.compile(r'^bytes=(\d*)-(\d*)$')
|
|
96
|
+
JSON_CT = 'application/json; charset=utf-8'
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class RhioBlobRecord(TypedDict):
|
|
100
|
+
hash: str
|
|
101
|
+
size: int
|
|
102
|
+
tree_root: str
|
|
103
|
+
synced_s3: bool
|
|
104
|
+
created_at: str
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class RhioStoreError(FleetmeshError):
|
|
108
|
+
"""Base error for Rhio blob storage and tree hash verification."""
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class FiberAccessError(Exception):
|
|
113
|
+
"""An HTTP-mappable refusal (`status`, machine-readable `error`) for fiber operations."""
|
|
114
|
+
|
|
115
|
+
def __init__(self, status: int, error: str) -> None:
|
|
116
|
+
super().__init__(error)
|
|
117
|
+
self.status = status
|
|
118
|
+
self.error = error
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _norm_principal_key(raw: Any) -> str:
|
|
122
|
+
return str(raw or '').strip().lower().lstrip('@')
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _principal_key(principal: Any) -> str:
|
|
126
|
+
if not principal:
|
|
127
|
+
return ''
|
|
128
|
+
if isinstance(principal, str):
|
|
129
|
+
return _norm_principal_key(principal)
|
|
130
|
+
if isinstance(principal, Mapping):
|
|
131
|
+
return _norm_principal_key(principal.get('identity_id')) or _norm_principal_key(principal.get('did'))
|
|
132
|
+
return ''
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _require_principal(principal: Any) -> Mapping[str, Any]:
|
|
136
|
+
if isinstance(principal, str):
|
|
137
|
+
principal = {'identity_id': principal, 'did': principal if principal.startswith('did:') else '', 'handle': ''}
|
|
138
|
+
if not isinstance(principal, Mapping) or not _principal_key(principal):
|
|
139
|
+
raise FiberAccessError(401, 'authentication_required')
|
|
140
|
+
return principal
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _author_ref(principal: Any) -> dict[str, str]:
|
|
144
|
+
if isinstance(principal, str):
|
|
145
|
+
return {
|
|
146
|
+
'identity_id': principal,
|
|
147
|
+
'did': principal if principal.startswith('did:') else '',
|
|
148
|
+
'handle': '',
|
|
149
|
+
}
|
|
150
|
+
if not isinstance(principal, Mapping):
|
|
151
|
+
return {'identity_id': '', 'did': '', 'handle': ''}
|
|
152
|
+
return {
|
|
153
|
+
'identity_id': str(principal.get('identity_id') or ''),
|
|
154
|
+
'did': str(principal.get('did') or ''),
|
|
155
|
+
'handle': str(principal.get('handle') or ''),
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _safe_int(raw: Any, default: int = 0) -> int:
|
|
160
|
+
try:
|
|
161
|
+
return int(raw)
|
|
162
|
+
except (TypeError, ValueError):
|
|
163
|
+
return default
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _decode_b64(raw: Any, max_bytes: int) -> Optional[bytes]:
|
|
167
|
+
clean = str(raw or '').strip()
|
|
168
|
+
if not clean:
|
|
169
|
+
return None
|
|
170
|
+
try:
|
|
171
|
+
data = base64.b64decode(clean, validate=True)
|
|
172
|
+
except (binascii.Error, ValueError):
|
|
173
|
+
raise FiberAccessError(400, 'invalid_base64')
|
|
174
|
+
if len(data) > max_bytes:
|
|
175
|
+
raise FiberAccessError(413, 'binary_too_large')
|
|
176
|
+
return data
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
from node.protocols.isotime import now_iso as _now_iso # noqa: E402
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class RhioBaoTree:
|
|
183
|
+
"""Bao-compatible hierarchical chunked Merkle tree hashing."""
|
|
184
|
+
|
|
185
|
+
CHUNK_SIZE: int = 1024
|
|
186
|
+
SUBTREE_POW: int = 13 # 2^13 * 1024 B = 8 MiB exact Merkle subtree segments
|
|
187
|
+
SEGMENT_SIZE: int = CHUNK_SIZE * (1 << SUBTREE_POW) # 8,388,608 bytes (8 MiB)
|
|
188
|
+
|
|
189
|
+
@classmethod
|
|
190
|
+
def leaf_count(cls, size_bytes: int, chunk_size: int = CHUNK_SIZE) -> int:
|
|
191
|
+
"""Return the number of `chunk_size`-byte leaves for `size_bytes` (minimum 1)."""
|
|
192
|
+
return _bao.bao_leaf_count(size_bytes, chunk_size=chunk_size)
|
|
193
|
+
|
|
194
|
+
@classmethod
|
|
195
|
+
def compute_root(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> str:
|
|
196
|
+
"""Compute the Bao-compatible Merkle root hex digest."""
|
|
197
|
+
root, _ = cls.compute_tree(data, chunk_size=chunk_size)
|
|
198
|
+
return root
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def compute_tree(cls, data: bytes, chunk_size: int = CHUNK_SIZE) -> tuple[str, list[bytes]]:
|
|
202
|
+
"""Compute Merkle tree root and leaf hashes for data (`fleetmesh_core.textile.bao`)."""
|
|
203
|
+
if not data:
|
|
204
|
+
return _bao.EMPTY_BAO_ROOT_HEX, [hashlib.sha256(b'\x00').digest()]
|
|
205
|
+
leaves = [hashlib.sha256(b'\x00' + data[i:i + chunk_size]).digest() for i in range(0, len(data), chunk_size)]
|
|
206
|
+
return _bao._fold(list(leaves)).hex(), leaves
|
|
207
|
+
|
|
208
|
+
@classmethod
|
|
209
|
+
def promote_segment_root(
|
|
210
|
+
cls,
|
|
211
|
+
segment_root_hex: str,
|
|
212
|
+
segment_size: int,
|
|
213
|
+
*,
|
|
214
|
+
is_single_segment: bool,
|
|
215
|
+
chunk_size: int = CHUNK_SIZE,
|
|
216
|
+
subtree_pow: int = SUBTREE_POW,
|
|
217
|
+
) -> str:
|
|
218
|
+
"""Promote a standalone segment Bao root to its Level-`subtree_pow` subtree root."""
|
|
219
|
+
return _bao.promote_segment_root(
|
|
220
|
+
segment_root_hex, segment_size, is_single_segment=is_single_segment, chunk_size=chunk_size, subtree_pow=subtree_pow
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
@classmethod
|
|
224
|
+
def combine_segment_roots(
|
|
225
|
+
cls,
|
|
226
|
+
segments: Sequence[tuple[str, int]],
|
|
227
|
+
*,
|
|
228
|
+
chunk_size: int = CHUNK_SIZE,
|
|
229
|
+
subtree_pow: int = SUBTREE_POW,
|
|
230
|
+
) -> str:
|
|
231
|
+
"""Combine `(segment_bao_root_hex, segment_byte_size)` pairs into the global Bao tree root."""
|
|
232
|
+
return _bao.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow)
|
|
233
|
+
|
|
234
|
+
@classmethod
|
|
235
|
+
def compute_file_root(
|
|
236
|
+
cls,
|
|
237
|
+
file_path: str,
|
|
238
|
+
*,
|
|
239
|
+
chunk_size: int = CHUNK_SIZE,
|
|
240
|
+
subtree_pow: int = SUBTREE_POW,
|
|
241
|
+
on_segment: Optional[Callable[[int, int, int, int], None]] = None,
|
|
242
|
+
) -> tuple[str, str, int]:
|
|
243
|
+
"""Stream `file_path` in `2^subtree_pow * chunk_size` (8 MiB) segments and return `(sha256_hex, tree_root_hex, total_size)`."""
|
|
244
|
+
seg_max = chunk_size * (1 << subtree_pow)
|
|
245
|
+
file_size = os.path.getsize(file_path)
|
|
246
|
+
total_segments = max(1, (file_size + seg_max - 1) // seg_max) if file_size > 0 else 0
|
|
247
|
+
sha = hashlib.sha256()
|
|
248
|
+
segments: list[tuple[str, int]] = []
|
|
249
|
+
total_size = 0
|
|
250
|
+
with open(file_path, 'rb') as f:
|
|
251
|
+
while True:
|
|
252
|
+
buf = f.read(seg_max)
|
|
253
|
+
if not buf:
|
|
254
|
+
break
|
|
255
|
+
sha.update(buf)
|
|
256
|
+
total_size += len(buf)
|
|
257
|
+
seg_root = cls.compute_root(buf, chunk_size=chunk_size)
|
|
258
|
+
segments.append((seg_root, len(buf)))
|
|
259
|
+
if on_segment is not None:
|
|
260
|
+
on_segment(len(segments), max(len(segments), total_segments), total_size, file_size)
|
|
261
|
+
if total_size == 0:
|
|
262
|
+
empty_root = hashlib.sha256(b'\x00').hexdigest()
|
|
263
|
+
return sha.hexdigest(), empty_root, 0
|
|
264
|
+
tree_root = cls.combine_segment_roots(segments, chunk_size=chunk_size, subtree_pow=subtree_pow)
|
|
265
|
+
return sha.hexdigest(), tree_root, total_size
|
|
266
|
+
|
|
267
|
+
@classmethod
|
|
268
|
+
def verify(cls, data: bytes, expected_root: str, chunk_size: int = CHUNK_SIZE) -> bool:
|
|
269
|
+
"""Verify that data produces expected Merkle tree root."""
|
|
270
|
+
actual_root, _ = cls.compute_tree(data, chunk_size=chunk_size)
|
|
271
|
+
return actual_root.lower() == expected_root.lower().strip()
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class RhioBlobSync(SqliteStore):
|
|
275
|
+
"""Metadata and CAS storage for Rhio cluster blobs.
|
|
276
|
+
|
|
277
|
+
A `SqliteStore` rather than a hand-rolled `sqlite3.connect`: the base owns
|
|
278
|
+
the connection, the lock, the schema version (per store, not per database -
|
|
279
|
+
which matters the moment two stores share one server), transaction control
|
|
280
|
+
and the dialect that rewrites `?` and `INSERT OR IGNORE` for PostgreSQL.
|
|
281
|
+
This store opened its own connection per call and declared its schema as a
|
|
282
|
+
SQL string, so it was SQLite-only and invisible to the locator - see
|
|
283
|
+
docs/storage-backends.md.
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
SCHEMA = (
|
|
287
|
+
Table('rhio_blobs', (
|
|
288
|
+
Column('hash', 'TEXT', primary_key=True),
|
|
289
|
+
Column('size', 'INTEGER', not_null=True),
|
|
290
|
+
Column('tree_root', 'TEXT', not_null=True),
|
|
291
|
+
Column('synced_s3', 'INTEGER', not_null=True, default='0'),
|
|
292
|
+
Column('created_at', 'TEXT', not_null=True),
|
|
293
|
+
), indices=(
|
|
294
|
+
Index('idx_rhio_blobs_tree', ('tree_root',)),
|
|
295
|
+
)),
|
|
296
|
+
Table('rhio_fiber_claims', (
|
|
297
|
+
Column('fiber_root', 'TEXT', not_null=True),
|
|
298
|
+
Column('author_key', 'TEXT', not_null=True),
|
|
299
|
+
Column('payload_json', 'TEXT', not_null=True),
|
|
300
|
+
Column('created_at', 'INTEGER', not_null=True),
|
|
301
|
+
), primary_key=('fiber_root', 'author_key'), indices=(
|
|
302
|
+
Index('idx_rhio_fiber_claims_root', ('fiber_root',)),
|
|
303
|
+
Index('idx_rhio_fiber_claims_author', ('author_key',)),
|
|
304
|
+
)),
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
_EXTERNAL_FILE_THRESHOLD: int = 32 * 1024 * 1024 # 32 MiB
|
|
308
|
+
|
|
309
|
+
def __init__(
|
|
310
|
+
self,
|
|
311
|
+
db_path: Optional[str] = ":memory:",
|
|
312
|
+
cas_dir: Optional[str] = None,
|
|
313
|
+
overflow_cas_dirs: Optional[Sequence[str]] = None,
|
|
314
|
+
):
|
|
315
|
+
db_path = (db_path or ":memory:").strip() or ":memory:"
|
|
316
|
+
# A file-backed store with no cas_dir kept the bytes in a process dict
|
|
317
|
+
# while persisting the row that says it holds them. After a restart
|
|
318
|
+
# has_blob() was true, count_blobs() counted it, synced_s3 said the
|
|
319
|
+
# cluster already had a copy - and get_blob() returned nothing. Two of
|
|
320
|
+
# the three callers passed a directory; the video compartment did not,
|
|
321
|
+
# and nothing could tell, because every suite used `:memory:` where one
|
|
322
|
+
# connection reads back whatever was written.
|
|
323
|
+
#
|
|
324
|
+
# So derive it rather than trust each caller to remember: a persistent
|
|
325
|
+
# database gets a persistent CAS beside it. The dict stays for
|
|
326
|
+
# `:memory:` alone, where losing the bytes with the rows is honest.
|
|
327
|
+
if db_path != ":memory:":
|
|
328
|
+
os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True)
|
|
329
|
+
if cas_dir is None:
|
|
330
|
+
cas_dir = os.path.splitext(os.path.abspath(db_path))[0] + '_blobs'
|
|
331
|
+
self._cas_dir = cas_dir
|
|
332
|
+
self._mem_blobs: dict[str, bytes] = {}
|
|
333
|
+
self._external_files: dict[str, str] = {}
|
|
334
|
+
self._claims: dict[str, dict[str, dict[str, Any]]] = {}
|
|
335
|
+
self._by_target: dict[str, set[tuple[str, str]]] = {}
|
|
336
|
+
self._root_meta: dict[str, RhioBlobRecord] = {}
|
|
337
|
+
if cas_dir is not None:
|
|
338
|
+
os.makedirs(cas_dir, exist_ok=True)
|
|
339
|
+
raw_env_ov = (
|
|
340
|
+
os.environ.get('MESHTUBE_OVERFLOW_CAS_DIRS')
|
|
341
|
+
or os.environ.get('RHIO_OVERFLOW_CAS_DIRS')
|
|
342
|
+
or ''
|
|
343
|
+
).strip()
|
|
344
|
+
init_ov: list[str] = []
|
|
345
|
+
if overflow_cas_dirs:
|
|
346
|
+
init_ov.extend(str(d).strip() for d in overflow_cas_dirs if str(d).strip())
|
|
347
|
+
elif raw_env_ov:
|
|
348
|
+
for part in raw_env_ov.replace(',', ':').split(':'):
|
|
349
|
+
p = part.strip()
|
|
350
|
+
if p:
|
|
351
|
+
init_ov.append(p)
|
|
352
|
+
self._overflow_cas_dirs: list[str] = []
|
|
353
|
+
# Local files evict_local_file could not delete (permissions, a
|
|
354
|
+
# concurrent remover); the metadata row is kept either way.
|
|
355
|
+
self.evict_errors: int = 0
|
|
356
|
+
self._primary_min_free_bytes: int = int(
|
|
357
|
+
os.environ.get('MESHTUBE_PRIMARY_MIN_FREE_BYTES', str(5 * 1024 * 1024 * 1024))
|
|
358
|
+
)
|
|
359
|
+
super().__init__(db_path)
|
|
360
|
+
if init_ov:
|
|
361
|
+
self.set_overflow_cas_dirs(init_ov)
|
|
362
|
+
|
|
363
|
+
def _post_schema(self) -> None:
|
|
364
|
+
for h_raw, sz_raw, root_raw, s3_raw, cat_raw in self._all(
|
|
365
|
+
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs"
|
|
366
|
+
):
|
|
367
|
+
r = str(root_raw or '').strip().lower()
|
|
368
|
+
if r:
|
|
369
|
+
self._root_meta[r] = {
|
|
370
|
+
"hash": str(h_raw),
|
|
371
|
+
"size": int(sz_raw),
|
|
372
|
+
"tree_root": r,
|
|
373
|
+
"synced_s3": bool(s3_raw),
|
|
374
|
+
"created_at": str(cat_raw),
|
|
375
|
+
}
|
|
376
|
+
for root_raw, akey_raw, pjson in self._all(
|
|
377
|
+
"SELECT fiber_root, author_key, payload_json FROM rhio_fiber_claims"
|
|
378
|
+
):
|
|
379
|
+
try:
|
|
380
|
+
claim = json.loads(str(pjson))
|
|
381
|
+
except (TypeError, ValueError):
|
|
382
|
+
continue
|
|
383
|
+
root = str(root_raw or '').strip().lower()
|
|
384
|
+
akey = str(akey_raw or '').strip().lower()
|
|
385
|
+
if isinstance(claim, dict) and root and akey:
|
|
386
|
+
self._claims.setdefault(root, {})[akey] = claim
|
|
387
|
+
self._index_target(root, akey, claim)
|
|
388
|
+
|
|
389
|
+
def set_overflow_cas_dirs(self, dirs: Sequence[str]) -> list[str]:
|
|
390
|
+
"""Configure secondary/overflow CAS directories (e.g. attached block storage volumes)."""
|
|
391
|
+
cleaned: list[str] = []
|
|
392
|
+
primary_abs = os.path.abspath(self._cas_dir) if self._cas_dir else ''
|
|
393
|
+
with self._lock:
|
|
394
|
+
for raw_d in dirs:
|
|
395
|
+
d = str(raw_d or '').strip()
|
|
396
|
+
if not d:
|
|
397
|
+
continue
|
|
398
|
+
abs_d = os.path.abspath(d)
|
|
399
|
+
if abs_d == primary_abs or abs_d in cleaned:
|
|
400
|
+
continue
|
|
401
|
+
try:
|
|
402
|
+
os.makedirs(abs_d, exist_ok=True)
|
|
403
|
+
cleaned.append(abs_d)
|
|
404
|
+
except OSError:
|
|
405
|
+
continue
|
|
406
|
+
self._overflow_cas_dirs = cleaned
|
|
407
|
+
return list(cleaned)
|
|
408
|
+
|
|
409
|
+
def get_overflow_cas_dirs(self) -> list[str]:
|
|
410
|
+
with self._lock:
|
|
411
|
+
return list(self._overflow_cas_dirs)
|
|
412
|
+
|
|
413
|
+
def _select_target_cas_dir(self, required_bytes: int = 0) -> Optional[str]:
|
|
414
|
+
"""Return `self._cas_dir` when it has sufficient free space, or fail over to `_overflow_cas_dirs`."""
|
|
415
|
+
if not self._cas_dir:
|
|
416
|
+
return None
|
|
417
|
+
need = max(int(required_bytes) + 64 * 1024 * 1024, self._primary_min_free_bytes)
|
|
418
|
+
if not self._overflow_cas_dirs:
|
|
419
|
+
return self._cas_dir
|
|
420
|
+
try:
|
|
421
|
+
pri_free = shutil.disk_usage(self._cas_dir).free
|
|
422
|
+
if pri_free >= need:
|
|
423
|
+
return self._cas_dir
|
|
424
|
+
except OSError:
|
|
425
|
+
pri_free = 0
|
|
426
|
+
best_dir = self._cas_dir
|
|
427
|
+
best_free = pri_free
|
|
428
|
+
for ov_dir in list(self._overflow_cas_dirs):
|
|
429
|
+
try:
|
|
430
|
+
os.makedirs(ov_dir, exist_ok=True)
|
|
431
|
+
ov_free = shutil.disk_usage(ov_dir).free
|
|
432
|
+
if ov_free >= int(required_bytes) + 64 * 1024 * 1024 and ov_free > best_free:
|
|
433
|
+
return ov_dir
|
|
434
|
+
if ov_free > best_free:
|
|
435
|
+
best_free = ov_free
|
|
436
|
+
best_dir = ov_dir
|
|
437
|
+
except OSError:
|
|
438
|
+
continue
|
|
439
|
+
return best_dir
|
|
440
|
+
|
|
441
|
+
def store_blob(
|
|
442
|
+
self,
|
|
443
|
+
data: bytes,
|
|
444
|
+
*,
|
|
445
|
+
sha256_hex: Optional[str] = None,
|
|
446
|
+
tree_root: Optional[str] = None,
|
|
447
|
+
chunk_count: Optional[int] = None,
|
|
448
|
+
) -> tuple[str, str, int]:
|
|
449
|
+
"""Store blob, computing sha256 hash and Bao tree root if not provided.
|
|
450
|
+
|
|
451
|
+
Returns (sha256_hash, tree_root, size).
|
|
452
|
+
"""
|
|
453
|
+
blob_hash = sha256_hex.strip().lower() if sha256_hex else hashlib.sha256(data).hexdigest()
|
|
454
|
+
final_root = tree_root.strip().lower() if tree_root else RhioBaoTree.compute_root(data)
|
|
455
|
+
size = len(data)
|
|
456
|
+
_ = chunk_count
|
|
457
|
+
|
|
458
|
+
with self._lock:
|
|
459
|
+
# Save raw bytes
|
|
460
|
+
if self._cas_dir:
|
|
461
|
+
target_dir = self._select_target_cas_dir(required_bytes=size) or self._cas_dir
|
|
462
|
+
path = os.path.join(target_dir, blob_hash)
|
|
463
|
+
if not os.path.exists(path):
|
|
464
|
+
try:
|
|
465
|
+
with open(path, "wb") as f:
|
|
466
|
+
f.write(data)
|
|
467
|
+
except OSError:
|
|
468
|
+
for ov_dir in self._overflow_cas_dirs:
|
|
469
|
+
if ov_dir == target_dir:
|
|
470
|
+
continue
|
|
471
|
+
try:
|
|
472
|
+
ov_path = os.path.join(ov_dir, blob_hash)
|
|
473
|
+
with open(ov_path, "wb") as f:
|
|
474
|
+
f.write(data)
|
|
475
|
+
break
|
|
476
|
+
except OSError:
|
|
477
|
+
continue
|
|
478
|
+
else:
|
|
479
|
+
raise
|
|
480
|
+
else:
|
|
481
|
+
self._mem_blobs[blob_hash] = data
|
|
482
|
+
|
|
483
|
+
# INSERT OR IGNORE preserves synced_s3 and the original created_at.
|
|
484
|
+
# The dialect rewrites it for a backend that spells it differently.
|
|
485
|
+
created_iso = _now_iso()
|
|
486
|
+
with self.transaction() as cur:
|
|
487
|
+
cur.execute(
|
|
488
|
+
"""
|
|
489
|
+
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
|
|
490
|
+
VALUES (?, ?, ?, 0, ?)
|
|
491
|
+
""",
|
|
492
|
+
(blob_hash, size, final_root, created_iso),
|
|
493
|
+
)
|
|
494
|
+
prev_meta = self._root_meta.get(final_root)
|
|
495
|
+
self._root_meta[final_root] = {
|
|
496
|
+
"hash": blob_hash,
|
|
497
|
+
"size": size,
|
|
498
|
+
"tree_root": final_root,
|
|
499
|
+
"synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False,
|
|
500
|
+
"created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso,
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
return blob_hash, final_root, size
|
|
504
|
+
|
|
505
|
+
def store_blob_from_file(
|
|
506
|
+
self,
|
|
507
|
+
file_path: str,
|
|
508
|
+
*,
|
|
509
|
+
sha256_hex: Optional[str] = None,
|
|
510
|
+
tree_root: Optional[str] = None,
|
|
511
|
+
size: Optional[int] = None,
|
|
512
|
+
move: bool = False,
|
|
513
|
+
chunk_count: Optional[int] = None,
|
|
514
|
+
on_segment: Optional[Callable[[int, int, int, int], None]] = None,
|
|
515
|
+
) -> tuple[str, str, int]:
|
|
516
|
+
"""Store a file-backed blob into CAS without loading the full file into memory."""
|
|
517
|
+
_ = chunk_count
|
|
518
|
+
if not sha256_hex or not tree_root or size is None:
|
|
519
|
+
comp_sha, comp_root, comp_size = RhioBaoTree.compute_file_root(
|
|
520
|
+
file_path, on_segment=on_segment
|
|
521
|
+
)
|
|
522
|
+
blob_hash = (sha256_hex or comp_sha).lower().strip()
|
|
523
|
+
final_root = (tree_root or comp_root).lower().strip()
|
|
524
|
+
final_size = int(size if size is not None else comp_size)
|
|
525
|
+
else:
|
|
526
|
+
blob_hash = sha256_hex.lower().strip()
|
|
527
|
+
final_root = tree_root.lower().strip()
|
|
528
|
+
final_size = int(size)
|
|
529
|
+
|
|
530
|
+
if self._cas_dir:
|
|
531
|
+
with self._lock:
|
|
532
|
+
overflow_dirs = list(self._overflow_cas_dirs)
|
|
533
|
+
target_cas_dir = self._select_target_cas_dir(required_bytes=final_size) or self._cas_dir
|
|
534
|
+
dest_path = os.path.join(target_cas_dir, blob_hash)
|
|
535
|
+
src_abs = os.path.abspath(file_path)
|
|
536
|
+
dst_abs = os.path.abspath(dest_path)
|
|
537
|
+
if src_abs != dst_abs:
|
|
538
|
+
need_stage = True
|
|
539
|
+
existing_loc = self.get_blob_path(blob_hash)
|
|
540
|
+
if existing_loc and os.path.isfile(existing_loc):
|
|
541
|
+
try:
|
|
542
|
+
if os.path.getsize(existing_loc) == final_size:
|
|
543
|
+
need_stage = False
|
|
544
|
+
except OSError:
|
|
545
|
+
need_stage = True
|
|
546
|
+
if need_stage:
|
|
547
|
+
placed = False
|
|
548
|
+
cas_parent = os.path.dirname(os.path.abspath(self._cas_dir))
|
|
549
|
+
alt_candidates = [
|
|
550
|
+
os.path.join(cas_parent, 'uploads', os.path.basename(file_path)),
|
|
551
|
+
os.path.join(cas_parent, 'uploads', 'staging', os.path.basename(file_path)),
|
|
552
|
+
]
|
|
553
|
+
if move:
|
|
554
|
+
for cand_src in (file_path, *alt_candidates):
|
|
555
|
+
if not os.path.isfile(cand_src):
|
|
556
|
+
continue
|
|
557
|
+
try:
|
|
558
|
+
os.rename(cand_src, dest_path)
|
|
559
|
+
placed = True
|
|
560
|
+
if cand_src != file_path and os.path.exists(file_path):
|
|
561
|
+
try:
|
|
562
|
+
os.unlink(file_path)
|
|
563
|
+
except OSError:
|
|
564
|
+
pass
|
|
565
|
+
break
|
|
566
|
+
except OSError:
|
|
567
|
+
placed = False
|
|
568
|
+
else:
|
|
569
|
+
for cand_src in (file_path, *alt_candidates):
|
|
570
|
+
if not os.path.isfile(cand_src):
|
|
571
|
+
continue
|
|
572
|
+
for link_dir in (target_cas_dir, self._cas_dir, *overflow_dirs):
|
|
573
|
+
if not link_dir:
|
|
574
|
+
continue
|
|
575
|
+
cand_dest = os.path.join(link_dir, blob_hash)
|
|
576
|
+
try:
|
|
577
|
+
if os.path.getsize(cand_src) != final_size:
|
|
578
|
+
continue
|
|
579
|
+
if os.path.exists(cand_dest):
|
|
580
|
+
os.unlink(cand_dest)
|
|
581
|
+
os.link(cand_src, cand_dest)
|
|
582
|
+
dest_path = cand_dest
|
|
583
|
+
placed = True
|
|
584
|
+
break
|
|
585
|
+
except OSError:
|
|
586
|
+
placed = False
|
|
587
|
+
if placed:
|
|
588
|
+
break
|
|
589
|
+
if not placed:
|
|
590
|
+
candidate_dirs = [target_cas_dir] + [
|
|
591
|
+
d for d in (self._cas_dir, *overflow_dirs) if d and d != target_cas_dir
|
|
592
|
+
]
|
|
593
|
+
last_err: Optional[OSError] = None
|
|
594
|
+
for write_dir in candidate_dirs:
|
|
595
|
+
cand_dest = os.path.join(write_dir, blob_hash)
|
|
596
|
+
tmp_path = f"{cand_dest}.tmp.{os.getpid()}.{threading.get_ident()}"
|
|
597
|
+
try:
|
|
598
|
+
shutil.copyfile(file_path, tmp_path)
|
|
599
|
+
os.replace(tmp_path, cand_dest)
|
|
600
|
+
dest_path = cand_dest
|
|
601
|
+
placed = True
|
|
602
|
+
break
|
|
603
|
+
except OSError as exc:
|
|
604
|
+
last_err = exc
|
|
605
|
+
finally:
|
|
606
|
+
if os.path.exists(tmp_path):
|
|
607
|
+
try:
|
|
608
|
+
os.unlink(tmp_path)
|
|
609
|
+
except OSError:
|
|
610
|
+
pass
|
|
611
|
+
if not placed:
|
|
612
|
+
if not move and os.path.isfile(file_path):
|
|
613
|
+
with self._lock:
|
|
614
|
+
self._external_files[blob_hash] = os.path.abspath(file_path)
|
|
615
|
+
placed = True
|
|
616
|
+
elif last_err is not None:
|
|
617
|
+
raise last_err
|
|
618
|
+
if move and os.path.exists(file_path):
|
|
619
|
+
try:
|
|
620
|
+
os.unlink(file_path)
|
|
621
|
+
except OSError:
|
|
622
|
+
pass
|
|
623
|
+
elif move and os.path.exists(file_path):
|
|
624
|
+
try:
|
|
625
|
+
os.unlink(file_path)
|
|
626
|
+
except OSError:
|
|
627
|
+
pass
|
|
628
|
+
|
|
629
|
+
with self._lock:
|
|
630
|
+
if not self._cas_dir:
|
|
631
|
+
if final_size > self._EXTERNAL_FILE_THRESHOLD:
|
|
632
|
+
self._external_files[blob_hash] = os.path.abspath(file_path)
|
|
633
|
+
else:
|
|
634
|
+
with open(file_path, "rb") as f:
|
|
635
|
+
self._mem_blobs[blob_hash] = f.read()
|
|
636
|
+
if move and os.path.exists(file_path):
|
|
637
|
+
try:
|
|
638
|
+
os.unlink(file_path)
|
|
639
|
+
except OSError:
|
|
640
|
+
pass
|
|
641
|
+
|
|
642
|
+
created_iso = _now_iso()
|
|
643
|
+
with self.transaction() as cur:
|
|
644
|
+
cur.execute(
|
|
645
|
+
"""
|
|
646
|
+
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
|
|
647
|
+
VALUES (?, ?, ?, 0, ?)
|
|
648
|
+
""",
|
|
649
|
+
(blob_hash, final_size, final_root, created_iso),
|
|
650
|
+
)
|
|
651
|
+
prev_meta = self._root_meta.get(final_root)
|
|
652
|
+
self._root_meta[final_root] = {
|
|
653
|
+
"hash": blob_hash,
|
|
654
|
+
"size": final_size,
|
|
655
|
+
"tree_root": final_root,
|
|
656
|
+
"synced_s3": bool(prev_meta.get("synced_s3")) if prev_meta else False,
|
|
657
|
+
"created_at": str(prev_meta.get("created_at") or created_iso) if prev_meta else created_iso,
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
return blob_hash, final_root, final_size
|
|
661
|
+
|
|
662
|
+
def get_blob_path(self, blob_hash: str) -> Optional[str]:
|
|
663
|
+
"""Return the filesystem path of a stored CAS blob if backed by `cas_dir`, `_overflow_cas_dirs`, or an external file."""
|
|
664
|
+
clean_hash = blob_hash.lower().strip()
|
|
665
|
+
if not clean_hash:
|
|
666
|
+
return None
|
|
667
|
+
with self._lock:
|
|
668
|
+
if self._cas_dir:
|
|
669
|
+
path = os.path.join(self._cas_dir, clean_hash)
|
|
670
|
+
if os.path.isfile(path):
|
|
671
|
+
return path
|
|
672
|
+
for ov_dir in self._overflow_cas_dirs:
|
|
673
|
+
ov_path = os.path.join(ov_dir, clean_hash)
|
|
674
|
+
if os.path.isfile(ov_path):
|
|
675
|
+
return ov_path
|
|
676
|
+
ext_path = self._external_files.get(clean_hash)
|
|
677
|
+
if ext_path and os.path.isfile(ext_path):
|
|
678
|
+
return ext_path
|
|
679
|
+
return None
|
|
680
|
+
|
|
681
|
+
def has_local_blob(self, blob_hash: str) -> bool:
|
|
682
|
+
"""Return True if the raw bytes for `blob_hash` are currently present on local/overflow disk or in memory."""
|
|
683
|
+
clean_hash = blob_hash.lower().strip()
|
|
684
|
+
if not clean_hash:
|
|
685
|
+
return False
|
|
686
|
+
if self.get_blob_path(clean_hash) is not None:
|
|
687
|
+
return True
|
|
688
|
+
with self._lock:
|
|
689
|
+
return clean_hash in self._mem_blobs
|
|
690
|
+
|
|
691
|
+
def migrate_blob_to_overflow(
|
|
692
|
+
self,
|
|
693
|
+
blob_hash: str,
|
|
694
|
+
target_dir: Optional[str] = None,
|
|
695
|
+
) -> Optional[str]:
|
|
696
|
+
"""Move `blob_hash` from primary `_cas_dir` to an overflow CAS volume and return the new path."""
|
|
697
|
+
clean_hash = blob_hash.lower().strip()
|
|
698
|
+
if not clean_hash or not self._cas_dir:
|
|
699
|
+
return None
|
|
700
|
+
pri_path = os.path.join(self._cas_dir, clean_hash)
|
|
701
|
+
if not os.path.isfile(pri_path):
|
|
702
|
+
return self.get_blob_path(clean_hash)
|
|
703
|
+
try:
|
|
704
|
+
fsize = os.path.getsize(pri_path)
|
|
705
|
+
except OSError:
|
|
706
|
+
return None
|
|
707
|
+
|
|
708
|
+
candidates = [target_dir] if target_dir else list(self._overflow_cas_dirs)
|
|
709
|
+
for ov_dir in candidates:
|
|
710
|
+
if not ov_dir:
|
|
711
|
+
continue
|
|
712
|
+
abs_ov = os.path.abspath(ov_dir)
|
|
713
|
+
if abs_ov == os.path.abspath(self._cas_dir):
|
|
714
|
+
continue
|
|
715
|
+
try:
|
|
716
|
+
os.makedirs(abs_ov, exist_ok=True)
|
|
717
|
+
if shutil.disk_usage(abs_ov).free < fsize + 32 * 1024 * 1024:
|
|
718
|
+
continue
|
|
719
|
+
dest_path = os.path.join(abs_ov, clean_hash)
|
|
720
|
+
if not os.path.isfile(dest_path) or os.path.getsize(dest_path) != fsize:
|
|
721
|
+
tmp_path = f"{dest_path}.mig.{os.getpid()}.{threading.get_ident()}"
|
|
722
|
+
try:
|
|
723
|
+
shutil.copyfile(pri_path, tmp_path)
|
|
724
|
+
os.replace(tmp_path, dest_path)
|
|
725
|
+
finally:
|
|
726
|
+
if os.path.exists(tmp_path):
|
|
727
|
+
try:
|
|
728
|
+
os.unlink(tmp_path)
|
|
729
|
+
except OSError:
|
|
730
|
+
pass
|
|
731
|
+
if os.path.isfile(dest_path) and os.path.getsize(dest_path) == fsize:
|
|
732
|
+
with self._lock:
|
|
733
|
+
if abs_ov not in self._overflow_cas_dirs:
|
|
734
|
+
self._overflow_cas_dirs.append(abs_ov)
|
|
735
|
+
try:
|
|
736
|
+
os.unlink(pri_path)
|
|
737
|
+
except OSError:
|
|
738
|
+
pass
|
|
739
|
+
return dest_path
|
|
740
|
+
except OSError:
|
|
741
|
+
continue
|
|
742
|
+
return None
|
|
743
|
+
|
|
744
|
+
def _remove_file(self, path: str) -> int:
|
|
745
|
+
"""Delete `path` and return the bytes freed; a failure is counted in evict_errors."""
|
|
746
|
+
try:
|
|
747
|
+
size = os.path.getsize(path)
|
|
748
|
+
os.unlink(path)
|
|
749
|
+
except OSError:
|
|
750
|
+
self.evict_errors += 1
|
|
751
|
+
return 0
|
|
752
|
+
return size
|
|
753
|
+
|
|
754
|
+
def evict_local_file(self, blob_hash: str, *, include_overflow: bool = False) -> int:
|
|
755
|
+
"""Evict the local file for `blob_hash` to free disk space while preserving its `rhio_blobs` metadata row."""
|
|
756
|
+
clean_hash = blob_hash.lower().strip()
|
|
757
|
+
if not clean_hash:
|
|
758
|
+
return 0
|
|
759
|
+
freed_bytes = 0
|
|
760
|
+
with self._lock:
|
|
761
|
+
dirs_to_check = [self._cas_dir] if self._cas_dir else []
|
|
762
|
+
if include_overflow:
|
|
763
|
+
dirs_to_check.extend(self._overflow_cas_dirs)
|
|
764
|
+
for d in dirs_to_check:
|
|
765
|
+
if not d:
|
|
766
|
+
continue
|
|
767
|
+
path = os.path.join(d, clean_hash)
|
|
768
|
+
if os.path.isfile(path):
|
|
769
|
+
freed_bytes += self._remove_file(path)
|
|
770
|
+
if clean_hash in self._mem_blobs:
|
|
771
|
+
freed_bytes += len(self._mem_blobs.pop(clean_hash))
|
|
772
|
+
ext_path = self._external_files.pop(clean_hash, None)
|
|
773
|
+
if ext_path and os.path.isfile(ext_path):
|
|
774
|
+
freed_bytes += self._remove_file(ext_path)
|
|
775
|
+
return freed_bytes
|
|
776
|
+
|
|
777
|
+
def get_blob(self, blob_hash: str) -> Optional[bytes]:
|
|
778
|
+
"""Retrieve blob bytes by sha256 hash."""
|
|
779
|
+
clean_hash = blob_hash.lower().strip()
|
|
780
|
+
path = self.get_blob_path(clean_hash)
|
|
781
|
+
if path and os.path.isfile(path):
|
|
782
|
+
with open(path, "rb") as f:
|
|
783
|
+
return f.read()
|
|
784
|
+
with self._lock:
|
|
785
|
+
if clean_hash in self._mem_blobs:
|
|
786
|
+
return self._mem_blobs[clean_hash]
|
|
787
|
+
return None
|
|
788
|
+
|
|
789
|
+
def get_blob_meta(self, blob_hash: str) -> Optional[RhioBlobRecord]:
|
|
790
|
+
"""Retrieve stored metadata (hash, size, tree_root, synced_s3, created_at) without reading the blob."""
|
|
791
|
+
clean_hash = blob_hash.lower().strip()
|
|
792
|
+
if not clean_hash:
|
|
793
|
+
return None
|
|
794
|
+
with self._lock:
|
|
795
|
+
row = self._one(
|
|
796
|
+
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs WHERE hash = ?",
|
|
797
|
+
(clean_hash,),
|
|
798
|
+
)
|
|
799
|
+
if not row:
|
|
800
|
+
return None
|
|
801
|
+
return {
|
|
802
|
+
"hash": str(row[0]),
|
|
803
|
+
"size": int(row[1]),
|
|
804
|
+
"tree_root": str(row[2]),
|
|
805
|
+
"synced_s3": bool(row[3]),
|
|
806
|
+
"created_at": str(row[4]),
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
def get_blob_slice(self, blob_hash: str, start: int, length: int) -> Optional[bytes]:
|
|
810
|
+
"""Read a byte range `[start : start + length]` directly from CAS disk without loading the full blob into RAM."""
|
|
811
|
+
clean_hash = blob_hash.lower().strip()
|
|
812
|
+
if start < 0 or length < 0:
|
|
813
|
+
return None
|
|
814
|
+
path = self.get_blob_path(clean_hash)
|
|
815
|
+
if path and os.path.isfile(path):
|
|
816
|
+
with open(path, "rb") as f:
|
|
817
|
+
f.seek(start)
|
|
818
|
+
return f.read(length)
|
|
819
|
+
with self._lock:
|
|
820
|
+
raw = self._mem_blobs.get(clean_hash)
|
|
821
|
+
if raw is None:
|
|
822
|
+
return None
|
|
823
|
+
return raw[start : start + length]
|
|
824
|
+
|
|
825
|
+
def get_blob_meta_by_root(self, tree_root: str) -> Optional[RhioBlobRecord]:
|
|
826
|
+
"""Retrieve stored blob metadata by its 1,024-byte-leaf Bao `tree_root`."""
|
|
827
|
+
clean_root = str(tree_root or '').lower().strip()
|
|
828
|
+
if not clean_root:
|
|
829
|
+
return None
|
|
830
|
+
with self._lock:
|
|
831
|
+
cached = self._root_meta.get(clean_root)
|
|
832
|
+
if cached is not None:
|
|
833
|
+
return dict(cached) # type: ignore[return-value]
|
|
834
|
+
row = self._one(
|
|
835
|
+
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs WHERE tree_root = ?",
|
|
836
|
+
(clean_root,),
|
|
837
|
+
)
|
|
838
|
+
if not row:
|
|
839
|
+
return None
|
|
840
|
+
rec: RhioBlobRecord = {
|
|
841
|
+
"hash": str(row[0]),
|
|
842
|
+
"size": int(row[1]),
|
|
843
|
+
"tree_root": str(row[2]),
|
|
844
|
+
"synced_s3": bool(row[3]),
|
|
845
|
+
"created_at": str(row[4]),
|
|
846
|
+
}
|
|
847
|
+
self._root_meta[clean_root] = rec
|
|
848
|
+
return dict(rec) # type: ignore[return-value]
|
|
849
|
+
|
|
850
|
+
def get_blob_path_by_root(self, tree_root: str) -> Optional[str]:
|
|
851
|
+
"""Return the CAS filesystem path for the blob matching `tree_root`."""
|
|
852
|
+
meta = self.get_blob_meta_by_root(tree_root)
|
|
853
|
+
return self.get_blob_path(meta["hash"]) if meta else None
|
|
854
|
+
|
|
855
|
+
def get_blob_by_root(self, tree_root: str) -> Optional[bytes]:
|
|
856
|
+
"""Retrieve blob bytes by `tree_root`."""
|
|
857
|
+
meta = self.get_blob_meta_by_root(tree_root)
|
|
858
|
+
return self.get_blob(meta["hash"]) if meta else None
|
|
859
|
+
|
|
860
|
+
def get_blob_slice_by_root(self, tree_root: str, start: int, length: int) -> Optional[bytes]:
|
|
861
|
+
"""Read a byte range `[start : start + length]` by `tree_root` without loading the full blob into RAM."""
|
|
862
|
+
meta = self.get_blob_meta_by_root(tree_root)
|
|
863
|
+
return self.get_blob_slice(meta["hash"], start, length) if meta else None
|
|
864
|
+
|
|
865
|
+
def delete_blob(self, blob_hash: str) -> bool:
|
|
866
|
+
"""Delete an ephemeral blob from CAS disk/memory and metadata store."""
|
|
867
|
+
clean_hash = blob_hash.lower().strip()
|
|
868
|
+
if not clean_hash:
|
|
869
|
+
return False
|
|
870
|
+
removed = False
|
|
871
|
+
with self._lock:
|
|
872
|
+
meta = self.get_blob_meta(clean_hash)
|
|
873
|
+
root_to_check = str(meta["tree_root"]).lower() if meta else ""
|
|
874
|
+
for d in ([self._cas_dir] if self._cas_dir else []) + list(self._overflow_cas_dirs):
|
|
875
|
+
if not d:
|
|
876
|
+
continue
|
|
877
|
+
path = os.path.join(d, clean_hash)
|
|
878
|
+
if os.path.exists(path):
|
|
879
|
+
try:
|
|
880
|
+
os.unlink(path)
|
|
881
|
+
except OSError:
|
|
882
|
+
pass
|
|
883
|
+
else:
|
|
884
|
+
removed = True
|
|
885
|
+
if clean_hash in self._mem_blobs:
|
|
886
|
+
del self._mem_blobs[clean_hash]
|
|
887
|
+
removed = True
|
|
888
|
+
if clean_hash in self._external_files:
|
|
889
|
+
del self._external_files[clean_hash]
|
|
890
|
+
removed = True
|
|
891
|
+
with self.transaction() as cur:
|
|
892
|
+
cur.execute("DELETE FROM rhio_blobs WHERE hash = ?", (clean_hash,))
|
|
893
|
+
stale_roots = [
|
|
894
|
+
r for r, rm in self._root_meta.items() if rm.get("hash") == clean_hash
|
|
895
|
+
]
|
|
896
|
+
if root_to_check and root_to_check not in stale_roots:
|
|
897
|
+
stale_roots.append(root_to_check)
|
|
898
|
+
for r_del in stale_roots:
|
|
899
|
+
self._root_meta.pop(r_del, None)
|
|
900
|
+
if self._one("SELECT 1 FROM rhio_blobs WHERE tree_root = ?", (r_del,)) is None:
|
|
901
|
+
old_claims = self._claims.pop(r_del, {})
|
|
902
|
+
for akey, clm in old_claims.items():
|
|
903
|
+
self._unindex_target(r_del, akey, clm)
|
|
904
|
+
cur.execute("DELETE FROM rhio_fiber_claims WHERE fiber_root = ?", (r_del,))
|
|
905
|
+
return removed
|
|
906
|
+
|
|
907
|
+
def has_blob(self, blob_hash: str) -> bool:
|
|
908
|
+
"""Check if blob exists in storage."""
|
|
909
|
+
clean_hash = blob_hash.lower().strip()
|
|
910
|
+
with self._lock:
|
|
911
|
+
if clean_hash in self._external_files and os.path.isfile(self._external_files[clean_hash]):
|
|
912
|
+
return True
|
|
913
|
+
return self._one("SELECT 1 FROM rhio_blobs WHERE hash = ?", (clean_hash,)) is not None
|
|
914
|
+
|
|
915
|
+
def mark_s3_synced(self, blob_hash: str) -> None:
|
|
916
|
+
"""Mark blob as synchronized with S3 / MinIO cluster storage."""
|
|
917
|
+
clean_hash = blob_hash.lower().strip()
|
|
918
|
+
with self._lock:
|
|
919
|
+
with self.transaction() as cur:
|
|
920
|
+
cur.execute("UPDATE rhio_blobs SET synced_s3 = 1 WHERE hash = ?", (clean_hash,))
|
|
921
|
+
for rmeta in self._root_meta.values():
|
|
922
|
+
if rmeta.get("hash") == clean_hash:
|
|
923
|
+
rmeta["synced_s3"] = True
|
|
924
|
+
|
|
925
|
+
def count_blobs(self) -> int:
|
|
926
|
+
with self._lock:
|
|
927
|
+
row = self._one("SELECT COUNT(*) FROM rhio_blobs")
|
|
928
|
+
return int(row[0]) if row else 0
|
|
929
|
+
|
|
930
|
+
def list_blobs(self, limit: int = 100) -> list[RhioBlobRecord]:
|
|
931
|
+
with self._lock:
|
|
932
|
+
rows = self._all(
|
|
933
|
+
"SELECT hash, size, tree_root, synced_s3, created_at FROM rhio_blobs "
|
|
934
|
+
"ORDER BY created_at DESC LIMIT ?",
|
|
935
|
+
(limit,),
|
|
936
|
+
)
|
|
937
|
+
return [
|
|
938
|
+
{
|
|
939
|
+
"hash": str(r[0]),
|
|
940
|
+
"size": int(r[1]),
|
|
941
|
+
"tree_root": str(r[2]),
|
|
942
|
+
"synced_s3": bool(r[3]),
|
|
943
|
+
"created_at": str(r[4]),
|
|
944
|
+
}
|
|
945
|
+
for r in rows
|
|
946
|
+
]
|
|
947
|
+
|
|
948
|
+
# ---- Unified Content-Addressed Fiber Claims -----------------------------
|
|
949
|
+
|
|
950
|
+
def _index_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None:
|
|
951
|
+
target = str((claim.get('target') or {}).get('strand_id') or '')
|
|
952
|
+
if target:
|
|
953
|
+
self._by_target.setdefault(target, set()).add((root, key))
|
|
954
|
+
|
|
955
|
+
def _unindex_target(self, root: str, key: str, claim: Mapping[str, Any]) -> None:
|
|
956
|
+
target = str((claim.get('target') or {}).get('strand_id') or '')
|
|
957
|
+
refs = self._by_target.get(target)
|
|
958
|
+
if refs is not None:
|
|
959
|
+
refs.discard((root, key))
|
|
960
|
+
if not refs:
|
|
961
|
+
del self._by_target[target]
|
|
962
|
+
|
|
963
|
+
def _viewer_claim(self, root: str, principal: Optional[Mapping[str, Any]]) -> Optional[dict[str, Any]]:
|
|
964
|
+
"""The caller's own claim, else the earliest public one; `None` means not readable."""
|
|
965
|
+
claims = self._claims.get(root) or {}
|
|
966
|
+
own = claims.get(_principal_key(principal)) if principal else None
|
|
967
|
+
if own is not None:
|
|
968
|
+
return own
|
|
969
|
+
public = [c for c in claims.values() if c.get('visibility') == 'public']
|
|
970
|
+
return min(public, key=lambda c: _safe_int(c.get('created_at'))) if public else None
|
|
971
|
+
|
|
972
|
+
def _manifest(
|
|
973
|
+
self,
|
|
974
|
+
root: str,
|
|
975
|
+
claim: Mapping[str, Any],
|
|
976
|
+
meta: Optional[RhioBlobRecord] = None,
|
|
977
|
+
) -> dict[str, Any]:
|
|
978
|
+
if meta is None:
|
|
979
|
+
meta = self.get_blob_meta_by_root(root)
|
|
980
|
+
size = int(meta["size"]) if meta else 0
|
|
981
|
+
sha = str(meta["hash"]) if meta else ""
|
|
982
|
+
out: dict[str, Any] = {
|
|
983
|
+
'fiber_root': root,
|
|
984
|
+
'sha256_hex': sha,
|
|
985
|
+
'byte_length': size,
|
|
986
|
+
'leaf_count_1024b': RhioBaoTree.leaf_count(size),
|
|
987
|
+
'kind': claim.get('kind'),
|
|
988
|
+
'author': dict(claim.get('author') or {}),
|
|
989
|
+
'visibility': claim.get('visibility'),
|
|
990
|
+
'mime_type': claim.get('mime_type') or '',
|
|
991
|
+
'created_at': claim.get('created_at'),
|
|
992
|
+
}
|
|
993
|
+
if claim.get('target'):
|
|
994
|
+
out['target'] = dict(claim['target'])
|
|
995
|
+
return out
|
|
996
|
+
|
|
997
|
+
def has_fiber(self, root: str) -> bool:
|
|
998
|
+
clean_root = str(root or '').strip().lower()
|
|
999
|
+
if not clean_root:
|
|
1000
|
+
return False
|
|
1001
|
+
with self._lock:
|
|
1002
|
+
return bool(self._claims.get(clean_root)) and self.get_blob_meta_by_root(clean_root) is not None
|
|
1003
|
+
|
|
1004
|
+
def readable(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> bool:
|
|
1005
|
+
return self.manifest(root, principal) is not None
|
|
1006
|
+
|
|
1007
|
+
def manifest(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[dict[str, Any]]:
|
|
1008
|
+
clean_root = str(root or '').strip().lower()
|
|
1009
|
+
if not clean_root:
|
|
1010
|
+
return None
|
|
1011
|
+
with self._lock:
|
|
1012
|
+
meta = self.get_blob_meta_by_root(clean_root)
|
|
1013
|
+
if meta is None:
|
|
1014
|
+
return None
|
|
1015
|
+
claim = self._viewer_claim(clean_root, principal)
|
|
1016
|
+
return self._manifest(clean_root, claim, meta) if claim else None
|
|
1017
|
+
|
|
1018
|
+
def get(self, root: str, principal: Optional[Mapping[str, Any]] = None) -> Optional[tuple[bytes, dict[str, Any]]]:
|
|
1019
|
+
clean_root = str(root or '').strip().lower()
|
|
1020
|
+
if not clean_root:
|
|
1021
|
+
return None
|
|
1022
|
+
with self._lock:
|
|
1023
|
+
m = self.manifest(clean_root, principal)
|
|
1024
|
+
if m is None:
|
|
1025
|
+
return None
|
|
1026
|
+
raw = self.get_blob(str(m.get('sha256_hex') or ''))
|
|
1027
|
+
return (raw, m) if raw is not None else None
|
|
1028
|
+
|
|
1029
|
+
def list_own(self, principal: Optional[Mapping[str, Any]]) -> list[dict[str, Any]]:
|
|
1030
|
+
key = _principal_key(principal)
|
|
1031
|
+
if not key:
|
|
1032
|
+
return []
|
|
1033
|
+
with self._lock:
|
|
1034
|
+
out: list[dict[str, Any]] = []
|
|
1035
|
+
for root, claims in self._claims.items():
|
|
1036
|
+
if key in claims:
|
|
1037
|
+
meta = self.get_blob_meta_by_root(root)
|
|
1038
|
+
if meta is not None:
|
|
1039
|
+
out.append(self._manifest(root, claims[key], meta))
|
|
1040
|
+
return sorted(out, key=lambda m: _safe_int(m.get('created_at')), reverse=True)
|
|
1041
|
+
|
|
1042
|
+
def owner_bytes(self, principal: Optional[Mapping[str, Any]]) -> int:
|
|
1043
|
+
key = _principal_key(principal)
|
|
1044
|
+
if not key:
|
|
1045
|
+
return 0
|
|
1046
|
+
with self._lock:
|
|
1047
|
+
total = 0
|
|
1048
|
+
for root, claims in self._claims.items():
|
|
1049
|
+
if key in claims:
|
|
1050
|
+
meta = self.get_blob_meta_by_root(root)
|
|
1051
|
+
if meta is not None:
|
|
1052
|
+
total += int(meta["size"])
|
|
1053
|
+
return total
|
|
1054
|
+
|
|
1055
|
+
def interaction_counts(self, target_strand_id: str) -> dict[str, int]:
|
|
1056
|
+
"""Public interaction claims on a strand, by kind (each actor's like counts once)."""
|
|
1057
|
+
counts: dict[str, int] = {}
|
|
1058
|
+
with self._lock:
|
|
1059
|
+
for root, key in self._by_target.get(target_strand_id, ()):
|
|
1060
|
+
claim = (self._claims.get(root) or {}).get(key) or {}
|
|
1061
|
+
kind = str(claim.get('kind') or '')
|
|
1062
|
+
if claim.get('visibility') == 'public' and kind in INTERACTION_KINDS:
|
|
1063
|
+
counts[kind] = counts.get(kind, 0) + 1
|
|
1064
|
+
return counts
|
|
1065
|
+
|
|
1066
|
+
def claim_blob(
|
|
1067
|
+
self,
|
|
1068
|
+
tree_root: str,
|
|
1069
|
+
principal: Optional[Mapping[str, Any]],
|
|
1070
|
+
*,
|
|
1071
|
+
sha256_hex: str = '',
|
|
1072
|
+
size: Optional[int] = None,
|
|
1073
|
+
kind: str = 'binary',
|
|
1074
|
+
mime_type: str = '',
|
|
1075
|
+
visibility: str = 'public',
|
|
1076
|
+
target_strand_id: str = '',
|
|
1077
|
+
overwrite_visibility: bool = True,
|
|
1078
|
+
created_at: Optional[int] = None,
|
|
1079
|
+
) -> Optional[dict[str, Any]]:
|
|
1080
|
+
"""Register or update an author's fiber claim on an existing (or metadata-registered) blob."""
|
|
1081
|
+
clean_root = str(tree_root or '').strip().lower()
|
|
1082
|
+
if len(clean_root) != 64:
|
|
1083
|
+
return None
|
|
1084
|
+
who = principal if (principal and _principal_key(principal)) else {'identity_id': 'node', 'did': '', 'handle': ''}
|
|
1085
|
+
key = _principal_key(who) or 'node'
|
|
1086
|
+
vis = 'vault_only' if visibility == 'vault_only' else 'public'
|
|
1087
|
+
eff_kind = kind if kind in FIBER_KINDS else 'binary'
|
|
1088
|
+
now = int(created_at if created_at is not None else time.time())
|
|
1089
|
+
with self._lock:
|
|
1090
|
+
meta = self.get_blob_meta_by_root(clean_root)
|
|
1091
|
+
if meta is None and sha256_hex and size is not None:
|
|
1092
|
+
clean_sha = str(sha256_hex).strip().lower()
|
|
1093
|
+
if len(clean_sha) == 64:
|
|
1094
|
+
with self.transaction() as cur:
|
|
1095
|
+
cur.execute(
|
|
1096
|
+
"""
|
|
1097
|
+
INSERT OR IGNORE INTO rhio_blobs (hash, size, tree_root, synced_s3, created_at)
|
|
1098
|
+
VALUES (?, ?, ?, 0, ?)
|
|
1099
|
+
""",
|
|
1100
|
+
(clean_sha, int(size), clean_root, _now_iso()),
|
|
1101
|
+
)
|
|
1102
|
+
meta = self.get_blob_meta_by_root(clean_root)
|
|
1103
|
+
if meta is None:
|
|
1104
|
+
return None
|
|
1105
|
+
claims = self._claims.setdefault(clean_root, {})
|
|
1106
|
+
existing = claims.get(key)
|
|
1107
|
+
claim = dict(existing or {'author': _author_ref(who), 'kind': eff_kind, 'created_at': now})
|
|
1108
|
+
if existing is None or overwrite_visibility:
|
|
1109
|
+
claim['visibility'] = vis
|
|
1110
|
+
if mime_type or not claim.get('mime_type'):
|
|
1111
|
+
claim['mime_type'] = str(mime_type or claim.get('mime_type') or '')[:120]
|
|
1112
|
+
if kind in FIBER_KINDS:
|
|
1113
|
+
claim['kind'] = kind
|
|
1114
|
+
if target_strand_id:
|
|
1115
|
+
if existing is not None:
|
|
1116
|
+
self._unindex_target(clean_root, key, existing)
|
|
1117
|
+
claim['target'] = {'strand_id': str(target_strand_id)[:200]}
|
|
1118
|
+
claims[key] = claim
|
|
1119
|
+
self._index_target(clean_root, key, claim)
|
|
1120
|
+
pjson = json.dumps(claim, separators=(',', ':'))
|
|
1121
|
+
with self.transaction() as cur:
|
|
1122
|
+
cur.execute(
|
|
1123
|
+
"""
|
|
1124
|
+
INSERT OR REPLACE INTO rhio_fiber_claims (fiber_root, author_key, payload_json, created_at)
|
|
1125
|
+
VALUES (?, ?, ?, ?)
|
|
1126
|
+
""",
|
|
1127
|
+
(clean_root, key, pjson, _safe_int(claim.get('created_at'), now)),
|
|
1128
|
+
)
|
|
1129
|
+
return self._manifest(clean_root, claim, meta)
|
|
1130
|
+
|
|
1131
|
+
def put(
|
|
1132
|
+
self,
|
|
1133
|
+
raw: bytes,
|
|
1134
|
+
principal: Optional[Mapping[str, Any]],
|
|
1135
|
+
*,
|
|
1136
|
+
kind: str,
|
|
1137
|
+
mime_type: str = '',
|
|
1138
|
+
visibility: str = 'vault_only',
|
|
1139
|
+
target_strand_id: str = '',
|
|
1140
|
+
enforce_quota: bool = True,
|
|
1141
|
+
) -> dict[str, Any]:
|
|
1142
|
+
"""Store `raw` (once) in `RhioBlobSync` and record the caller's claim on it; returns the caller's manifest."""
|
|
1143
|
+
who = _require_principal(principal)
|
|
1144
|
+
if kind not in FIBER_KINDS:
|
|
1145
|
+
raise FiberAccessError(400, 'invalid_fiber_kind')
|
|
1146
|
+
if kind in INTERACTION_KINDS and not target_strand_id:
|
|
1147
|
+
raise FiberAccessError(400, 'interaction_target_required')
|
|
1148
|
+
if len(raw) > MAX_FIBER_BYTES:
|
|
1149
|
+
raise FiberAccessError(413, 'binary_too_large')
|
|
1150
|
+
vis = visibility if visibility in VISIBILITIES else 'vault_only'
|
|
1151
|
+
root = RhioBaoTree.compute_root(raw)
|
|
1152
|
+
key = _principal_key(who)
|
|
1153
|
+
now = int(time.time())
|
|
1154
|
+
with self._lock:
|
|
1155
|
+
existing = (self._claims.get(root) or {}).get(key)
|
|
1156
|
+
if existing is None and enforce_quota and self.owner_bytes(who) + len(raw) > MAX_OWNER_BYTES:
|
|
1157
|
+
raise FiberAccessError(413, 'owner_quota_exceeded')
|
|
1158
|
+
sha, _, size = self.store_blob(raw, tree_root=root)
|
|
1159
|
+
manifest = self.claim_blob(
|
|
1160
|
+
root,
|
|
1161
|
+
who,
|
|
1162
|
+
sha256_hex=sha,
|
|
1163
|
+
size=size,
|
|
1164
|
+
kind=kind,
|
|
1165
|
+
mime_type=mime_type,
|
|
1166
|
+
visibility=vis,
|
|
1167
|
+
target_strand_id=target_strand_id,
|
|
1168
|
+
overwrite_visibility=True,
|
|
1169
|
+
created_at=now,
|
|
1170
|
+
)
|
|
1171
|
+
assert manifest is not None
|
|
1172
|
+
return manifest
|
|
1173
|
+
|
|
1174
|
+
def set_visibility(self, root: str, principal: Optional[Mapping[str, Any]], visibility: str) -> dict[str, Any]:
|
|
1175
|
+
who = _require_principal(principal)
|
|
1176
|
+
key = _principal_key(who)
|
|
1177
|
+
clean_root = str(root or '').strip().lower()
|
|
1178
|
+
if visibility not in VISIBILITIES:
|
|
1179
|
+
raise FiberAccessError(400, 'invalid_visibility')
|
|
1180
|
+
with self._lock:
|
|
1181
|
+
claim = (self._claims.get(clean_root) or {}).get(key)
|
|
1182
|
+
meta = self.get_blob_meta_by_root(clean_root) if claim is not None else None
|
|
1183
|
+
if claim is None or meta is None:
|
|
1184
|
+
raise FiberAccessError(404, 'fiber_claim_not_found')
|
|
1185
|
+
if claim.get('visibility') != visibility:
|
|
1186
|
+
claim['visibility'] = visibility
|
|
1187
|
+
pjson = json.dumps(claim, separators=(',', ':'))
|
|
1188
|
+
with self.transaction() as cur:
|
|
1189
|
+
cur.execute(
|
|
1190
|
+
"UPDATE rhio_fiber_claims SET payload_json = ? WHERE fiber_root = ? AND author_key = ?",
|
|
1191
|
+
(pjson, clean_root, key),
|
|
1192
|
+
)
|
|
1193
|
+
return self._manifest(clean_root, claim, meta)
|
|
1194
|
+
|
|
1195
|
+
def release(self, root: str, principal: Optional[Mapping[str, Any]]) -> bool:
|
|
1196
|
+
"""Drop the caller's claim; the underlying blob is deleted when no claims remain."""
|
|
1197
|
+
who = _require_principal(principal)
|
|
1198
|
+
key = _principal_key(who)
|
|
1199
|
+
clean_root = str(root or '').strip().lower()
|
|
1200
|
+
with self._lock:
|
|
1201
|
+
claims = self._claims.get(clean_root) or {}
|
|
1202
|
+
if key not in claims:
|
|
1203
|
+
return False
|
|
1204
|
+
self._unindex_target(clean_root, key, claims.pop(key))
|
|
1205
|
+
with self.transaction() as cur:
|
|
1206
|
+
cur.execute(
|
|
1207
|
+
"DELETE FROM rhio_fiber_claims WHERE fiber_root = ? AND author_key = ?",
|
|
1208
|
+
(clean_root, key),
|
|
1209
|
+
)
|
|
1210
|
+
if not claims:
|
|
1211
|
+
self._claims.pop(clean_root, None)
|
|
1212
|
+
meta = self.get_blob_meta_by_root(clean_root)
|
|
1213
|
+
if meta is not None:
|
|
1214
|
+
self.delete_blob(meta["hash"])
|
|
1215
|
+
return True
|
|
1216
|
+
|
|
1217
|
+
def close(self) -> None:
|
|
1218
|
+
super().close()
|
|
1219
|
+
|
|
1220
|
+
|
|
1221
|
+
def binary_http_response(
|
|
1222
|
+
raw: bytes,
|
|
1223
|
+
*,
|
|
1224
|
+
bao_root: str,
|
|
1225
|
+
sha256_hex: str,
|
|
1226
|
+
mime_type: str,
|
|
1227
|
+
public: bool,
|
|
1228
|
+
filename: str,
|
|
1229
|
+
as_json: bool,
|
|
1230
|
+
json_extra: Optional[Mapping[str, Any]] = None,
|
|
1231
|
+
) -> tuple[int, Any, str, dict[str, str]]:
|
|
1232
|
+
"""Serve stored bytes so they can never render as an active page on the app's origin."""
|
|
1233
|
+
cache = 'public, max-age=3600, immutable' if public else 'private, no-store'
|
|
1234
|
+
hdrs: dict[str, str] = {
|
|
1235
|
+
'X-Bao-Tree-Root': bao_root,
|
|
1236
|
+
'Cache-Control': cache,
|
|
1237
|
+
'X-Content-Type-Options': 'nosniff',
|
|
1238
|
+
'Accept-Ranges': 'bytes',
|
|
1239
|
+
}
|
|
1240
|
+
mime = str(mime_type or '').strip().lower()
|
|
1241
|
+
if as_json:
|
|
1242
|
+
return 200, {
|
|
1243
|
+
'ok': True,
|
|
1244
|
+
**dict(json_extra or {}),
|
|
1245
|
+
'bao_root': bao_root,
|
|
1246
|
+
'sha256_hex': sha256_hex,
|
|
1247
|
+
'mime_type': mime or 'application/octet-stream',
|
|
1248
|
+
'size_bytes': len(raw),
|
|
1249
|
+
'data_b64': base64.b64encode(raw).decode('ascii'),
|
|
1250
|
+
}, JSON_CT, hdrs
|
|
1251
|
+
fname = re.sub(r'[^A-Za-z0-9._-]', '_', filename or 'fiber.bin')[:120]
|
|
1252
|
+
hdrs['Content-Disposition'] = f'attachment; filename="{fname}"'
|
|
1253
|
+
hdrs['Content-Security-Policy'] = "default-src 'none'; sandbox"
|
|
1254
|
+
return 200, raw, (mime if mime in _SAFE_BINARY_MIME else 'application/octet-stream'), hdrs
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
def is_fiber_route(route: str) -> bool:
|
|
1258
|
+
return route == '/api/fibers' or route.startswith('/api/fibers/')
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
def _not_found(root: str) -> tuple[int, Any, str, dict[str, str]]:
|
|
1262
|
+
return 404, {'ok': False, 'error': 'fiber_not_found', 'fiber_root': root}, JSON_CT, {}
|
|
1263
|
+
|
|
1264
|
+
|
|
1265
|
+
def handle_fiber_http(
|
|
1266
|
+
store: Any,
|
|
1267
|
+
method: str,
|
|
1268
|
+
route: str,
|
|
1269
|
+
query: Any = '',
|
|
1270
|
+
body: Optional[Mapping[str, Any]] = None,
|
|
1271
|
+
principal: Optional[Mapping[str, Any]] = None,
|
|
1272
|
+
*,
|
|
1273
|
+
range_header: str = '',
|
|
1274
|
+
) -> Optional[tuple[int, Any, str, dict[str, str]]]:
|
|
1275
|
+
"""Route `/api/fibers*`. `principal` must come from the server's own session resolution."""
|
|
1276
|
+
store = getattr(store, 'fiber_store', getattr(store, 'rhio_store', store))
|
|
1277
|
+
qp = query if isinstance(query, Mapping) else urllib.parse.parse_qs(str(query or ''))
|
|
1278
|
+
try:
|
|
1279
|
+
if method in ('GET', 'HEAD'):
|
|
1280
|
+
if route == '/api/fibers':
|
|
1281
|
+
if not principal or not _principal_key(principal):
|
|
1282
|
+
raise FiberAccessError(401, 'authentication_required')
|
|
1283
|
+
return 200, {'ok': True, 'fibers': store.list_own(principal)}, JSON_CT, {'Cache-Control': 'private, no-store'}
|
|
1284
|
+
if route == '/api/fibers/interactions':
|
|
1285
|
+
target = (qp.get('target', [''])[0] or '').strip()
|
|
1286
|
+
if not target:
|
|
1287
|
+
raise FiberAccessError(400, 'target_required')
|
|
1288
|
+
return 200, {'ok': True, 'target': target, 'counts': store.interaction_counts(target)}, JSON_CT, {'Cache-Control': 'public, max-age=15'}
|
|
1289
|
+
parts = [p for p in route[len('/api/fibers/'):].split('/') if p]
|
|
1290
|
+
if not parts or len(parts) > 2 or not _ROOT_RE.match(parts[0]):
|
|
1291
|
+
return None
|
|
1292
|
+
root = parts[0]
|
|
1293
|
+
if len(parts) == 2:
|
|
1294
|
+
if parts[1] != 'manifest':
|
|
1295
|
+
return None
|
|
1296
|
+
manifest = store.manifest(root, principal)
|
|
1297
|
+
return (200, {'ok': True, 'fiber': manifest}, JSON_CT, {'Cache-Control': 'private, no-store'}) if manifest else _not_found(root)
|
|
1298
|
+
as_json = (qp.get('format', [''])[0] or '').strip().lower() == 'json'
|
|
1299
|
+
rng = str(range_header or '').strip()
|
|
1300
|
+
if method == 'HEAD' and not rng and not as_json:
|
|
1301
|
+
manifest = store.manifest(root, principal)
|
|
1302
|
+
if manifest is None:
|
|
1303
|
+
return _not_found(root)
|
|
1304
|
+
_st, _body, ct, hdrs = binary_http_response(
|
|
1305
|
+
b'',
|
|
1306
|
+
bao_root=root,
|
|
1307
|
+
sha256_hex=manifest['sha256_hex'],
|
|
1308
|
+
mime_type=manifest.get('mime_type') or '',
|
|
1309
|
+
public=manifest.get('visibility') == 'public',
|
|
1310
|
+
filename=f"{root[:16]}.bin",
|
|
1311
|
+
as_json=False,
|
|
1312
|
+
)
|
|
1313
|
+
hdrs['Content-Length'] = str(int(manifest.get('byte_length') or 0))
|
|
1314
|
+
return 200, b'', ct, hdrs
|
|
1315
|
+
if rng and not as_json:
|
|
1316
|
+
manifest = store.manifest(root, principal)
|
|
1317
|
+
if manifest is None:
|
|
1318
|
+
return _not_found(root)
|
|
1319
|
+
total = int(manifest.get('byte_length') or 0)
|
|
1320
|
+
m = _RANGE_HEADER_RE.match(rng)
|
|
1321
|
+
if not m:
|
|
1322
|
+
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
|
|
1323
|
+
start_s, end_s = m.group(1), m.group(2)
|
|
1324
|
+
if start_s == '' and end_s == '':
|
|
1325
|
+
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
|
|
1326
|
+
if start_s == '' and end_s != '':
|
|
1327
|
+
suffix_len = int(end_s)
|
|
1328
|
+
start = max(0, total - suffix_len)
|
|
1329
|
+
end = total - 1
|
|
1330
|
+
else:
|
|
1331
|
+
start = int(start_s) if start_s else 0
|
|
1332
|
+
end = int(end_s) if end_s else (total - 1)
|
|
1333
|
+
if total <= 0 or start >= total or start > end:
|
|
1334
|
+
return 416, {'ok': False, 'error': 'range_not_satisfiable'}, JSON_CT, {'Content-Range': f'bytes */{total}', 'Accept-Ranges': 'bytes'}
|
|
1335
|
+
end = min(end, total - 1)
|
|
1336
|
+
slice_len = end - start + 1
|
|
1337
|
+
if method == 'HEAD':
|
|
1338
|
+
_st, _body, ct, hdrs = binary_http_response(
|
|
1339
|
+
b'',
|
|
1340
|
+
bao_root=root,
|
|
1341
|
+
sha256_hex=manifest['sha256_hex'],
|
|
1342
|
+
mime_type=manifest.get('mime_type') or '',
|
|
1343
|
+
public=manifest.get('visibility') == 'public',
|
|
1344
|
+
filename=f"{root[:16]}.bin",
|
|
1345
|
+
as_json=False,
|
|
1346
|
+
)
|
|
1347
|
+
hdrs['Content-Range'] = f'bytes {start}-{end}/{total}'
|
|
1348
|
+
hdrs['Content-Length'] = str(slice_len)
|
|
1349
|
+
hdrs['X-Bao-Range-Root'] = root
|
|
1350
|
+
return 206, b'', ct, hdrs
|
|
1351
|
+
if hasattr(store, 'get_blob_slice') and manifest.get('sha256_hex'):
|
|
1352
|
+
chunk = store.get_blob_slice(str(manifest['sha256_hex']), start, slice_len)
|
|
1353
|
+
elif hasattr(store, 'get_blob_slice_by_root'):
|
|
1354
|
+
chunk = store.get_blob_slice_by_root(root, start, slice_len)
|
|
1355
|
+
else:
|
|
1356
|
+
found_full = store.get(root, principal)
|
|
1357
|
+
chunk = found_full[0][start : end + 1] if found_full else None
|
|
1358
|
+
if chunk is None:
|
|
1359
|
+
return _not_found(root)
|
|
1360
|
+
_st, body_b, ct, hdrs = binary_http_response(
|
|
1361
|
+
chunk,
|
|
1362
|
+
bao_root=root,
|
|
1363
|
+
sha256_hex=manifest['sha256_hex'],
|
|
1364
|
+
mime_type=manifest.get('mime_type') or '',
|
|
1365
|
+
public=manifest.get('visibility') == 'public',
|
|
1366
|
+
filename=f"{root[:16]}.bin",
|
|
1367
|
+
as_json=False,
|
|
1368
|
+
)
|
|
1369
|
+
hdrs['Content-Range'] = f'bytes {start}-{end}/{total}'
|
|
1370
|
+
if len(chunk) <= 1024 * 1024:
|
|
1371
|
+
hdrs['X-Bao-Range-Root'] = RhioBaoTree.compute_root(chunk)
|
|
1372
|
+
else:
|
|
1373
|
+
hdrs['X-Bao-Range-Root'] = root
|
|
1374
|
+
return 206, body_b, ct, hdrs
|
|
1375
|
+
found = store.get(root, principal)
|
|
1376
|
+
if found is None:
|
|
1377
|
+
return _not_found(root)
|
|
1378
|
+
raw, manifest = found
|
|
1379
|
+
return binary_http_response(
|
|
1380
|
+
raw,
|
|
1381
|
+
bao_root=root,
|
|
1382
|
+
sha256_hex=manifest['sha256_hex'],
|
|
1383
|
+
mime_type=manifest.get('mime_type') or '',
|
|
1384
|
+
public=manifest.get('visibility') == 'public',
|
|
1385
|
+
filename=f"{root[:16]}.bin",
|
|
1386
|
+
as_json=as_json,
|
|
1387
|
+
json_extra={'fiber': manifest},
|
|
1388
|
+
)
|
|
1389
|
+
|
|
1390
|
+
if method != 'POST' or not is_fiber_route(route):
|
|
1391
|
+
return None
|
|
1392
|
+
payload: Mapping[str, Any] = body or {}
|
|
1393
|
+
if route in ('/api/fibers', '/api/fibers/put'):
|
|
1394
|
+
raw = _decode_b64(payload.get('data_b64'), MAX_FIBER_BYTES)
|
|
1395
|
+
if raw is None:
|
|
1396
|
+
raise FiberAccessError(400, 'data_b64_required')
|
|
1397
|
+
manifest = store.put(
|
|
1398
|
+
raw,
|
|
1399
|
+
principal,
|
|
1400
|
+
kind=str(payload.get('kind') or ''),
|
|
1401
|
+
mime_type=str(payload.get('mime_type') or ''),
|
|
1402
|
+
visibility=str(payload.get('visibility') or 'vault_only'),
|
|
1403
|
+
target_strand_id=str(payload.get('target_strand_id') or ''),
|
|
1404
|
+
)
|
|
1405
|
+
return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {}
|
|
1406
|
+
if route == '/api/fibers/visibility':
|
|
1407
|
+
manifest = store.set_visibility(
|
|
1408
|
+
str(payload.get('fiber_root') or ''),
|
|
1409
|
+
principal,
|
|
1410
|
+
str(payload.get('visibility') or ''),
|
|
1411
|
+
)
|
|
1412
|
+
return 200, {'ok': True, 'fiber': manifest}, JSON_CT, {}
|
|
1413
|
+
if route == '/api/fibers/release':
|
|
1414
|
+
root = str(payload.get('fiber_root') or '')
|
|
1415
|
+
released = store.release(root, principal)
|
|
1416
|
+
return (200 if released else 404), {'ok': released, 'fiber_root': root}, JSON_CT, {}
|
|
1417
|
+
return None
|
|
1418
|
+
except Exception as exc:
|
|
1419
|
+
status = getattr(exc, 'status', None)
|
|
1420
|
+
error = getattr(exc, 'error', None)
|
|
1421
|
+
if isinstance(status, int) and isinstance(error, str):
|
|
1422
|
+
return status, {'ok': False, 'error': error}, JSON_CT, {}
|
|
1423
|
+
raise
|