basic-ipfs 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- basic_ipfs/__init__.py +2229 -0
- basic_ipfs/cli.py +301 -0
- basic_ipfs/kubo_checksums.py +28 -0
- basic_ipfs/py.typed +0 -0
- basic_ipfs-1.0.0.dist-info/METADATA +343 -0
- basic_ipfs-1.0.0.dist-info/RECORD +10 -0
- basic_ipfs-1.0.0.dist-info/WHEEL +5 -0
- basic_ipfs-1.0.0.dist-info/entry_points.txt +2 -0
- basic_ipfs-1.0.0.dist-info/licenses/LICENSE +21 -0
- basic_ipfs-1.0.0.dist-info/top_level.txt +1 -0
basic_ipfs/__init__.py
ADDED
|
@@ -0,0 +1,2229 @@
|
|
|
1
|
+
"""
|
|
2
|
+
basic_ipfs — stupid-simple IPFS for Python.
|
|
3
|
+
|
|
4
|
+
Runs a real Kubo node under the hood. Auto-downloads the binary on first use.
|
|
5
|
+
|
|
6
|
+
Quick start:
|
|
7
|
+
|
|
8
|
+
import basic_ipfs
|
|
9
|
+
|
|
10
|
+
cid = basic_ipfs.add("photo.jpg") # or bytes — adds AND pins
|
|
11
|
+
cid = basic_ipfs.announce("photo.jpg") # adds without pinning (may be GC'd)
|
|
12
|
+
data = basic_ipfs.get(cid) # → bytes
|
|
13
|
+
basic_ipfs.get(cid, "copy.jpg") # or write to disk
|
|
14
|
+
|
|
15
|
+
basic_ipfs.pin(cid) # keep forever
|
|
16
|
+
basic_ipfs.unpin(cid) # let GC reclaim
|
|
17
|
+
|
|
18
|
+
The daemon starts lazily on the first call and stops cleanly on process exit.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import atexit
|
|
24
|
+
import datetime
|
|
25
|
+
import glob as _glob
|
|
26
|
+
import hashlib
|
|
27
|
+
import hmac
|
|
28
|
+
import io
|
|
29
|
+
import ipaddress
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
import os
|
|
33
|
+
import platform
|
|
34
|
+
import re
|
|
35
|
+
import shutil
|
|
36
|
+
import signal
|
|
37
|
+
import socket
|
|
38
|
+
import stat
|
|
39
|
+
import subprocess
|
|
40
|
+
import sys
|
|
41
|
+
import tarfile
|
|
42
|
+
import threading
|
|
43
|
+
import time
|
|
44
|
+
import zipfile
|
|
45
|
+
from collections.abc import Iterable
|
|
46
|
+
from pathlib import Path
|
|
47
|
+
from typing import Any, TypedDict, Union, overload
|
|
48
|
+
from urllib.parse import urlparse
|
|
49
|
+
|
|
50
|
+
import requests
|
|
51
|
+
from platformdirs import user_data_dir
|
|
52
|
+
|
|
53
|
+
from . import kubo_checksums
|
|
54
|
+
|
|
55
|
+
# Single source of truth: the version that pip resolved when installing this
|
|
56
|
+
# wheel. Avoids drift between pyproject.toml and a hardcoded literal — if
|
|
57
|
+
# __version__ ever lies about which fix is deployed, security advisories
|
|
58
|
+
# become very hard to reason about.
|
|
59
|
+
try:
|
|
60
|
+
from importlib.metadata import PackageNotFoundError
|
|
61
|
+
from importlib.metadata import version as _pkg_version
|
|
62
|
+
|
|
63
|
+
__version__ = _pkg_version("basic-ipfs")
|
|
64
|
+
except PackageNotFoundError:
|
|
65
|
+
# Running from a source checkout that was never installed (e.g. tests
|
|
66
|
+
# invoked with `pytest` against the working tree). Stay obviously fake.
|
|
67
|
+
__version__ = "0+unknown"
|
|
68
|
+
|
|
69
|
+
logger = logging.getLogger(__name__)
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Configuration — assign before the first API call if you want to override
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
APP_NAME = "basic_ipfs"
|
|
76
|
+
|
|
77
|
+
# Pinned Kubo version — downloaded automatically if not already present.
|
|
78
|
+
KUBO_VERSION = "v0.40.1"
|
|
79
|
+
KUBO_DIST_BASE = "https://dist.ipfs.tech/kubo"
|
|
80
|
+
|
|
81
|
+
# Hostnames the download is permitted to redirect to. TLS still catches a
|
|
82
|
+
# MitM, but pinning the origin means a 30x to attacker.example also fails
|
|
83
|
+
# instead of relying on the SHA-512 check alone.
|
|
84
|
+
_ALLOWED_DOWNLOAD_HOSTS = ("dist.ipfs.tech",)
|
|
85
|
+
|
|
86
|
+
# Where the IPFS repo lives. None = platformdirs.user_data_dir(APP_NAME)
|
|
87
|
+
REPO_PATH: Path | None = None
|
|
88
|
+
|
|
89
|
+
# Local API the daemon listens on
|
|
90
|
+
API_HOST = "127.0.0.1"
|
|
91
|
+
API_PORT = 5001
|
|
92
|
+
|
|
93
|
+
# Local HTTP gateway address. Kubo defaults to 127.0.0.1:8080 — only
|
|
94
|
+
# override if you need to run a second basic_ipfs node on the same host.
|
|
95
|
+
GATEWAY_HOST = "127.0.0.1"
|
|
96
|
+
GATEWAY_PORT = 8080
|
|
97
|
+
|
|
98
|
+
# Optional override for ``Addresses.Swarm``. When ``None``, Kubo's defaults
|
|
99
|
+
# apply (listens on 0.0.0.0:4001 etc.). Set to a list of multiaddrs to pin
|
|
100
|
+
# the swarm listener — e.g. ``["/ip4/127.0.0.1/tcp/4001"]`` for
|
|
101
|
+
# loopback-only on a multi-tenant host. Applied before daemon launch.
|
|
102
|
+
SWARM_ADDRESSES: list[str] | None = None
|
|
103
|
+
|
|
104
|
+
# Repo size cap. Empty string = unlimited.
|
|
105
|
+
STORAGE_MAX = "50GB"
|
|
106
|
+
|
|
107
|
+
# Seconds to wait for the daemon to come up on startup
|
|
108
|
+
DAEMON_STARTUP_TIMEOUT = 60
|
|
109
|
+
|
|
110
|
+
# Free-space buffer required before auto-download (~115 MB archive + extracted).
|
|
111
|
+
_INSTALL_FREE_BYTES = 300 * 1024 * 1024
|
|
112
|
+
|
|
113
|
+
# How many CIDs to send per batch pin/unpin HTTP call. Kubo's URL is the limit;
|
|
114
|
+
# 500 × ~60-byte CIDs ≈ 30 KB — comfortable headroom under typical 8 KB+ limits.
|
|
115
|
+
_PIN_BATCH_SIZE = 500
|
|
116
|
+
|
|
117
|
+
# Kubo daemon log: kept inside the repo for easy bug reports.
|
|
118
|
+
_DAEMON_LOG_NAME = "basic_ipfs_daemon.log"
|
|
119
|
+
_DAEMON_LOG_MAX_BYTES = 5 * 1024 * 1024 # rotate when the previous log exceeds this
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# Exceptions
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class IPFSError(RuntimeError):
|
|
127
|
+
"""Base class for everything this package raises."""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class IPFSBinaryNotFound(IPFSError):
|
|
131
|
+
"""Kubo binary missing and auto-download failed."""
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class IPFSDaemonTimeout(IPFSError):
|
|
135
|
+
"""Daemon did not become ready in time."""
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class IPFSOperationError(IPFSError):
|
|
139
|
+
"""An IPFS API call returned an error.
|
|
140
|
+
|
|
141
|
+
Attributes mirror Kubo's JSON error envelope so callers can branch on
|
|
142
|
+
the structured fields instead of pattern-matching the message string.
|
|
143
|
+
Any of them may be ``None`` if the response was not a well-formed Kubo
|
|
144
|
+
error body (e.g. an HTML 502 from a reverse proxy).
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
def __init__(
|
|
148
|
+
self,
|
|
149
|
+
message: str,
|
|
150
|
+
*,
|
|
151
|
+
status_code: int | None = None,
|
|
152
|
+
kubo_code: int | None = None,
|
|
153
|
+
kubo_type: str | None = None,
|
|
154
|
+
kubo_message: str | None = None,
|
|
155
|
+
) -> None:
|
|
156
|
+
super().__init__(message)
|
|
157
|
+
self.status_code = status_code
|
|
158
|
+
self.kubo_code = kubo_code
|
|
159
|
+
self.kubo_type = kubo_type
|
|
160
|
+
self.kubo_message = kubo_message
|
|
161
|
+
# Populated by pin/unpin batch fallbacks when a partial failure
|
|
162
|
+
# leaves some CIDs processed and others not. Callers can branch on
|
|
163
|
+
# ``isinstance(exc, IPFSOperationError) and exc.failed_cids`` to
|
|
164
|
+
# decide what to retry.
|
|
165
|
+
self.succeeded_cids: list[str] = []
|
|
166
|
+
self.failed_cids: list[str] = []
|
|
167
|
+
|
|
168
|
+
def is_not_pinned(self) -> bool:
|
|
169
|
+
"""True iff this error is Kubo's ``<cid> is not pinned`` response.
|
|
170
|
+
|
|
171
|
+
Used to make pin-removal idempotent without swallowing real failures.
|
|
172
|
+
Requires both a Kubo-shaped error body (``Type == "error"``) and the
|
|
173
|
+
canonical ``not pinned`` substring — guards against accidentally
|
|
174
|
+
treating an unrelated 500 as a successful no-op.
|
|
175
|
+
"""
|
|
176
|
+
if self.kubo_type != "error":
|
|
177
|
+
return False
|
|
178
|
+
msg = (self.kubo_message or "").lower()
|
|
179
|
+
return "not pinned" in msg
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class IPFSPortInUse(IPFSError):
|
|
183
|
+
"""The configured API port is already bound by something other than our daemon."""
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class IPFSRepoLocked(IPFSError):
|
|
187
|
+
"""The IPFS repo is locked by another running Kubo daemon."""
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class IPFSRepoCorrupt(IPFSError):
|
|
191
|
+
"""The IPFS repo on disk failed a sanity check."""
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
# TypedDicts
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class StatusDict(TypedDict):
|
|
200
|
+
peer_id: str | None
|
|
201
|
+
agent_version: str | None
|
|
202
|
+
repo_size_bytes: int | None
|
|
203
|
+
num_objects: int | None
|
|
204
|
+
pinned_cids: int
|
|
205
|
+
addresses: list[str]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# ---------------------------------------------------------------------------
|
|
209
|
+
# Platform detection
|
|
210
|
+
# ---------------------------------------------------------------------------
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _base_dir() -> Path:
|
|
214
|
+
"""Package directory — respects PyInstaller one-file bundles."""
|
|
215
|
+
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
|
|
216
|
+
return Path(sys._MEIPASS)
|
|
217
|
+
return Path(__file__).resolve().parent
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _is_musl() -> bool:
|
|
221
|
+
"""True on musl-libc Linux (e.g. Alpine). Kubo ships glibc only."""
|
|
222
|
+
if sys.platform != "linux":
|
|
223
|
+
return False
|
|
224
|
+
return bool(_glob.glob("/lib/ld-musl-*")) or bool(_glob.glob("/lib64/ld-musl-*"))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
_SUPPORTED_TRIPLES = (
|
|
228
|
+
"linux-amd64", "linux-arm64", "linux-riscv64",
|
|
229
|
+
"darwin-amd64", "darwin-arm64",
|
|
230
|
+
"windows-amd64",
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _platform_key() -> str:
|
|
235
|
+
system = platform.system().lower()
|
|
236
|
+
machine = platform.machine().lower()
|
|
237
|
+
|
|
238
|
+
if system == "linux":
|
|
239
|
+
if _is_musl():
|
|
240
|
+
manual = Path(user_data_dir(APP_NAME)) / "bin" / f"linux-{machine}" / "ipfs"
|
|
241
|
+
raise IPFSBinaryNotFound(
|
|
242
|
+
"Detected Alpine/musl Linux. Kubo publishes glibc binaries only.\n"
|
|
243
|
+
" - With `apk add gcompat` the glibc build *may* run, but it isn't officially supported.\n"
|
|
244
|
+
" - Otherwise, place a musl-compatible `ipfs` binary at:\n"
|
|
245
|
+
f" {manual}"
|
|
246
|
+
)
|
|
247
|
+
if machine in ("x86_64", "amd64"):
|
|
248
|
+
return "linux-amd64"
|
|
249
|
+
if machine in ("aarch64", "arm64"):
|
|
250
|
+
return "linux-arm64"
|
|
251
|
+
if machine in ("riscv64",):
|
|
252
|
+
return "linux-riscv64"
|
|
253
|
+
if machine in ("armv7l", "armv6l", "armhf", "arm"):
|
|
254
|
+
raise IPFSBinaryNotFound(
|
|
255
|
+
f"32-bit ARM ({machine}) is not supported by current Kubo releases. "
|
|
256
|
+
"Run a 64-bit OS (aarch64) on Raspberry Pi 4/5 or similar."
|
|
257
|
+
)
|
|
258
|
+
if system == "darwin":
|
|
259
|
+
if machine in ("arm64", "aarch64"):
|
|
260
|
+
return "darwin-arm64"
|
|
261
|
+
if machine in ("x86_64", "amd64"):
|
|
262
|
+
return "darwin-amd64"
|
|
263
|
+
if system == "windows":
|
|
264
|
+
if machine in ("amd64", "x86_64"):
|
|
265
|
+
return "windows-amd64"
|
|
266
|
+
|
|
267
|
+
manual = Path(user_data_dir(APP_NAME)) / "bin" / f"{system}-{machine}" / "ipfs"
|
|
268
|
+
raise IPFSBinaryNotFound(
|
|
269
|
+
f"Unsupported platform: {system}/{machine}.\n"
|
|
270
|
+
f" Supported: {', '.join(_SUPPORTED_TRIPLES)}.\n"
|
|
271
|
+
f" To use an unsupported platform, place an ipfs binary manually at:\n"
|
|
272
|
+
f" {manual}"
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _binary_name() -> str:
|
|
277
|
+
return "ipfs.exe" if sys.platform == "win32" else "ipfs"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _bundled_binary_path() -> Path:
|
|
281
|
+
"""Path inside the installed wheel — used for pre-placed binaries
|
|
282
|
+
(PyInstaller / Briefcase / air-gapped deploys)."""
|
|
283
|
+
return _base_dir() / "bin" / _platform_key() / _binary_name()
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _user_binary_path() -> Path:
|
|
287
|
+
"""Per-user install path. The auto-downloader writes here so we never
|
|
288
|
+
mutate site-packages — site-packages is a trust boundary owned by the
|
|
289
|
+
package manager, not by runtime code, and on a multi-user host any
|
|
290
|
+
other user with write access there could swap the binary."""
|
|
291
|
+
return Path(user_data_dir(APP_NAME)) / "bin" / _platform_key() / _binary_name()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _binary_path() -> Path:
|
|
295
|
+
"""Back-compat alias. Returns the user-install path that the downloader
|
|
296
|
+
writes to. Lookup order is implemented in _find_or_install_kubo()."""
|
|
297
|
+
return _user_binary_path()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
# ---------------------------------------------------------------------------
|
|
301
|
+
# Kubo auto-download
|
|
302
|
+
# ---------------------------------------------------------------------------
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# Kubo version strings look like ``v0.40.1`` or ``v0.41.0-rc1``. We
|
|
306
|
+
# interpolate KUBO_VERSION into a download URL, so anything outside this
|
|
307
|
+
# shape (path traversal, query injection, accidental whitespace) is
|
|
308
|
+
# rejected before it can reach _download(). Defence in depth: the redirect
|
|
309
|
+
# host pin and the baked-in SHA-512 table also block the attack, but a
|
|
310
|
+
# regex gate makes the failure mode obvious and local.
|
|
311
|
+
_KUBO_VERSION_RE = re.compile(r"^v\d+\.\d+\.\d+(-rc\d+)?$")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _archive_info() -> tuple[str, str]:
|
|
315
|
+
if not _KUBO_VERSION_RE.match(KUBO_VERSION):
|
|
316
|
+
raise IPFSBinaryNotFound(
|
|
317
|
+
f"Refusing to download with malformed KUBO_VERSION {KUBO_VERSION!r}. "
|
|
318
|
+
f"Expected something like 'v0.40.1' or 'v0.41.0-rc1'."
|
|
319
|
+
)
|
|
320
|
+
key = _platform_key()
|
|
321
|
+
os_name, arch = key.split("-", 1)
|
|
322
|
+
ext = "zip" if os_name == "windows" else "tar.gz"
|
|
323
|
+
name = f"kubo_{KUBO_VERSION}_{os_name}-{arch}.{ext}"
|
|
324
|
+
return f"{KUBO_DIST_BASE}/{KUBO_VERSION}/{name}", ext
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _download_session() -> requests.Session:
|
|
328
|
+
"""A session with retry+backoff and host-pinned redirects."""
|
|
329
|
+
try:
|
|
330
|
+
from urllib3.util.retry import Retry
|
|
331
|
+
except ImportError: # very old urllib3
|
|
332
|
+
from requests.packages.urllib3.util.retry import Retry # type: ignore
|
|
333
|
+
|
|
334
|
+
s = requests.Session()
|
|
335
|
+
retry = Retry(
|
|
336
|
+
total=3,
|
|
337
|
+
backoff_factor=2,
|
|
338
|
+
status_forcelist=(502, 503, 504, 520, 522, 524),
|
|
339
|
+
allowed_methods=frozenset(["GET", "HEAD"]),
|
|
340
|
+
raise_on_status=False,
|
|
341
|
+
)
|
|
342
|
+
adapter = _PinnedRedirectAdapter(max_retries=retry)
|
|
343
|
+
s.mount("http://", adapter)
|
|
344
|
+
s.mount("https://", adapter)
|
|
345
|
+
return s
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _check_redirect_origin(final_url: str) -> None:
|
|
349
|
+
host = (urlparse(final_url).hostname or "").lower()
|
|
350
|
+
for allowed in _ALLOWED_DOWNLOAD_HOSTS:
|
|
351
|
+
if host == allowed or host.endswith("." + allowed):
|
|
352
|
+
return
|
|
353
|
+
raise IPFSBinaryNotFound(
|
|
354
|
+
f"Refusing to follow redirect to unexpected host {host!r}. "
|
|
355
|
+
f"Allowed: {', '.join(_ALLOWED_DOWNLOAD_HOSTS)}."
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
class _PinnedRedirectAdapter(requests.adapters.HTTPAdapter):
|
|
360
|
+
"""HTTP adapter that re-checks the host pin at every redirect hop.
|
|
361
|
+
|
|
362
|
+
Plain ``allow_redirects=True`` only lets us inspect the *final* URL —
|
|
363
|
+
a 30x chain through ``attacker.example`` (with valid TLS) would still
|
|
364
|
+
leak the request headers and User-Agent before landing back on
|
|
365
|
+
``dist.ipfs.tech``. Override ``send`` to walk the chain manually,
|
|
366
|
+
enforcing _check_redirect_origin on each hop's destination before
|
|
367
|
+
we follow it. The hash check is still fail-closed against bad
|
|
368
|
+
bytes, but this closes the request-side-channel gap.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
def send(self, request, **kwargs): # type: ignore[override]
|
|
372
|
+
# Disable urllib3-level redirect following so each hop bubbles up
|
|
373
|
+
# here for inspection. ``requests.Session.resolve_redirects`` will
|
|
374
|
+
# then iterate over them, and we override its behaviour by
|
|
375
|
+
# rebuilding the chain ourselves.
|
|
376
|
+
kwargs["allow_redirects"] = False
|
|
377
|
+
resp = super().send(request, **kwargs)
|
|
378
|
+
# Walk redirects up to a sane cap (matches requests' default of 30).
|
|
379
|
+
seen = 0
|
|
380
|
+
while resp.is_redirect and seen < 30:
|
|
381
|
+
seen += 1
|
|
382
|
+
next_url = resp.headers.get("Location", "")
|
|
383
|
+
if not next_url:
|
|
384
|
+
break
|
|
385
|
+
# Resolve relative redirects against the previous URL.
|
|
386
|
+
from urllib.parse import urljoin
|
|
387
|
+
next_url = urljoin(resp.url, next_url)
|
|
388
|
+
_check_redirect_origin(next_url)
|
|
389
|
+
# Drop the body and follow.
|
|
390
|
+
try:
|
|
391
|
+
resp.close()
|
|
392
|
+
except Exception:
|
|
393
|
+
pass
|
|
394
|
+
new_req = request.copy()
|
|
395
|
+
new_req.url = next_url
|
|
396
|
+
# 303 / 302 / 301 on POST → GET, but we only do GET here.
|
|
397
|
+
resp = super().send(new_req, **kwargs)
|
|
398
|
+
return resp
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _download(url: str, dest: Path, timeout: int = 600) -> str:
|
|
402
|
+
"""Stream ``url`` into ``dest``, returning the SHA-512 hex digest.
|
|
403
|
+
|
|
404
|
+
The whole archive used to be buffered in memory, which let a hostile
|
|
405
|
+
origin (or any 30x to a misconfigured mirror) waste up to a gigabyte
|
|
406
|
+
of RAM before the SHA mismatch ever rejected it. Stream to disk
|
|
407
|
+
instead, hash incrementally, and abort early if the byte count
|
|
408
|
+
exceeds ``_MAX_DOWNLOAD_BYTES`` — the hash check is still
|
|
409
|
+
fail-closed, this just bounds the resource cost of a bad response.
|
|
410
|
+
"""
|
|
411
|
+
logger.info("Downloading %s", url)
|
|
412
|
+
last_pct_logged = -1
|
|
413
|
+
hasher = hashlib.sha512()
|
|
414
|
+
received = 0
|
|
415
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
416
|
+
with _download_session() as session, session.get(url, stream=True, timeout=timeout) as r:
|
|
417
|
+
_check_redirect_origin(r.url)
|
|
418
|
+
r.raise_for_status()
|
|
419
|
+
total = int(r.headers.get("Content-Length", 0))
|
|
420
|
+
with open(dest, "wb") as fh:
|
|
421
|
+
for chunk in r.iter_content(chunk_size=1 << 16):
|
|
422
|
+
if not chunk:
|
|
423
|
+
continue
|
|
424
|
+
received += len(chunk)
|
|
425
|
+
if received > _MAX_DOWNLOAD_BYTES:
|
|
426
|
+
raise IPFSBinaryNotFound(
|
|
427
|
+
f"Refusing to download more than {_MAX_DOWNLOAD_BYTES} bytes "
|
|
428
|
+
f"from {url} — got {received} bytes and counting. "
|
|
429
|
+
f"Either the origin is misbehaving or KUBO_VERSION points at "
|
|
430
|
+
f"something unexpected."
|
|
431
|
+
)
|
|
432
|
+
hasher.update(chunk)
|
|
433
|
+
fh.write(chunk)
|
|
434
|
+
if total:
|
|
435
|
+
pct = received * 100 // total
|
|
436
|
+
if pct >= last_pct_logged + 10:
|
|
437
|
+
logger.info(" %3d%% (%d / %d bytes)", pct, received, total)
|
|
438
|
+
last_pct_logged = pct
|
|
439
|
+
return hasher.hexdigest().lower()
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _expected_sha512(archive_url: str) -> str:
|
|
443
|
+
"""Look up the SHA-512 we expect for this version+platform.
|
|
444
|
+
|
|
445
|
+
Only the baked-in table is trusted. Fetching the companion ``.sha512``
|
|
446
|
+
from the same origin is rejected: an attacker who controls
|
|
447
|
+
``dist.ipfs.tech`` can swap both files together. New Kubo versions must
|
|
448
|
+
have their hash committed to ``kubo_checksums.py`` before they can be
|
|
449
|
+
installed.
|
|
450
|
+
"""
|
|
451
|
+
baked = kubo_checksums.known_checksum(KUBO_VERSION, _platform_key())
|
|
452
|
+
if baked is not None:
|
|
453
|
+
return baked.lower()
|
|
454
|
+
raise IPFSBinaryNotFound(
|
|
455
|
+
f"No baked-in SHA-512 for Kubo {KUBO_VERSION}/{_platform_key()}. "
|
|
456
|
+
f"Refusing to fetch a digest from the same origin as the archive — "
|
|
457
|
+
f"that defeats the verification. Add the hash to "
|
|
458
|
+
f"basic_ipfs/kubo_checksums.py and reinstall, or pin "
|
|
459
|
+
f"basic_ipfs.KUBO_VERSION to a version already in the table."
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
# Hardening caps for archive extraction. Real Kubo binaries are ~115 MB
|
|
464
|
+
# inside a tarball with a single top-level ``kubo/`` directory, so anything
|
|
465
|
+
# materially larger or more deeply nested is suspicious — refuse rather
|
|
466
|
+
# than allocate.
|
|
467
|
+
_MAX_ARCHIVE_MEMBER_BYTES = 512 * 1024 * 1024 # 512 MB per entry
|
|
468
|
+
_MAX_ARCHIVE_PATH_DEPTH = 4
|
|
469
|
+
|
|
470
|
+
# Hard ceiling on the streamed download itself — independent of the per-member
|
|
471
|
+
# cap above, because streaming happens before extraction. A hostile origin (or
|
|
472
|
+
# a Content-Length that lies) cannot cost us more than this in disk + time.
|
|
473
|
+
# Twice the per-member cap leaves comfortable headroom for archive overhead
|
|
474
|
+
# while still bounding the worst case.
|
|
475
|
+
_MAX_DOWNLOAD_BYTES = 2 * _MAX_ARCHIVE_MEMBER_BYTES
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
# Default cap when ``get(cid)`` is asked to return bytes (no output_path).
|
|
479
|
+
# Without this, a 4 GB CID returned via `bytes` allocates 4 GB of process
|
|
480
|
+
# RSS — fine for a CLI tool, lethal for a long-running service. Callers
|
|
481
|
+
# who really want the entire payload in memory should write to disk and
|
|
482
|
+
# read back, or set ``MAX_GET_BYTES`` to ``None`` to opt out.
|
|
483
|
+
MAX_GET_BYTES: int | None = 256 * 1024 * 1024
|
|
484
|
+
|
|
485
|
+
# Default per-request timeout for ``get(cid)``. ``None`` would mean "wait
|
|
486
|
+
# forever," which means an unreachable CID hangs the calling thread until
|
|
487
|
+
# the process is killed — the worst possible failure mode for a library
|
|
488
|
+
# advertised for long-running services. 300 s is high enough that a slow
|
|
489
|
+
# but progressing fetch succeeds and low enough that a stuck swarm fetch
|
|
490
|
+
# fails with a useful exception. Callers who really want unbounded waits
|
|
491
|
+
# can pass ``timeout=None`` per call or override this global.
|
|
492
|
+
DEFAULT_GET_TIMEOUT: float | None = 300.0
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _safe_member_name(name: str) -> bool:
|
|
496
|
+
# Defence in depth: extraction never honours the member's path (we always
|
|
497
|
+
# write to a fixed `dest`), but reject obviously-hostile names anyway so
|
|
498
|
+
# the helper matches its name and a future refactor that does honour the
|
|
499
|
+
# path stays safe.
|
|
500
|
+
if not name or name.startswith(("/", "\\")):
|
|
501
|
+
return False
|
|
502
|
+
p = Path(name)
|
|
503
|
+
if p.is_absolute():
|
|
504
|
+
return False
|
|
505
|
+
parts = p.parts
|
|
506
|
+
if not 0 < len(parts) <= _MAX_ARCHIVE_PATH_DEPTH:
|
|
507
|
+
return False
|
|
508
|
+
return all(part not in ("..",) for part in parts)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _extract_binary(archive_path: Path, ext: str, dest: Path) -> None:
|
|
512
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
513
|
+
if ext == "tar.gz":
|
|
514
|
+
with tarfile.open(str(archive_path), mode="r:gz") as tf:
|
|
515
|
+
for member in tf.getmembers():
|
|
516
|
+
if not member.isfile() or Path(member.name).name != "ipfs":
|
|
517
|
+
continue
|
|
518
|
+
if not _safe_member_name(member.name):
|
|
519
|
+
raise IPFSBinaryNotFound(
|
|
520
|
+
f"Refusing archive member with unsafe path: {member.name!r}"
|
|
521
|
+
)
|
|
522
|
+
if member.size > _MAX_ARCHIVE_MEMBER_BYTES:
|
|
523
|
+
raise IPFSBinaryNotFound(
|
|
524
|
+
f"Archive member {member.name!r} is {member.size} bytes — "
|
|
525
|
+
f"exceeds the {_MAX_ARCHIVE_MEMBER_BYTES}-byte cap."
|
|
526
|
+
)
|
|
527
|
+
src = tf.extractfile(member)
|
|
528
|
+
if src is not None:
|
|
529
|
+
with open(dest, "wb") as out:
|
|
530
|
+
shutil.copyfileobj(src, out, length=1 << 16)
|
|
531
|
+
break
|
|
532
|
+
else:
|
|
533
|
+
with zipfile.ZipFile(str(archive_path)) as zf:
|
|
534
|
+
for info in zf.infolist():
|
|
535
|
+
if Path(info.filename).name != "ipfs.exe":
|
|
536
|
+
continue
|
|
537
|
+
if not _safe_member_name(info.filename):
|
|
538
|
+
raise IPFSBinaryNotFound(
|
|
539
|
+
f"Refusing archive member with unsafe path: {info.filename!r}"
|
|
540
|
+
)
|
|
541
|
+
if info.file_size > _MAX_ARCHIVE_MEMBER_BYTES:
|
|
542
|
+
raise IPFSBinaryNotFound(
|
|
543
|
+
f"Archive member {info.filename!r} is {info.file_size} bytes — "
|
|
544
|
+
f"exceeds the {_MAX_ARCHIVE_MEMBER_BYTES}-byte cap."
|
|
545
|
+
)
|
|
546
|
+
with zf.open(info.filename) as src, open(dest, "wb") as out:
|
|
547
|
+
shutil.copyfileobj(src, out, length=1 << 16)
|
|
548
|
+
break
|
|
549
|
+
|
|
550
|
+
if not dest.exists():
|
|
551
|
+
raise IPFSBinaryNotFound(
|
|
552
|
+
f"Could not find ipfs binary inside Kubo archive for {_platform_key()}"
|
|
553
|
+
)
|
|
554
|
+
# Owner-execute only — site-packages is typically owned by the install
|
|
555
|
+
# user, and a multi-user host has its own kubo install.
|
|
556
|
+
dest.chmod(dest.stat().st_mode | stat.S_IEXEC)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _check_disk_space(dest: Path) -> None:
|
|
560
|
+
target = dest.parent
|
|
561
|
+
while not target.exists() and target != target.parent:
|
|
562
|
+
target = target.parent
|
|
563
|
+
try:
|
|
564
|
+
free = shutil.disk_usage(target).free
|
|
565
|
+
except OSError:
|
|
566
|
+
return # best-effort
|
|
567
|
+
if free < _INSTALL_FREE_BYTES:
|
|
568
|
+
raise IPFSBinaryNotFound(
|
|
569
|
+
f"Not enough disk space to install Kubo: {free // (1024 * 1024)} MB free at "
|
|
570
|
+
f"{target}, need at least {_INSTALL_FREE_BYTES // (1024 * 1024)} MB. "
|
|
571
|
+
f"Free up space or set basic_ipfs.REPO_PATH to a different disk."
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _write_provenance(binary_path: Path, url: str, sha512_hex: str) -> None:
|
|
576
|
+
"""Audit trail next to the binary: where it came from and how we verified it."""
|
|
577
|
+
prov = binary_path.parent / ".provenance.json"
|
|
578
|
+
data = {
|
|
579
|
+
"version": KUBO_VERSION,
|
|
580
|
+
"url": url,
|
|
581
|
+
"sha512": sha512_hex,
|
|
582
|
+
"verification": "baked-in",
|
|
583
|
+
"installed_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
584
|
+
"platform": _platform_key(),
|
|
585
|
+
}
|
|
586
|
+
try:
|
|
587
|
+
prov.write_text(json.dumps(data, indent=2))
|
|
588
|
+
except OSError:
|
|
589
|
+
pass # non-fatal
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _auto_download_kubo(dest: Path) -> None:
|
|
593
|
+
archive_url, ext = _archive_info()
|
|
594
|
+
_check_disk_space(dest)
|
|
595
|
+
expected = _expected_sha512(archive_url)
|
|
596
|
+
|
|
597
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
598
|
+
archive_tmp = dest.parent / (dest.name + ".archive.partial")
|
|
599
|
+
archive_tmp.unlink(missing_ok=True)
|
|
600
|
+
partial = dest.parent / (dest.name + ".partial")
|
|
601
|
+
partial.unlink(missing_ok=True)
|
|
602
|
+
|
|
603
|
+
try:
|
|
604
|
+
actual = _download(archive_url, archive_tmp)
|
|
605
|
+
if not hmac.compare_digest(actual, expected):
|
|
606
|
+
raise IPFSBinaryNotFound(
|
|
607
|
+
f"SHA-512 mismatch for Kubo {KUBO_VERSION} ({_platform_key()}): "
|
|
608
|
+
f"expected {expected}, got {actual}. Refusing to install an unverified binary."
|
|
609
|
+
)
|
|
610
|
+
# Extract to a sibling .partial path, then atomic rename. Prevents a
|
|
611
|
+
# half-written binary on interrupted installs.
|
|
612
|
+
_extract_binary(archive_tmp, ext, partial)
|
|
613
|
+
os.replace(partial, dest)
|
|
614
|
+
finally:
|
|
615
|
+
# Always clean up the streamed archive; never leave a half-written
|
|
616
|
+
# .partial executable around on a verification or extraction failure.
|
|
617
|
+
archive_tmp.unlink(missing_ok=True)
|
|
618
|
+
if partial.exists() and not dest.exists():
|
|
619
|
+
partial.unlink(missing_ok=True)
|
|
620
|
+
|
|
621
|
+
_write_provenance(dest, archive_url, expected)
|
|
622
|
+
logger.info("Kubo %s installed at %s (verified via baked-in SHA-512)",
|
|
623
|
+
KUBO_VERSION, dest)
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _find_or_install_kubo() -> Path:
|
|
627
|
+
"""user_data_dir bin → bundled bin (PyInstaller / pre-placed) → system PATH → auto-download.
|
|
628
|
+
|
|
629
|
+
Resolution order matters: a binary the user explicitly dropped into
|
|
630
|
+
user_data_dir wins so air-gapped or pre-vetted installs are honoured;
|
|
631
|
+
a binary inside the wheel wins next so PyInstaller / Briefcase
|
|
632
|
+
bundles still work; only then do we fall back to whatever ``ipfs``
|
|
633
|
+
is on PATH or download a fresh copy. The downloader always writes
|
|
634
|
+
to the user_data_dir path — never into site-packages.
|
|
635
|
+
"""
|
|
636
|
+
user = _user_binary_path()
|
|
637
|
+
if user.exists():
|
|
638
|
+
try:
|
|
639
|
+
user.chmod(user.stat().st_mode | stat.S_IEXEC)
|
|
640
|
+
except OSError as exc:
|
|
641
|
+
logger.warning("could not chmod %s: %s", user, exc)
|
|
642
|
+
return user
|
|
643
|
+
|
|
644
|
+
bundled = _bundled_binary_path()
|
|
645
|
+
if bundled.exists():
|
|
646
|
+
try:
|
|
647
|
+
bundled.chmod(bundled.stat().st_mode | stat.S_IEXEC)
|
|
648
|
+
except OSError as exc:
|
|
649
|
+
logger.warning("could not chmod %s: %s", bundled, exc)
|
|
650
|
+
return bundled
|
|
651
|
+
|
|
652
|
+
on_path = shutil.which("ipfs")
|
|
653
|
+
if on_path:
|
|
654
|
+
logger.info("Using system-installed Kubo at %s", on_path)
|
|
655
|
+
return Path(on_path)
|
|
656
|
+
|
|
657
|
+
logger.info("Kubo binary not found locally — downloading %s…", KUBO_VERSION)
|
|
658
|
+
try:
|
|
659
|
+
_auto_download_kubo(user)
|
|
660
|
+
except IPFSBinaryNotFound:
|
|
661
|
+
raise
|
|
662
|
+
except (requests.RequestException, OSError) as exc:
|
|
663
|
+
raise IPFSBinaryNotFound(
|
|
664
|
+
f"Failed to auto-download Kubo {KUBO_VERSION} for {_platform_key()}: {exc}\n"
|
|
665
|
+
f" Check your internet connection (or HTTPS_PROXY / REQUESTS_CA_BUNDLE), "
|
|
666
|
+
f" or place the binary manually at:\n {user}"
|
|
667
|
+
) from exc
|
|
668
|
+
return user
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
# ---------------------------------------------------------------------------
|
|
672
|
+
# Repo path
|
|
673
|
+
# ---------------------------------------------------------------------------
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _get_repo_path() -> Path:
|
|
677
|
+
if REPO_PATH is not None:
|
|
678
|
+
return Path(REPO_PATH)
|
|
679
|
+
return Path(user_data_dir(APP_NAME)) / "ipfs_repo"
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
# ---------------------------------------------------------------------------
|
|
683
|
+
# Small helpers
|
|
684
|
+
# ---------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _as_str_list(arg: str | Iterable[str]) -> list[str]:
|
|
688
|
+
"""Normalise a single string or an iterable of strings into a plain list."""
|
|
689
|
+
if isinstance(arg, str):
|
|
690
|
+
return [arg]
|
|
691
|
+
if isinstance(arg, (bytes, bytearray)):
|
|
692
|
+
raise TypeError("argument must be str or iterable of str, not bytes")
|
|
693
|
+
try:
|
|
694
|
+
items = list(arg)
|
|
695
|
+
except TypeError as exc:
|
|
696
|
+
raise TypeError(
|
|
697
|
+
f"argument must be str or iterable of str, not {type(arg).__name__}"
|
|
698
|
+
) from exc
|
|
699
|
+
for c in items:
|
|
700
|
+
if not isinstance(c, str):
|
|
701
|
+
raise TypeError(f"every element must be str, got {type(c).__name__}")
|
|
702
|
+
return items
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _chunked(items: list[str], size: int) -> Iterable[list[str]]:
|
|
706
|
+
for i in range(0, len(items), size):
|
|
707
|
+
yield items[i:i + size]
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _addr_score(multiaddr: str) -> int:
|
|
711
|
+
"""Score a multiaddr for sharing with others. Higher = better. -1 = exclude."""
|
|
712
|
+
try:
|
|
713
|
+
parts = multiaddr.split("/")
|
|
714
|
+
proto = parts[1] if len(parts) > 1 else ""
|
|
715
|
+
ip_str = parts[2] if len(parts) > 2 else ""
|
|
716
|
+
if proto == "ip4":
|
|
717
|
+
v4 = ipaddress.IPv4Address(ip_str)
|
|
718
|
+
if v4.is_loopback:
|
|
719
|
+
return -1
|
|
720
|
+
return 2 if v4.is_global else 1 # public > private/LAN
|
|
721
|
+
if proto == "ip6":
|
|
722
|
+
v6 = ipaddress.IPv6Address(ip_str)
|
|
723
|
+
if v6.is_loopback or v6.is_link_local:
|
|
724
|
+
return -1
|
|
725
|
+
return 2 if v6.is_global else 1
|
|
726
|
+
except Exception:
|
|
727
|
+
pass
|
|
728
|
+
return -1
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _quote_multipart_filename(name: str) -> str:
|
|
732
|
+
# Match what requests/urllib3 send: a literal filename inside double
|
|
733
|
+
# quotes, with backslash and double-quote backslash-escaped per RFC
|
|
734
|
+
# 7578 §4.2. CR/LF in a header value would forge a header injection,
|
|
735
|
+
# so refuse anything that contains them outright. Forward slashes are
|
|
736
|
+
# preserved — Kubo's add handler treats them as path separators when
|
|
737
|
+
# building the directory tree, which is exactly what add_folder()
|
|
738
|
+
# relies on.
|
|
739
|
+
if "\r" in name or "\n" in name:
|
|
740
|
+
raise ValueError(f"refusing CR/LF in multipart filename: {name!r}")
|
|
741
|
+
return name.replace("\\", "\\\\").replace('"', '\\"')
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
# A streaming-multipart "part": (filename, content_type, body) where body
|
|
745
|
+
# is bytes, an os.PathLike (opened lazily and streamed), or a file-like
|
|
746
|
+
# object supporting .read(n). The encoder reads in 64 KB chunks, so RAM
|
|
747
|
+
# usage is constant regardless of total upload size.
|
|
748
|
+
_MULTIPART_PART = tuple
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def _iter_multipart(
|
|
752
|
+
parts: list[tuple[str, str, bytes | os.PathLike[str] | io.IOBase]],
|
|
753
|
+
) -> tuple[str, Iterable[bytes]]:
|
|
754
|
+
"""Build a streaming multipart/form-data body.
|
|
755
|
+
|
|
756
|
+
Returns the boundary string and a generator that yields the wire body
|
|
757
|
+
in chunks. Suitable for ``requests.post(data=gen, headers={...})`` —
|
|
758
|
+
requests will pass through the iterable without buffering, so a
|
|
759
|
+
100 GB upload uses ~64 KB of RSS regardless of total size. (The
|
|
760
|
+
legacy ``files=`` path of requests reads every part into memory
|
|
761
|
+
via ``urllib3.filepost.encode_multipart_formdata``, which is the
|
|
762
|
+
OOM hazard this replaces.)
|
|
763
|
+
"""
|
|
764
|
+
# Random boundary unlikely to appear in any input. 16 hex bytes is
|
|
765
|
+
# ample uniqueness; the leading dashes match what RFC 7578 examples use.
|
|
766
|
+
boundary = "----basic_ipfs_" + os.urandom(16).hex()
|
|
767
|
+
boundary_bytes = boundary.encode("ascii")
|
|
768
|
+
|
|
769
|
+
def gen() -> Iterable[bytes]:
|
|
770
|
+
for filename, content_type, body in parts:
|
|
771
|
+
quoted = _quote_multipart_filename(filename)
|
|
772
|
+
yield b"--" + boundary_bytes + b"\r\n"
|
|
773
|
+
yield (
|
|
774
|
+
f'Content-Disposition: form-data; name="file"; filename="{quoted}"\r\n'
|
|
775
|
+
f"Content-Type: {content_type}\r\n\r\n"
|
|
776
|
+
).encode()
|
|
777
|
+
if isinstance(body, (bytes, bytearray, memoryview)):
|
|
778
|
+
yield bytes(body)
|
|
779
|
+
elif hasattr(body, "read"):
|
|
780
|
+
while True:
|
|
781
|
+
chunk = body.read(1 << 16)
|
|
782
|
+
if not chunk:
|
|
783
|
+
break
|
|
784
|
+
yield chunk
|
|
785
|
+
else:
|
|
786
|
+
with open(os.fspath(body), "rb") as fh:
|
|
787
|
+
while True:
|
|
788
|
+
chunk = fh.read(1 << 16)
|
|
789
|
+
if not chunk:
|
|
790
|
+
break
|
|
791
|
+
yield chunk
|
|
792
|
+
yield b"\r\n"
|
|
793
|
+
yield b"--" + boundary_bytes + b"--\r\n"
|
|
794
|
+
|
|
795
|
+
return boundary, gen()
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _build_api_session() -> requests.Session:
|
|
799
|
+
"""Session for the local Kubo HTTP API.
|
|
800
|
+
|
|
801
|
+
Retries connection-level failures only — never status codes, since
|
|
802
|
+
most Kubo POSTs (`add`, `pin/add`, `pin/rm`) have observable side
|
|
803
|
+
effects per attempt and a server-side 5xx mid-write means the
|
|
804
|
+
operation may already have succeeded in part. Connection errors
|
|
805
|
+
(ECONNREFUSED, broken pipe during a graceful daemon restart) are
|
|
806
|
+
safe to redrive; they happen mostly when the daemon is migrating
|
|
807
|
+
its repo or rotating logs.
|
|
808
|
+
"""
|
|
809
|
+
try:
|
|
810
|
+
from urllib3.util.retry import Retry
|
|
811
|
+
except ImportError: # very old urllib3
|
|
812
|
+
from requests.packages.urllib3.util.retry import Retry # type: ignore
|
|
813
|
+
|
|
814
|
+
s = requests.Session()
|
|
815
|
+
retry = Retry(
|
|
816
|
+
total=2,
|
|
817
|
+
connect=2,
|
|
818
|
+
read=0,
|
|
819
|
+
status=0,
|
|
820
|
+
backoff_factor=0.25,
|
|
821
|
+
# `False` means "no method restriction" in urllib3 — connection
|
|
822
|
+
# errors happen before any request is sent, so retrying every
|
|
823
|
+
# method is safe. urllib3's type stub for allowed_methods is too
|
|
824
|
+
# narrow to accept the documented sentinel.
|
|
825
|
+
allowed_methods=False, # type: ignore[arg-type]
|
|
826
|
+
raise_on_status=False,
|
|
827
|
+
)
|
|
828
|
+
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
|
|
829
|
+
s.mount("http://", adapter)
|
|
830
|
+
s.mount("https://", adapter)
|
|
831
|
+
return s
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def _is_port_in_use(host: str, port: int) -> bool:
|
|
835
|
+
"""True if something is accepting on host:port."""
|
|
836
|
+
try:
|
|
837
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
838
|
+
s.settimeout(1.0)
|
|
839
|
+
s.connect((host, port))
|
|
840
|
+
return True
|
|
841
|
+
except OSError:
|
|
842
|
+
return False
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _secure_open(path: Path, mode: int = 0o600) -> int:
|
|
846
|
+
"""Open ``path`` for writing with restrictive perms applied atomically.
|
|
847
|
+
|
|
848
|
+
Closes the TOCTOU window between ``write_text`` and ``chmod`` — without
|
|
849
|
+
this, the file briefly exists with the process umask before its mode is
|
|
850
|
+
tightened, so a co-tenant can race in and read a credential. Returns an
|
|
851
|
+
OS-level file descriptor; caller is responsible for closing it.
|
|
852
|
+
"""
|
|
853
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
|
854
|
+
if hasattr(os, "O_NOFOLLOW"):
|
|
855
|
+
flags |= os.O_NOFOLLOW # refuse to write through a symlink swap
|
|
856
|
+
return os.open(str(path), flags, mode)
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
def _secure_write_text(path: Path, text: str, mode: int = 0o600) -> None:
|
|
860
|
+
"""Write ``text`` to ``path`` with ``mode`` applied at create time."""
|
|
861
|
+
fd = _secure_open(path, mode=mode)
|
|
862
|
+
try:
|
|
863
|
+
with os.fdopen(fd, "w") as fh:
|
|
864
|
+
fh.write(text)
|
|
865
|
+
except BaseException:
|
|
866
|
+
# Clean up a half-written file on any failure (KeyboardInterrupt
|
|
867
|
+
# included) so the next run starts from a known-empty state.
|
|
868
|
+
try:
|
|
869
|
+
path.unlink()
|
|
870
|
+
except OSError:
|
|
871
|
+
pass
|
|
872
|
+
raise
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def _secure_mkdir(path: Path, mode: int = 0o700) -> None:
|
|
876
|
+
"""Create directories up to ``path`` with restrictive ``mode``.
|
|
877
|
+
|
|
878
|
+
Each newly-created leaf is mkdir'd with the requested mode so an
|
|
879
|
+
attacker can't snapshot the directory's contents during the umask
|
|
880
|
+
window between ``mkdir`` and ``chmod``. Existing directories are left
|
|
881
|
+
alone — the operator may have intentionally widened them.
|
|
882
|
+
"""
|
|
883
|
+
path = Path(path)
|
|
884
|
+
if path.exists():
|
|
885
|
+
return
|
|
886
|
+
parent = path.parent
|
|
887
|
+
if parent != path and not parent.exists():
|
|
888
|
+
_secure_mkdir(parent, mode=mode)
|
|
889
|
+
try:
|
|
890
|
+
os.mkdir(str(path), mode)
|
|
891
|
+
except FileExistsError:
|
|
892
|
+
return
|
|
893
|
+
# umask can still mask bits off; force them on for the leaf.
|
|
894
|
+
try:
|
|
895
|
+
os.chmod(str(path), mode)
|
|
896
|
+
except OSError:
|
|
897
|
+
pass
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# ---------------------------------------------------------------------------
|
|
901
|
+
# IPFSManager — process-singleton daemon lifecycle (managed via module locks)
|
|
902
|
+
# ---------------------------------------------------------------------------
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
class IPFSManager:
|
|
906
|
+
"""Manages the Kubo daemon subprocess for the lifetime of the process.
|
|
907
|
+
|
|
908
|
+
Public surface used by the module-level functions: ``start``, ``stop``,
|
|
909
|
+
``add``, ``announce``, ``add_folder``, ``cat``, ``pin_add``, ``pin_rm``,
|
|
910
|
+
``pin_ls``, ``pin_check``, ``block_exists``, ``repo_gc``, ``swarm_peers``,
|
|
911
|
+
``swarm_connect``, ``my_addrs``, ``status``.
|
|
912
|
+
"""
|
|
913
|
+
|
|
914
|
+
def __init__(self) -> None:
|
|
915
|
+
self._binary: Path | None = None
|
|
916
|
+
self._repo: Path | None = None
|
|
917
|
+
self._process: subprocess.Popen[bytes] | None = None
|
|
918
|
+
self._owns_daemon = False
|
|
919
|
+
self._api_url = f"http://{API_HOST}:{API_PORT}/api/v0"
|
|
920
|
+
self._session: requests.Session | None = None
|
|
921
|
+
self._log_path: Path | None = None
|
|
922
|
+
self._log_file: io.IOBase | None = None # writable handle for subprocess stderr
|
|
923
|
+
self._initialised = False
|
|
924
|
+
|
|
925
|
+
# ---------------- startup ----------------
|
|
926
|
+
|
|
927
|
+
def start(self) -> None:
|
|
928
|
+
if self._initialised:
|
|
929
|
+
return
|
|
930
|
+
|
|
931
|
+
self._binary = _find_or_install_kubo()
|
|
932
|
+
self._repo = _get_repo_path()
|
|
933
|
+
self._session = _build_api_session()
|
|
934
|
+
# Re-read config every start so callers can change API_HOST/API_PORT
|
|
935
|
+
# between stop() and start().
|
|
936
|
+
self._api_url = f"http://{API_HOST}:{API_PORT}/api/v0"
|
|
937
|
+
|
|
938
|
+
self._ensure_repo()
|
|
939
|
+
|
|
940
|
+
# Apply config to the on-disk repo, but only if no daemon is already
|
|
941
|
+
# running — we don't want to edit another process's live config out
|
|
942
|
+
# from under it.
|
|
943
|
+
if not self._is_api_up():
|
|
944
|
+
self._configure_api_address()
|
|
945
|
+
self._configure_gateway_address()
|
|
946
|
+
self._configure_storage_limit()
|
|
947
|
+
self._configure_swarm_addresses()
|
|
948
|
+
|
|
949
|
+
_swarm_key_warn_if_world_readable()
|
|
950
|
+
self._start_daemon()
|
|
951
|
+
self._wait_for_api()
|
|
952
|
+
|
|
953
|
+
# atexit registration is module-scoped (see _ensure_atexit_registered
|
|
954
|
+
# below) so successive stop()/start() cycles don't accumulate stale
|
|
955
|
+
# bound-method references on the atexit chain.
|
|
956
|
+
_ensure_atexit_registered()
|
|
957
|
+
|
|
958
|
+
self._initialised = True
|
|
959
|
+
logger.info("IPFS node ready — repo: %s", self._repo)
|
|
960
|
+
|
|
961
|
+
def _env(self) -> dict[str, str]:
|
|
962
|
+
env = os.environ.copy()
|
|
963
|
+
env["IPFS_PATH"] = str(self._repo)
|
|
964
|
+
return env
|
|
965
|
+
|
|
966
|
+
def _run_cli(
|
|
967
|
+
self,
|
|
968
|
+
*args: str,
|
|
969
|
+
check: bool = True,
|
|
970
|
+
timeout: int | None = 30,
|
|
971
|
+
) -> subprocess.CompletedProcess[str]:
|
|
972
|
+
return subprocess.run(
|
|
973
|
+
[str(self._binary), *args],
|
|
974
|
+
env=self._env(),
|
|
975
|
+
capture_output=True,
|
|
976
|
+
text=True,
|
|
977
|
+
timeout=timeout,
|
|
978
|
+
check=check,
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
def _ensure_repo(self) -> None:
|
|
982
|
+
assert self._repo is not None
|
|
983
|
+
# Create with restrictive mode atomically so a multi-user host can't
|
|
984
|
+
# peek into a freshly-created repo before chmod runs. mkdir() applies
|
|
985
|
+
# the mode under the process umask; on POSIX we re-apply afterwards
|
|
986
|
+
# to defeat a 0o077 umask, but the directory was never world-traversable
|
|
987
|
+
# because mkdir(mode=0o700) clamps via the umask only for *additional*
|
|
988
|
+
# bits — owner bits stay set. Existing dirs are left as-is (operator
|
|
989
|
+
# may have widened them deliberately).
|
|
990
|
+
if os.name == "posix":
|
|
991
|
+
_secure_mkdir(self._repo, mode=0o700)
|
|
992
|
+
else:
|
|
993
|
+
self._repo.mkdir(parents=True, exist_ok=True)
|
|
994
|
+
config_path = self._repo / "config"
|
|
995
|
+
|
|
996
|
+
if not config_path.exists():
|
|
997
|
+
logger.info("Initialising IPFS repo at %s", self._repo)
|
|
998
|
+
try:
|
|
999
|
+
self._run_cli("init", "--profile", "server")
|
|
1000
|
+
except subprocess.CalledProcessError as exc:
|
|
1001
|
+
raise IPFSError(
|
|
1002
|
+
f"Failed to initialise IPFS repo at {self._repo}: {exc.stderr.strip()}"
|
|
1003
|
+
) from exc
|
|
1004
|
+
return
|
|
1005
|
+
|
|
1006
|
+
# Cheap corruption check: config exists and parses as JSON.
|
|
1007
|
+
try:
|
|
1008
|
+
json.loads(config_path.read_text())
|
|
1009
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
1010
|
+
raise IPFSRepoCorrupt(
|
|
1011
|
+
f"IPFS repo config at {config_path} is unreadable: {exc}\n"
|
|
1012
|
+
f" If salvageable, fix it manually. Otherwise delete\n"
|
|
1013
|
+
f" {self._repo}\n"
|
|
1014
|
+
f" and let basic_ipfs recreate it (you will lose local pins)."
|
|
1015
|
+
) from exc
|
|
1016
|
+
|
|
1017
|
+
def _configure_api_address(self) -> None:
|
|
1018
|
+
try:
|
|
1019
|
+
self._run_cli(
|
|
1020
|
+
"config", "Addresses.API",
|
|
1021
|
+
f"/ip4/{API_HOST}/tcp/{API_PORT}",
|
|
1022
|
+
)
|
|
1023
|
+
except subprocess.CalledProcessError as exc:
|
|
1024
|
+
logger.warning("Could not set API address: %s", exc.stderr)
|
|
1025
|
+
|
|
1026
|
+
def _configure_gateway_address(self) -> None:
|
|
1027
|
+
try:
|
|
1028
|
+
self._run_cli(
|
|
1029
|
+
"config", "Addresses.Gateway",
|
|
1030
|
+
f"/ip4/{GATEWAY_HOST}/tcp/{GATEWAY_PORT}",
|
|
1031
|
+
)
|
|
1032
|
+
except subprocess.CalledProcessError as exc:
|
|
1033
|
+
logger.warning("Could not set Gateway address: %s", exc.stderr)
|
|
1034
|
+
|
|
1035
|
+
def _configure_storage_limit(self) -> None:
|
|
1036
|
+
if not STORAGE_MAX:
|
|
1037
|
+
return
|
|
1038
|
+
try:
|
|
1039
|
+
self._run_cli("config", "Datastore.StorageMax", STORAGE_MAX)
|
|
1040
|
+
except subprocess.CalledProcessError as exc:
|
|
1041
|
+
logger.warning("Could not set storage limit: %s", exc.stderr)
|
|
1042
|
+
|
|
1043
|
+
def _configure_swarm_addresses(self) -> None:
|
|
1044
|
+
if SWARM_ADDRESSES is None:
|
|
1045
|
+
return
|
|
1046
|
+
try:
|
|
1047
|
+
self._run_cli(
|
|
1048
|
+
"config", "Addresses.Swarm",
|
|
1049
|
+
"--json", json.dumps(list(SWARM_ADDRESSES)),
|
|
1050
|
+
)
|
|
1051
|
+
except subprocess.CalledProcessError as exc:
|
|
1052
|
+
logger.warning("Could not set Addresses.Swarm: %s", exc.stderr)
|
|
1053
|
+
|
|
1054
|
+
def _open_daemon_log(self) -> None:
|
|
1055
|
+
"""Open the daemon log file, rotating if the previous one is large."""
|
|
1056
|
+
assert self._repo is not None
|
|
1057
|
+
self._log_path = self._repo / _DAEMON_LOG_NAME
|
|
1058
|
+
try:
|
|
1059
|
+
if self._log_path.exists() and self._log_path.stat().st_size > _DAEMON_LOG_MAX_BYTES:
|
|
1060
|
+
old = self._log_path.with_suffix(self._log_path.suffix + ".old")
|
|
1061
|
+
if old.exists():
|
|
1062
|
+
try:
|
|
1063
|
+
old.unlink()
|
|
1064
|
+
except OSError:
|
|
1065
|
+
pass
|
|
1066
|
+
self._log_path.replace(old)
|
|
1067
|
+
except OSError:
|
|
1068
|
+
pass # rotation is best-effort
|
|
1069
|
+
# Buffered: stderr write rates from Kubo are low, no need for unbuffered.
|
|
1070
|
+
# Open via os.open with mode 0o600 on POSIX so the file is never
|
|
1071
|
+
# world-readable, even briefly. The daemon log captures CIDs, peer
|
|
1072
|
+
# IDs, multiaddrs — fingerprintable activity.
|
|
1073
|
+
if os.name == "posix":
|
|
1074
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
|
|
1075
|
+
fd = os.open(str(self._log_path), flags, 0o600)
|
|
1076
|
+
self._log_file = os.fdopen(fd, "ab")
|
|
1077
|
+
# If the file pre-existed with a wider mode, narrow it.
|
|
1078
|
+
try:
|
|
1079
|
+
os.chmod(self._log_path, 0o600)
|
|
1080
|
+
except OSError as exc:
|
|
1081
|
+
logger.warning("could not restrict log perms at %s: %s", self._log_path, exc)
|
|
1082
|
+
else:
|
|
1083
|
+
self._log_file = open(self._log_path, "ab")
|
|
1084
|
+
|
|
1085
|
+
def _start_daemon(self) -> None:
|
|
1086
|
+
if self._is_api_up():
|
|
1087
|
+
logger.info("IPFS API already reachable — skipping daemon launch")
|
|
1088
|
+
return
|
|
1089
|
+
|
|
1090
|
+
# If something is bound to the port and isn't a Kubo daemon, fail loud.
|
|
1091
|
+
if _is_port_in_use(API_HOST, API_PORT):
|
|
1092
|
+
raise IPFSPortInUse(
|
|
1093
|
+
f"Port {API_HOST}:{API_PORT} is in use but does not look like an IPFS daemon. "
|
|
1094
|
+
f"Set basic_ipfs.API_PORT to a different port, or stop whatever is using it."
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1097
|
+
self._open_daemon_log()
|
|
1098
|
+
|
|
1099
|
+
cmd = [
|
|
1100
|
+
str(self._binary), "daemon",
|
|
1101
|
+
"--migrate=true",
|
|
1102
|
+
]
|
|
1103
|
+
# Critical: never pass PIPE for stdout/stderr on a long-lived process;
|
|
1104
|
+
# the OS pipe buffer (~64 KB) fills and the daemon blocks on its next
|
|
1105
|
+
# write. Discard stdout, send stderr to a rotating file.
|
|
1106
|
+
popen_kwargs: dict[str, Any] = {
|
|
1107
|
+
"env": self._env(),
|
|
1108
|
+
"stdout": subprocess.DEVNULL,
|
|
1109
|
+
"stderr": self._log_file,
|
|
1110
|
+
}
|
|
1111
|
+
if sys.platform == "win32":
|
|
1112
|
+
popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
|
|
1113
|
+
|
|
1114
|
+
self._process = subprocess.Popen(cmd, **popen_kwargs)
|
|
1115
|
+
self._owns_daemon = True
|
|
1116
|
+
logger.info("Kubo daemon started (pid %d)", self._process.pid)
|
|
1117
|
+
|
|
1118
|
+
def _read_log_tail(self, n_bytes: int = 4096) -> str:
|
|
1119
|
+
if not self._log_path or not self._log_path.exists():
|
|
1120
|
+
return "(no daemon log yet)"
|
|
1121
|
+
try:
|
|
1122
|
+
with open(self._log_path, "rb") as fh:
|
|
1123
|
+
fh.seek(0, os.SEEK_END)
|
|
1124
|
+
size = fh.tell()
|
|
1125
|
+
fh.seek(max(0, size - n_bytes))
|
|
1126
|
+
return fh.read().decode(errors="replace")
|
|
1127
|
+
except OSError:
|
|
1128
|
+
return "(log unreadable)"
|
|
1129
|
+
|
|
1130
|
+
def _wait_for_api(self) -> None:
|
|
1131
|
+
deadline = time.monotonic() + DAEMON_STARTUP_TIMEOUT
|
|
1132
|
+
while time.monotonic() < deadline:
|
|
1133
|
+
if self._is_api_up():
|
|
1134
|
+
return
|
|
1135
|
+
if self._process and self._process.poll() is not None:
|
|
1136
|
+
tail = self._read_log_tail()
|
|
1137
|
+
lower = tail.lower()
|
|
1138
|
+
if "lock" in lower and "repo" in lower:
|
|
1139
|
+
raise IPFSRepoLocked(
|
|
1140
|
+
f"The IPFS repo at {self._repo} is locked by another process. "
|
|
1141
|
+
"Only one Kubo daemon can use a repo at a time. Stop the other "
|
|
1142
|
+
"process, or set basic_ipfs.REPO_PATH to a different path.\n"
|
|
1143
|
+
f"--- daemon log tail ---\n{tail}"
|
|
1144
|
+
)
|
|
1145
|
+
raise IPFSError(
|
|
1146
|
+
f"Kubo daemon exited unexpectedly (code {self._process.returncode}):\n{tail}"
|
|
1147
|
+
)
|
|
1148
|
+
time.sleep(0.5)
|
|
1149
|
+
raise IPFSDaemonTimeout(
|
|
1150
|
+
f"Kubo daemon did not become ready within {DAEMON_STARTUP_TIMEOUT}s. "
|
|
1151
|
+
f"See log: {self._log_path}"
|
|
1152
|
+
)
|
|
1153
|
+
|
|
1154
|
+
# ---------------- shutdown ----------------
|
|
1155
|
+
|
|
1156
|
+
def stop(self) -> None:
|
|
1157
|
+
if not self._initialised:
|
|
1158
|
+
return
|
|
1159
|
+
|
|
1160
|
+
if not self._owns_daemon:
|
|
1161
|
+
# We piggy-backed on a daemon another process started. Don't
|
|
1162
|
+
# shut it down — that'd pull the rug out from under them.
|
|
1163
|
+
self._close_session_and_log()
|
|
1164
|
+
self._initialised = False
|
|
1165
|
+
return
|
|
1166
|
+
|
|
1167
|
+
logger.info("Shutting down IPFS daemon…")
|
|
1168
|
+
try:
|
|
1169
|
+
self._post("shutdown", timeout=10)
|
|
1170
|
+
except Exception:
|
|
1171
|
+
pass
|
|
1172
|
+
|
|
1173
|
+
if self._process:
|
|
1174
|
+
try:
|
|
1175
|
+
self._process.wait(timeout=10)
|
|
1176
|
+
except subprocess.TimeoutExpired:
|
|
1177
|
+
if sys.platform == "win32":
|
|
1178
|
+
self._process.terminate()
|
|
1179
|
+
else:
|
|
1180
|
+
self._process.send_signal(signal.SIGTERM)
|
|
1181
|
+
try:
|
|
1182
|
+
self._process.wait(timeout=5)
|
|
1183
|
+
except subprocess.TimeoutExpired:
|
|
1184
|
+
self._process.kill()
|
|
1185
|
+
|
|
1186
|
+
self._close_session_and_log()
|
|
1187
|
+
self._owns_daemon = False
|
|
1188
|
+
self._initialised = False
|
|
1189
|
+
|
|
1190
|
+
def _close_session_and_log(self) -> None:
|
|
1191
|
+
if self._session is not None:
|
|
1192
|
+
try:
|
|
1193
|
+
self._session.close()
|
|
1194
|
+
except Exception:
|
|
1195
|
+
pass
|
|
1196
|
+
self._session = None
|
|
1197
|
+
if self._log_file is not None:
|
|
1198
|
+
try:
|
|
1199
|
+
self._log_file.close()
|
|
1200
|
+
except Exception:
|
|
1201
|
+
pass
|
|
1202
|
+
self._log_file = None
|
|
1203
|
+
|
|
1204
|
+
# ---------------- HTTP helpers ----------------
|
|
1205
|
+
|
|
1206
|
+
def _is_api_up(self) -> bool:
|
|
1207
|
+
# Confirm the listener on API_HOST:API_PORT is actually a Kubo
|
|
1208
|
+
# daemon, not a stray service that happens to return 200 on POST.
|
|
1209
|
+
# Without this check, a port collision on a shared dev host could
|
|
1210
|
+
# silently route add/pin calls into an unrelated service.
|
|
1211
|
+
if self._session is None:
|
|
1212
|
+
return False
|
|
1213
|
+
try:
|
|
1214
|
+
r = self._session.post(f"{self._api_url}/version", timeout=2)
|
|
1215
|
+
except Exception:
|
|
1216
|
+
return False
|
|
1217
|
+
if r.status_code != 200:
|
|
1218
|
+
return False
|
|
1219
|
+
try:
|
|
1220
|
+
body = r.json()
|
|
1221
|
+
except Exception:
|
|
1222
|
+
return False
|
|
1223
|
+
return isinstance(body, dict) and isinstance(body.get("Version"), str)
|
|
1224
|
+
|
|
1225
|
+
def _post(
|
|
1226
|
+
self,
|
|
1227
|
+
endpoint: str,
|
|
1228
|
+
*,
|
|
1229
|
+
params: dict[str, Any] | list[tuple[str, Any]] | None = None,
|
|
1230
|
+
files: Any | None = None,
|
|
1231
|
+
data: Any | None = None,
|
|
1232
|
+
headers: dict[str, str] | None = None,
|
|
1233
|
+
stream: bool = False,
|
|
1234
|
+
timeout: float | None = 60,
|
|
1235
|
+
) -> requests.Response:
|
|
1236
|
+
assert self._session is not None
|
|
1237
|
+
url = f"{self._api_url}/{endpoint}"
|
|
1238
|
+
try:
|
|
1239
|
+
resp = self._session.post(
|
|
1240
|
+
url, params=params, files=files, data=data, headers=headers,
|
|
1241
|
+
stream=stream, timeout=timeout,
|
|
1242
|
+
)
|
|
1243
|
+
except requests.ConnectionError as exc:
|
|
1244
|
+
raise IPFSOperationError(
|
|
1245
|
+
f"Cannot reach IPFS daemon at {url}. Is it running?"
|
|
1246
|
+
) from exc
|
|
1247
|
+
|
|
1248
|
+
if not resp.ok:
|
|
1249
|
+
kubo_code: int | None = None
|
|
1250
|
+
kubo_type: str | None = None
|
|
1251
|
+
kubo_message: str | None = None
|
|
1252
|
+
try:
|
|
1253
|
+
body = resp.json()
|
|
1254
|
+
except Exception:
|
|
1255
|
+
body = None
|
|
1256
|
+
if isinstance(body, dict):
|
|
1257
|
+
kubo_message = body.get("Message")
|
|
1258
|
+
raw_code = body.get("Code")
|
|
1259
|
+
if isinstance(raw_code, int):
|
|
1260
|
+
kubo_code = raw_code
|
|
1261
|
+
raw_type = body.get("Type")
|
|
1262
|
+
if isinstance(raw_type, str):
|
|
1263
|
+
kubo_type = raw_type
|
|
1264
|
+
# Truncate raw bodies aggressively — a misconfigured reverse
|
|
1265
|
+
# proxy can return multi-MB HTML pages that would otherwise
|
|
1266
|
+
# land in the exception message and the daemon log.
|
|
1267
|
+
raw = resp.text or ""
|
|
1268
|
+
if len(raw) > 1024:
|
|
1269
|
+
raw = raw[:1024] + f"... [truncated {len(raw) - 1024} bytes]"
|
|
1270
|
+
detail = kubo_message if kubo_message is not None else raw
|
|
1271
|
+
raise IPFSOperationError(
|
|
1272
|
+
f"IPFS API error [{resp.status_code}] {endpoint}: {detail}",
|
|
1273
|
+
status_code=resp.status_code,
|
|
1274
|
+
kubo_code=kubo_code,
|
|
1275
|
+
kubo_type=kubo_type,
|
|
1276
|
+
kubo_message=kubo_message,
|
|
1277
|
+
)
|
|
1278
|
+
return resp
|
|
1279
|
+
|
|
1280
|
+
# ---------------- core operations ----------------
|
|
1281
|
+
|
|
1282
|
+
def _add(
|
|
1283
|
+
self,
|
|
1284
|
+
source: str | bytes | os.PathLike[str],
|
|
1285
|
+
pin: bool,
|
|
1286
|
+
provide: bool = True,
|
|
1287
|
+
only_hash: bool = False,
|
|
1288
|
+
timeout: float | None = None,
|
|
1289
|
+
) -> str:
|
|
1290
|
+
# Build a single multipart part that streams from disk (or from
|
|
1291
|
+
# bytes wrapped in BytesIO). _iter_multipart() ensures the upload
|
|
1292
|
+
# never holds more than 64 KB at a time in RAM, so even a multi-GB
|
|
1293
|
+
# add() is RSS-flat.
|
|
1294
|
+
body: bytes | io.IOBase
|
|
1295
|
+
if isinstance(source, (bytes, bytearray, memoryview)):
|
|
1296
|
+
body = bytes(source)
|
|
1297
|
+
filename = "data"
|
|
1298
|
+
elif isinstance(source, (str, os.PathLike)):
|
|
1299
|
+
path = Path(os.fspath(source))
|
|
1300
|
+
if not path.exists():
|
|
1301
|
+
raise FileNotFoundError(f"File not found: {path}")
|
|
1302
|
+
if path.is_dir():
|
|
1303
|
+
raise IsADirectoryError(
|
|
1304
|
+
f"{path} is a directory — use add_folder() for directories."
|
|
1305
|
+
)
|
|
1306
|
+
# Hand the open handle to the streaming generator; it closes
|
|
1307
|
+
# after EOF. We intentionally do not use a context manager here
|
|
1308
|
+
# because the generator owns the read lifecycle.
|
|
1309
|
+
body = open(path, "rb")
|
|
1310
|
+
filename = path.name
|
|
1311
|
+
else:
|
|
1312
|
+
raise TypeError(
|
|
1313
|
+
f"source must be str, PathLike, or bytes, not {type(source).__name__}"
|
|
1314
|
+
)
|
|
1315
|
+
|
|
1316
|
+
params: list[tuple[str, str]] = [
|
|
1317
|
+
("pin", "true" if pin else "false"),
|
|
1318
|
+
("cid-version", "1"),
|
|
1319
|
+
]
|
|
1320
|
+
if not provide:
|
|
1321
|
+
# Skip DHT announce. Content is still locally addressable; peers
|
|
1322
|
+
# who already know the CID can fetch via bitswap, but no provider
|
|
1323
|
+
# record is published.
|
|
1324
|
+
params.append(("provide", "false"))
|
|
1325
|
+
if only_hash:
|
|
1326
|
+
# Compute the CID without writing a block to the repo, pinning,
|
|
1327
|
+
# or announcing anything. Pin/provide flags are silently ignored
|
|
1328
|
+
# by Kubo when only-hash is set; we still send the same shape so
|
|
1329
|
+
# the request is uniform with the regular add path.
|
|
1330
|
+
params.append(("only-hash", "true"))
|
|
1331
|
+
|
|
1332
|
+
boundary, body_iter = _iter_multipart(
|
|
1333
|
+
[(filename, "application/octet-stream", body)]
|
|
1334
|
+
)
|
|
1335
|
+
try:
|
|
1336
|
+
resp = self._post(
|
|
1337
|
+
"add",
|
|
1338
|
+
params=params,
|
|
1339
|
+
data=body_iter,
|
|
1340
|
+
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
|
1341
|
+
timeout=timeout,
|
|
1342
|
+
)
|
|
1343
|
+
finally:
|
|
1344
|
+
if hasattr(body, "close"):
|
|
1345
|
+
try:
|
|
1346
|
+
body.close()
|
|
1347
|
+
except Exception:
|
|
1348
|
+
pass
|
|
1349
|
+
|
|
1350
|
+
lines = [ln for ln in resp.text.strip().splitlines() if ln.strip()]
|
|
1351
|
+
cid: str = json.loads(lines[-1])["Hash"]
|
|
1352
|
+
return cid
|
|
1353
|
+
|
|
1354
|
+
def add(
|
|
1355
|
+
self,
|
|
1356
|
+
source: str | bytes | os.PathLike[str],
|
|
1357
|
+
provide: bool = True,
|
|
1358
|
+
*,
|
|
1359
|
+
timeout: float | None = None,
|
|
1360
|
+
) -> str:
|
|
1361
|
+
return self._add(source, pin=True, provide=provide, timeout=timeout)
|
|
1362
|
+
|
|
1363
|
+
def announce(
|
|
1364
|
+
self,
|
|
1365
|
+
source: str | bytes | os.PathLike[str],
|
|
1366
|
+
provide: bool = True,
|
|
1367
|
+
*,
|
|
1368
|
+
timeout: float | None = None,
|
|
1369
|
+
) -> str:
|
|
1370
|
+
return self._add(source, pin=False, provide=provide, timeout=timeout)
|
|
1371
|
+
|
|
1372
|
+
def compute_cid_locally(
|
|
1373
|
+
self,
|
|
1374
|
+
source: str | bytes | os.PathLike[str],
|
|
1375
|
+
*,
|
|
1376
|
+
timeout: float | None = None,
|
|
1377
|
+
) -> str:
|
|
1378
|
+
# only-hash=true is Kubo's "chunk and hash, do not write" mode. No
|
|
1379
|
+
# block is stored in the repo, no pin is created, no provider record
|
|
1380
|
+
# is announced, no peer ever learns the CID exists.
|
|
1381
|
+
return self._add(
|
|
1382
|
+
source, pin=False, provide=False, only_hash=True, timeout=timeout
|
|
1383
|
+
)
|
|
1384
|
+
|
|
1385
|
+
def add_folder(
|
|
1386
|
+
self,
|
|
1387
|
+
path: str | os.PathLike[str],
|
|
1388
|
+
*,
|
|
1389
|
+
timeout: float | None = None,
|
|
1390
|
+
) -> str:
|
|
1391
|
+
src = Path(os.fspath(path)).resolve()
|
|
1392
|
+
if not src.is_dir():
|
|
1393
|
+
raise NotADirectoryError(f"Not a directory: {src}")
|
|
1394
|
+
|
|
1395
|
+
# Build the parts list eagerly (cheap — just (name, type, path)
|
|
1396
|
+
# tuples) but defer file opens to _iter_multipart, which opens
|
|
1397
|
+
# each file as it streams it and closes it on EOF. This keeps
|
|
1398
|
+
# the open-FD count at 1 throughout the upload regardless of
|
|
1399
|
+
# how many files are in the tree, dodging both RAM and ulimit
|
|
1400
|
+
# exhaustion that the previous open-everything-up-front
|
|
1401
|
+
# implementation hit on large trees.
|
|
1402
|
+
parts: list[tuple[str, str, bytes | os.PathLike[str] | io.IOBase]] = []
|
|
1403
|
+
for current_dir, dirnames, filenames in os.walk(src):
|
|
1404
|
+
dirnames.sort()
|
|
1405
|
+
filenames.sort()
|
|
1406
|
+
current_path = Path(current_dir)
|
|
1407
|
+
rel = current_path.relative_to(src)
|
|
1408
|
+
dir_name = src.name if rel == Path(".") else (Path(src.name) / rel).as_posix()
|
|
1409
|
+
parts.append((dir_name, "application/x-directory", b""))
|
|
1410
|
+
for fname in filenames:
|
|
1411
|
+
rel_file = (Path(src.name) / rel / fname).as_posix()
|
|
1412
|
+
parts.append((rel_file, "application/octet-stream", current_path / fname))
|
|
1413
|
+
|
|
1414
|
+
boundary, body_iter = _iter_multipart(parts)
|
|
1415
|
+
resp = self._post(
|
|
1416
|
+
"add",
|
|
1417
|
+
params=[
|
|
1418
|
+
("recursive", "true"),
|
|
1419
|
+
("wrap-with-directory", "false"),
|
|
1420
|
+
("pin", "true"),
|
|
1421
|
+
("cid-version", "1"),
|
|
1422
|
+
("quieter", "true"),
|
|
1423
|
+
],
|
|
1424
|
+
data=body_iter,
|
|
1425
|
+
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
|
|
1426
|
+
timeout=timeout,
|
|
1427
|
+
)
|
|
1428
|
+
|
|
1429
|
+
# quieter=true emits only the root entry — but parse defensively in
|
|
1430
|
+
# case Kubo decides to be chatty in a future version.
|
|
1431
|
+
lines = [ln for ln in resp.text.strip().splitlines() if ln.strip()]
|
|
1432
|
+
if not lines:
|
|
1433
|
+
raise IPFSOperationError("ipfs add returned no output")
|
|
1434
|
+
cid: str = json.loads(lines[-1])["Hash"]
|
|
1435
|
+
return cid
|
|
1436
|
+
|
|
1437
|
+
def cat(
|
|
1438
|
+
self,
|
|
1439
|
+
cid: str,
|
|
1440
|
+
output_path: str | os.PathLike[str] | None = None,
|
|
1441
|
+
*,
|
|
1442
|
+
timeout: float | None = None,
|
|
1443
|
+
max_bytes: int | None = -1,
|
|
1444
|
+
) -> bytes | None:
|
|
1445
|
+
# Streaming path: writing to disk is unbounded — the caller chose to
|
|
1446
|
+
# spend disk, not RAM. The in-memory path is bounded by
|
|
1447
|
+
# ``max_bytes`` (or the module-level ``MAX_GET_BYTES`` default), so a
|
|
1448
|
+
# malicious / oversized CID can't OOM a long-running service.
|
|
1449
|
+
resp = self._post("cat", params={"arg": cid}, stream=True, timeout=timeout)
|
|
1450
|
+
if output_path is not None:
|
|
1451
|
+
with open(os.fspath(output_path), "wb") as fh:
|
|
1452
|
+
for chunk in resp.iter_content(chunk_size=1 << 16):
|
|
1453
|
+
fh.write(chunk)
|
|
1454
|
+
return None
|
|
1455
|
+
# Sentinel ``-1`` means "use the module default"; explicit ``None``
|
|
1456
|
+
# means "no cap" so callers who really want unbounded bytes can opt
|
|
1457
|
+
# out without monkey-patching the module global.
|
|
1458
|
+
cap = MAX_GET_BYTES if max_bytes == -1 else max_bytes
|
|
1459
|
+
buf = bytearray()
|
|
1460
|
+
for chunk in resp.iter_content(chunk_size=1 << 16):
|
|
1461
|
+
if not chunk:
|
|
1462
|
+
continue
|
|
1463
|
+
buf.extend(chunk)
|
|
1464
|
+
if cap is not None and len(buf) > cap:
|
|
1465
|
+
# Abort the stream rather than continue reading; close the
|
|
1466
|
+
# connection so we don't keep the daemon writing.
|
|
1467
|
+
resp.close()
|
|
1468
|
+
raise IPFSOperationError(
|
|
1469
|
+
f"Refusing to load CID {cid} into memory: exceeds "
|
|
1470
|
+
f"{cap} bytes (already received {len(buf)}). "
|
|
1471
|
+
f"Pass an output_path to stream to disk, or set "
|
|
1472
|
+
f"basic_ipfs.MAX_GET_BYTES = None / a larger value."
|
|
1473
|
+
)
|
|
1474
|
+
return bytes(buf)
|
|
1475
|
+
|
|
1476
|
+
def pin_add(
|
|
1477
|
+
self,
|
|
1478
|
+
cids: str | Iterable[str],
|
|
1479
|
+
*,
|
|
1480
|
+
timeout: float | None = None,
|
|
1481
|
+
) -> None:
|
|
1482
|
+
cid_list = _as_str_list(cids)
|
|
1483
|
+
for chunk in _chunked(cid_list, _PIN_BATCH_SIZE):
|
|
1484
|
+
params = [("arg", c) for c in chunk]
|
|
1485
|
+
params.append(("recursive", "true"))
|
|
1486
|
+
self._post("pin/add", params=params, timeout=timeout)
|
|
1487
|
+
|
|
1488
|
+
def pin_rm(
|
|
1489
|
+
self,
|
|
1490
|
+
cids: str | Iterable[str],
|
|
1491
|
+
*,
|
|
1492
|
+
timeout: float | None = 60,
|
|
1493
|
+
) -> None:
|
|
1494
|
+
cid_list = _as_str_list(cids)
|
|
1495
|
+
succeeded: list[str] = []
|
|
1496
|
+
failed: list[tuple[str, str]] = []
|
|
1497
|
+
for chunk in _chunked(cid_list, _PIN_BATCH_SIZE):
|
|
1498
|
+
try:
|
|
1499
|
+
params = [("arg", c) for c in chunk]
|
|
1500
|
+
params.append(("recursive", "true"))
|
|
1501
|
+
self._post("pin/rm", params=params, timeout=timeout)
|
|
1502
|
+
succeeded.extend(chunk)
|
|
1503
|
+
except IPFSOperationError as exc:
|
|
1504
|
+
if not exc.is_not_pinned():
|
|
1505
|
+
# Hard failure on the batch — record what was processed
|
|
1506
|
+
# so the caller can retry the rest, then re-raise with
|
|
1507
|
+
# the partial-success context attached. The whole
|
|
1508
|
+
# current chunk is the "failed" set since Kubo aborts
|
|
1509
|
+
# an erroring batch atomically.
|
|
1510
|
+
wrapped = IPFSOperationError(
|
|
1511
|
+
f"pin/rm failed mid-batch after {len(succeeded)} "
|
|
1512
|
+
f"successful unpins: {exc}",
|
|
1513
|
+
status_code=exc.status_code,
|
|
1514
|
+
kubo_code=exc.kubo_code,
|
|
1515
|
+
kubo_type=exc.kubo_type,
|
|
1516
|
+
kubo_message=exc.kubo_message,
|
|
1517
|
+
)
|
|
1518
|
+
wrapped.succeeded_cids = succeeded
|
|
1519
|
+
wrapped.failed_cids = list(chunk)
|
|
1520
|
+
raise wrapped from exc
|
|
1521
|
+
# One or more CIDs in the batch weren't pinned. Kubo aborts
|
|
1522
|
+
# the whole batch in that case, so fall back to per-CID
|
|
1523
|
+
# removal to preserve idempotency. Track per-CID outcomes
|
|
1524
|
+
# so a hard failure mid-fallback surfaces the partial state
|
|
1525
|
+
# to the caller instead of silently leaving half the chunk
|
|
1526
|
+
# unpinned.
|
|
1527
|
+
for c in chunk:
|
|
1528
|
+
try:
|
|
1529
|
+
self._post(
|
|
1530
|
+
"pin/rm",
|
|
1531
|
+
params={"arg": c, "recursive": "true"},
|
|
1532
|
+
timeout=timeout,
|
|
1533
|
+
)
|
|
1534
|
+
succeeded.append(c)
|
|
1535
|
+
except IPFSOperationError as per_exc:
|
|
1536
|
+
if per_exc.is_not_pinned():
|
|
1537
|
+
succeeded.append(c) # idempotent no-op counts as success
|
|
1538
|
+
continue
|
|
1539
|
+
failed.append((c, str(per_exc)))
|
|
1540
|
+
if failed:
|
|
1541
|
+
err = IPFSOperationError(
|
|
1542
|
+
f"unpin partially succeeded: {len(succeeded)} ok, "
|
|
1543
|
+
f"{len(failed)} failed; first failure: {failed[0][1]}"
|
|
1544
|
+
)
|
|
1545
|
+
err.failed_cids = [c for c, _ in failed]
|
|
1546
|
+
err.succeeded_cids = succeeded
|
|
1547
|
+
raise err
|
|
1548
|
+
|
|
1549
|
+
def pin_ls(self) -> list[str]:
|
|
1550
|
+
resp = self._post("pin/ls", params={"type": "recursive"}, timeout=10).json()
|
|
1551
|
+
return list(resp.get("Keys", {}).keys())
|
|
1552
|
+
|
|
1553
|
+
def pin_check(self, cid: str) -> bool:
|
|
1554
|
+
# Use type=recursive (matches pin_ls). Indirect pins (folder children)
|
|
1555
|
+
# are intentionally NOT counted — the only pins basic_ipfs creates are
|
|
1556
|
+
# recursive ones via add(), so this is the consistent answer.
|
|
1557
|
+
try:
|
|
1558
|
+
self._post("pin/ls", params={"arg": cid, "type": "recursive"}, timeout=10)
|
|
1559
|
+
return True
|
|
1560
|
+
except IPFSOperationError as exc:
|
|
1561
|
+
if exc.is_not_pinned():
|
|
1562
|
+
return False
|
|
1563
|
+
raise
|
|
1564
|
+
|
|
1565
|
+
def block_exists(self, cid: str) -> bool:
|
|
1566
|
+
# offline=true is critical: without it, Kubo will try to fetch the
|
|
1567
|
+
# block from the swarm if it isn't local, which can hang for the
|
|
1568
|
+
# full timeout and contradicts what exists() promises.
|
|
1569
|
+
try:
|
|
1570
|
+
self._post(
|
|
1571
|
+
"block/stat",
|
|
1572
|
+
params={"arg": cid, "offline": "true"},
|
|
1573
|
+
timeout=5,
|
|
1574
|
+
)
|
|
1575
|
+
return True
|
|
1576
|
+
except (IPFSOperationError, requests.exceptions.Timeout):
|
|
1577
|
+
return False
|
|
1578
|
+
|
|
1579
|
+
def repo_gc(self, *, timeout: float | None = None) -> None:
|
|
1580
|
+
self._post("repo/gc", timeout=timeout)
|
|
1581
|
+
|
|
1582
|
+
def swarm_peers(self) -> list[str]:
|
|
1583
|
+
resp = self._post("swarm/peers", timeout=10).json()
|
|
1584
|
+
result = []
|
|
1585
|
+
for p in resp.get("Peers", []) or []:
|
|
1586
|
+
addr = p.get("Addr", "")
|
|
1587
|
+
peer = p.get("Peer", "")
|
|
1588
|
+
if peer and "/p2p/" not in addr:
|
|
1589
|
+
addr = f"{addr}/p2p/{peer}"
|
|
1590
|
+
if addr:
|
|
1591
|
+
result.append(addr)
|
|
1592
|
+
return result
|
|
1593
|
+
|
|
1594
|
+
def swarm_connect(self, multiaddrs: str | Iterable[str]) -> None:
|
|
1595
|
+
for addr in _as_str_list(multiaddrs):
|
|
1596
|
+
self._post("swarm/connect", params={"arg": addr}, timeout=30)
|
|
1597
|
+
|
|
1598
|
+
def my_addrs(self) -> list[str]:
|
|
1599
|
+
resp = self._post("id", timeout=5).json()
|
|
1600
|
+
return resp.get("Addresses", []) or []
|
|
1601
|
+
|
|
1602
|
+
def status(self) -> StatusDict:
|
|
1603
|
+
ver = self._post("version", timeout=5).json()
|
|
1604
|
+
id_info = self._post("id", timeout=5).json()
|
|
1605
|
+
repo_stat = self._post("repo/stat", timeout=10).json()
|
|
1606
|
+
pins = self._post("pin/ls", params={"type": "recursive"}, timeout=10).json()
|
|
1607
|
+
return {
|
|
1608
|
+
"peer_id": id_info.get("ID"),
|
|
1609
|
+
"agent_version": ver.get("Version"),
|
|
1610
|
+
"repo_size_bytes": repo_stat.get("RepoSize"),
|
|
1611
|
+
"num_objects": repo_stat.get("NumObjects"),
|
|
1612
|
+
"pinned_cids": len(pins.get("Keys", {})),
|
|
1613
|
+
"addresses": id_info.get("Addresses", []) or [],
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
|
|
1617
|
+
# ---------------------------------------------------------------------------
|
|
1618
|
+
# Module-level singleton — lazy-started on first use
|
|
1619
|
+
# ---------------------------------------------------------------------------
|
|
1620
|
+
|
|
1621
|
+
_manager: IPFSManager | None = None
|
|
1622
|
+
_start_lock = threading.Lock()
|
|
1623
|
+
_atexit_registered = False
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
def _atexit_stop() -> None:
|
|
1627
|
+
"""Module-scoped atexit handler — always stops the *current* manager.
|
|
1628
|
+
|
|
1629
|
+
Registered exactly once per process so a long-running supervisor that
|
|
1630
|
+
cycles stop()/start() many times does not accumulate dead bound-method
|
|
1631
|
+
references on the atexit chain.
|
|
1632
|
+
"""
|
|
1633
|
+
global _manager
|
|
1634
|
+
if _manager is not None:
|
|
1635
|
+
try:
|
|
1636
|
+
_manager.stop()
|
|
1637
|
+
except Exception:
|
|
1638
|
+
pass
|
|
1639
|
+
_manager = None
|
|
1640
|
+
|
|
1641
|
+
|
|
1642
|
+
def _ensure_atexit_registered() -> None:
|
|
1643
|
+
global _atexit_registered
|
|
1644
|
+
if not _atexit_registered:
|
|
1645
|
+
atexit.register(_atexit_stop)
|
|
1646
|
+
_atexit_registered = True
|
|
1647
|
+
|
|
1648
|
+
|
|
1649
|
+
def _get_manager() -> IPFSManager:
|
|
1650
|
+
global _manager
|
|
1651
|
+
if _manager is None:
|
|
1652
|
+
with _start_lock:
|
|
1653
|
+
if _manager is None:
|
|
1654
|
+
m = IPFSManager()
|
|
1655
|
+
m.start()
|
|
1656
|
+
# Publish only after start() completes, so other threads
|
|
1657
|
+
# never observe a half-initialised manager.
|
|
1658
|
+
_manager = m
|
|
1659
|
+
return _manager
|
|
1660
|
+
|
|
1661
|
+
|
|
1662
|
+
# ---------------------------------------------------------------------------
|
|
1663
|
+
# Public API — the functions you actually call
|
|
1664
|
+
# ---------------------------------------------------------------------------
|
|
1665
|
+
|
|
1666
|
+
|
|
1667
|
+
def add(
|
|
1668
|
+
source: str | bytes | os.PathLike[str],
|
|
1669
|
+
provide: bool = True,
|
|
1670
|
+
*,
|
|
1671
|
+
timeout: float | None = None,
|
|
1672
|
+
) -> str:
|
|
1673
|
+
"""Add a file (path or bytes) to IPFS and pin it. Returns the CID.
|
|
1674
|
+
|
|
1675
|
+
Pass ``provide=False`` to skip the DHT announce — the content is still
|
|
1676
|
+
addressable by CID, but its presence on this node is not advertised to
|
|
1677
|
+
the public network. Useful for staging private data before deciding
|
|
1678
|
+
whether to publish.
|
|
1679
|
+
|
|
1680
|
+
``timeout`` (seconds) bounds the upload. Default ``None`` waits as
|
|
1681
|
+
long as the daemon needs (right for huge files).
|
|
1682
|
+
"""
|
|
1683
|
+
return _get_manager().add(source, provide=provide, timeout=timeout)
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
def announce(
|
|
1687
|
+
source: str | bytes | os.PathLike[str],
|
|
1688
|
+
provide: bool = True,
|
|
1689
|
+
*,
|
|
1690
|
+
timeout: float | None = None,
|
|
1691
|
+
) -> str:
|
|
1692
|
+
"""Add a file (path or bytes) to IPFS without pinning. Returns the CID.
|
|
1693
|
+
|
|
1694
|
+
Content is available on the network but may be garbage-collected unless
|
|
1695
|
+
pinned later. Pass ``provide=False`` to skip the DHT announce.
|
|
1696
|
+
``timeout`` (seconds) bounds the upload; default is no timeout.
|
|
1697
|
+
"""
|
|
1698
|
+
return _get_manager().announce(source, provide=provide, timeout=timeout)
|
|
1699
|
+
|
|
1700
|
+
|
|
1701
|
+
_GET_TIMEOUT_DEFAULT = object()
|
|
1702
|
+
|
|
1703
|
+
|
|
1704
|
+
@overload
|
|
1705
|
+
def get(cid: str) -> bytes: ...
|
|
1706
|
+
@overload
|
|
1707
|
+
def get(cid: str, output_path: str | os.PathLike[str]) -> None: ...
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
def get(
|
|
1711
|
+
cid: str,
|
|
1712
|
+
output_path: str | os.PathLike[str] | None = None,
|
|
1713
|
+
*,
|
|
1714
|
+
timeout: Any = _GET_TIMEOUT_DEFAULT,
|
|
1715
|
+
max_bytes: int | None = -1,
|
|
1716
|
+
) -> bytes | None:
|
|
1717
|
+
"""Retrieve content by CID.
|
|
1718
|
+
|
|
1719
|
+
Returns bytes unless ``output_path`` is given (then writes to disk and
|
|
1720
|
+
returns ``None``).
|
|
1721
|
+
|
|
1722
|
+
``timeout`` (seconds) bounds how long the daemon may spend fetching the
|
|
1723
|
+
content. The default uses ``basic_ipfs.DEFAULT_GET_TIMEOUT`` (300 s
|
|
1724
|
+
out of the box) so an unreachable CID raises rather than hanging the
|
|
1725
|
+
calling thread forever. Pass an explicit ``timeout=None`` to opt
|
|
1726
|
+
out and wait indefinitely.
|
|
1727
|
+
|
|
1728
|
+
``max_bytes`` caps the size of the in-memory return value. The default
|
|
1729
|
+
sentinel ``-1`` uses ``basic_ipfs.MAX_GET_BYTES`` (256 MB out of the
|
|
1730
|
+
box). Pass ``None`` to opt out of the cap entirely (e.g. a CLI tool
|
|
1731
|
+
that's fine letting the OS OOM-kill it). Ignored when ``output_path``
|
|
1732
|
+
is set — streaming to disk is unbounded by design.
|
|
1733
|
+
"""
|
|
1734
|
+
effective_timeout = (
|
|
1735
|
+
DEFAULT_GET_TIMEOUT if timeout is _GET_TIMEOUT_DEFAULT else timeout
|
|
1736
|
+
)
|
|
1737
|
+
return _get_manager().cat(
|
|
1738
|
+
cid, output_path, timeout=effective_timeout, max_bytes=max_bytes
|
|
1739
|
+
)
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
def pin(cids: str | Iterable[str], *, timeout: float | None = None) -> None:
|
|
1743
|
+
"""Recursively pin one or many CIDs so they are never garbage-collected.
|
|
1744
|
+
|
|
1745
|
+
Accepts a single CID string or any iterable of CID strings.
|
|
1746
|
+
``timeout`` (seconds) bounds each batch HTTP call to the daemon.
|
|
1747
|
+
"""
|
|
1748
|
+
_get_manager().pin_add(cids, timeout=timeout)
|
|
1749
|
+
|
|
1750
|
+
|
|
1751
|
+
def unpin(cids: str | Iterable[str], *, timeout: float | None = None) -> None:
|
|
1752
|
+
"""Unpin one or many CIDs. Accepts a single CID string or any iterable.
|
|
1753
|
+
|
|
1754
|
+
Idempotent — already-unpinned CIDs are silently skipped.
|
|
1755
|
+
``timeout`` (seconds) bounds each batch HTTP call to the daemon.
|
|
1756
|
+
"""
|
|
1757
|
+
_get_manager().pin_rm(cids, timeout=timeout)
|
|
1758
|
+
|
|
1759
|
+
|
|
1760
|
+
def get_all_pins() -> list[str]:
|
|
1761
|
+
"""Return a list of all currently pinned CIDs (recursive pins only)."""
|
|
1762
|
+
return _get_manager().pin_ls()
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
def is_pinned(cid: str) -> bool:
|
|
1766
|
+
"""Return True if cid has a recursive pin (matches get_all_pins())."""
|
|
1767
|
+
return _get_manager().pin_check(cid)
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def exists(cid: str) -> bool:
|
|
1771
|
+
"""Return True if cid is available in the local node."""
|
|
1772
|
+
return _get_manager().block_exists(cid)
|
|
1773
|
+
|
|
1774
|
+
|
|
1775
|
+
def garbage_collection(*, timeout: float | None = None) -> None:
|
|
1776
|
+
"""Trigger Kubo GC, freeing storage used by unpinned content.
|
|
1777
|
+
|
|
1778
|
+
``timeout`` (seconds) bounds the GC call; default is no timeout
|
|
1779
|
+
because GC over a large repo is legitimately slow.
|
|
1780
|
+
"""
|
|
1781
|
+
_get_manager().repo_gc(timeout=timeout)
|
|
1782
|
+
|
|
1783
|
+
|
|
1784
|
+
def add_folder(
|
|
1785
|
+
path: str | os.PathLike[str],
|
|
1786
|
+
*,
|
|
1787
|
+
timeout: float | None = None,
|
|
1788
|
+
) -> str:
|
|
1789
|
+
"""Add a directory to IPFS recursively and pin it. Returns the root CID.
|
|
1790
|
+
|
|
1791
|
+
``timeout`` (seconds) bounds the upload; default is no timeout.
|
|
1792
|
+
"""
|
|
1793
|
+
return _get_manager().add_folder(path, timeout=timeout)
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
def compute_cid_locally(
|
|
1797
|
+
source: str | bytes | os.PathLike[str],
|
|
1798
|
+
*,
|
|
1799
|
+
timeout: float | None = None,
|
|
1800
|
+
) -> str:
|
|
1801
|
+
"""Return the CID a file or bytes payload *would* have, without
|
|
1802
|
+
publishing anything.
|
|
1803
|
+
|
|
1804
|
+
Chunks and hashes the data using Kubo's ``add --only-hash`` mode:
|
|
1805
|
+
no block is written to the repo, no pin is created, no provider
|
|
1806
|
+
record is announced to the DHT, no peer ever learns the CID exists.
|
|
1807
|
+
Useful for previewing the CID of content you have not yet decided
|
|
1808
|
+
to publish.
|
|
1809
|
+
|
|
1810
|
+
Note: this routes through the local Kubo daemon's HTTP API, so the
|
|
1811
|
+
daemon must be running (and the binary must be installed). It does
|
|
1812
|
+
not by itself initiate any IPFS-network traffic — Kubo computes the
|
|
1813
|
+
CID locally and never advertises it. ``timeout`` (seconds) bounds
|
|
1814
|
+
the local hash computation; default is no timeout.
|
|
1815
|
+
|
|
1816
|
+
Output matches what ``add()`` / ``announce()`` would return for the
|
|
1817
|
+
same bytes (CIDv1, default chunker).
|
|
1818
|
+
"""
|
|
1819
|
+
return _get_manager().compute_cid_locally(source, timeout=timeout)
|
|
1820
|
+
|
|
1821
|
+
|
|
1822
|
+
def peers() -> list[str]:
|
|
1823
|
+
"""Return the full multiaddrs of all currently connected peers."""
|
|
1824
|
+
return _get_manager().swarm_peers()
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def connect_to_node(multiaddress: str) -> None:
|
|
1828
|
+
"""Connect to a peer by multiaddr (e.g. /ip4/1.2.3.4/tcp/4001/p2p/12D3KooW...)."""
|
|
1829
|
+
_get_manager().swarm_connect(multiaddress)
|
|
1830
|
+
|
|
1831
|
+
|
|
1832
|
+
def connect_to_nodes(multiaddresses: Iterable[str]) -> None:
|
|
1833
|
+
"""Connect to multiple peers. Errors raise on the first failed address."""
|
|
1834
|
+
_get_manager().swarm_connect(multiaddresses)
|
|
1835
|
+
|
|
1836
|
+
|
|
1837
|
+
def my_node_multiaddress() -> tuple[str | None, str | None]:
|
|
1838
|
+
"""Return (ipv4_multiaddr, ipv6_multiaddr) for this node.
|
|
1839
|
+
|
|
1840
|
+
Filters out loopback and link-local addresses and prefers public IPs.
|
|
1841
|
+
Either value may be None if that address family isn't available.
|
|
1842
|
+
Both strings are in the exact format connect_to_node() accepts.
|
|
1843
|
+
"""
|
|
1844
|
+
all_addrs = _get_manager().my_addrs()
|
|
1845
|
+
scored = [(a, _addr_score(a)) for a in all_addrs]
|
|
1846
|
+
usable = [(a, s) for a, s in scored if s >= 0]
|
|
1847
|
+
|
|
1848
|
+
ipv4 = next(
|
|
1849
|
+
(a for a, _ in sorted(
|
|
1850
|
+
((a, s) for a, s in usable if a.startswith("/ip4/")),
|
|
1851
|
+
key=lambda x: x[1], reverse=True,
|
|
1852
|
+
)),
|
|
1853
|
+
None,
|
|
1854
|
+
)
|
|
1855
|
+
ipv6 = next(
|
|
1856
|
+
(a for a, _ in sorted(
|
|
1857
|
+
((a, s) for a, s in usable if a.startswith("/ip6/")),
|
|
1858
|
+
key=lambda x: x[1], reverse=True,
|
|
1859
|
+
)),
|
|
1860
|
+
None,
|
|
1861
|
+
)
|
|
1862
|
+
return ipv4, ipv6
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
def status() -> StatusDict:
|
|
1866
|
+
"""Return peer ID, repo size, pinned count, etc."""
|
|
1867
|
+
return _get_manager().status()
|
|
1868
|
+
|
|
1869
|
+
|
|
1870
|
+
def start() -> None:
|
|
1871
|
+
"""Explicitly start the daemon. Also happens lazily on the first call."""
|
|
1872
|
+
_get_manager()
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def stop() -> None:
|
|
1876
|
+
"""Explicitly stop the daemon. Also happens automatically on process exit."""
|
|
1877
|
+
global _manager
|
|
1878
|
+
with _start_lock:
|
|
1879
|
+
if _manager is not None:
|
|
1880
|
+
_manager.stop()
|
|
1881
|
+
_manager = None
|
|
1882
|
+
|
|
1883
|
+
|
|
1884
|
+
class Node:
|
|
1885
|
+
"""Context manager: start on enter, stop on exit."""
|
|
1886
|
+
|
|
1887
|
+
def __enter__(self) -> Node:
|
|
1888
|
+
start()
|
|
1889
|
+
return self
|
|
1890
|
+
|
|
1891
|
+
def __exit__(self, *_: Any) -> None:
|
|
1892
|
+
stop()
|
|
1893
|
+
|
|
1894
|
+
|
|
1895
|
+
# Lowercase alias kept permanently for the documented `with basic_ipfs.node():` form.
|
|
1896
|
+
node = Node
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
# ---------------------------------------------------------------------------
|
|
1900
|
+
# Private network support
|
|
1901
|
+
#
|
|
1902
|
+
# By default every IPFS node joins the *public* IPFS network: anyone can
|
|
1903
|
+
# connect to you, and your CIDs are discoverable by the whole world.
|
|
1904
|
+
#
|
|
1905
|
+
# A private network changes this completely. Every node in your group shares
|
|
1906
|
+
# a secret 256-bit key stored in a file called `swarm.key` inside the IPFS
|
|
1907
|
+
# repo. When Kubo starts with that file present it enters isolation mode:
|
|
1908
|
+
#
|
|
1909
|
+
# - It will ONLY complete the libp2p handshake with peers that present the
|
|
1910
|
+
# same key. Peers without it are silently dropped before any data is
|
|
1911
|
+
# exchanged — they can't even tell a node is there.
|
|
1912
|
+
# - It does NOT join the public DHT. Your group maintains its own private
|
|
1913
|
+
# routing table. The outside world cannot discover your peers or CIDs.
|
|
1914
|
+
# - XSalsa20 stream encryption is applied on top of every connection using
|
|
1915
|
+
# the shared key, so traffic between members is also encrypted in transit.
|
|
1916
|
+
#
|
|
1917
|
+
# The practical result: connect_to_nodes() + a shared swarm key gives you a
|
|
1918
|
+
# completely isolated peer-to-peer network. Only nodes you give the key to
|
|
1919
|
+
# can participate — this is your private DHT.
|
|
1920
|
+
#
|
|
1921
|
+
# ── How to set up a private network ─────────────────────────────────────────
|
|
1922
|
+
#
|
|
1923
|
+
# One node (the "founder") runs this BEFORE starting the daemon:
|
|
1924
|
+
#
|
|
1925
|
+
# import basic_ipfs
|
|
1926
|
+
# key = basic_ipfs.create_private_network() # generates key, writes swarm.key
|
|
1927
|
+
# basic_ipfs.start() # daemon starts in isolation mode
|
|
1928
|
+
# ipv4, ipv6 = basic_ipfs.my_node_multiaddress()
|
|
1929
|
+
# print(key) # share this secret string with trusted nodes
|
|
1930
|
+
# print(ipv4) # share your address so they can connect to you
|
|
1931
|
+
#
|
|
1932
|
+
# Every other node runs this BEFORE starting their daemon:
|
|
1933
|
+
#
|
|
1934
|
+
# import basic_ipfs
|
|
1935
|
+
# basic_ipfs.join_private_network("<key from founder>")
|
|
1936
|
+
# basic_ipfs.start()
|
|
1937
|
+
# basic_ipfs.connect_to_node("<founder's ipv4 or ipv6 multiaddr>")
|
|
1938
|
+
# # now fully peered — content flows directly between members
|
|
1939
|
+
#
|
|
1940
|
+
# IMPORTANT: swarm.key must exist before `ipfs daemon` launches. These
|
|
1941
|
+
# functions raise IPFSError if called while the daemon is already running.
|
|
1942
|
+
# ---------------------------------------------------------------------------
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def _swarm_key_path() -> Path:
|
|
1946
|
+
# Kubo looks for swarm.key directly inside $IPFS_PATH (the repo root).
|
|
1947
|
+
return _get_repo_path() / "swarm.key"
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
def _write_swarm_key(key_hex: str) -> None:
|
|
1951
|
+
# The swarm.key file format is defined by go-libp2p-pnet. It is a
|
|
1952
|
+
# three-line text file:
|
|
1953
|
+
#
|
|
1954
|
+
# /key/swarm/psk/1.0.0/ ← protocol identifier — must be this exact string
|
|
1955
|
+
# /base16/ ← encoding of the key material — base16 = hex
|
|
1956
|
+
# <64 hex chars> ← 32 random bytes = 256-bit key
|
|
1957
|
+
#
|
|
1958
|
+
# Kubo reads this at startup and refuses connections from any peer whose
|
|
1959
|
+
# swarm.key does not contain the same 64 hex chars.
|
|
1960
|
+
path = _swarm_key_path()
|
|
1961
|
+
if os.name == "posix":
|
|
1962
|
+
_secure_mkdir(path.parent, mode=0o700)
|
|
1963
|
+
else:
|
|
1964
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1965
|
+
body = f"/key/swarm/psk/1.0.0/\n/base16/\n{key_hex}\n"
|
|
1966
|
+
if os.name == "posix":
|
|
1967
|
+
# Atomic create with mode 0o600 — closes the TOCTOU window between
|
|
1968
|
+
# write_text() and chmod() that would otherwise leak this credential
|
|
1969
|
+
# to any local user during the umask gap. swarm.key is a 256-bit
|
|
1970
|
+
# symmetric secret; a single read leaks permanent network access.
|
|
1971
|
+
_secure_write_text(path, body, mode=0o600)
|
|
1972
|
+
else:
|
|
1973
|
+
path.write_text(body)
|
|
1974
|
+
|
|
1975
|
+
|
|
1976
|
+
def _swarm_key_warn_if_world_readable() -> None:
|
|
1977
|
+
if os.name != "posix":
|
|
1978
|
+
return
|
|
1979
|
+
path = _swarm_key_path()
|
|
1980
|
+
if not path.exists():
|
|
1981
|
+
return
|
|
1982
|
+
try:
|
|
1983
|
+
mode = path.stat().st_mode & 0o777
|
|
1984
|
+
except OSError:
|
|
1985
|
+
return
|
|
1986
|
+
if mode & 0o077:
|
|
1987
|
+
logger.warning(
|
|
1988
|
+
"swarm.key at %s is readable by other users (mode 0o%o). "
|
|
1989
|
+
"Run: chmod 600 %s", path, mode, path,
|
|
1990
|
+
)
|
|
1991
|
+
|
|
1992
|
+
|
|
1993
|
+
def _daemon_is_running() -> bool:
|
|
1994
|
+
return _manager is not None and getattr(_manager, "_initialised", False)
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
def create_private_network() -> str:
|
|
1998
|
+
"""Create a new private IPFS network by generating a random swarm key.
|
|
1999
|
+
|
|
2000
|
+
Writes swarm.key to the repo so that when the daemon starts it will only
|
|
2001
|
+
accept connections from peers that share the same key.
|
|
2002
|
+
|
|
2003
|
+
Returns the key as a 64-character hex string. Share this string (securely)
|
|
2004
|
+
with every node that should join your network — they pass it to
|
|
2005
|
+
join_private_network() before starting their daemon.
|
|
2006
|
+
|
|
2007
|
+
Must be called BEFORE start() / the first IPFS operation.
|
|
2008
|
+
Safe to call multiple times — overwrites any existing swarm.key.
|
|
2009
|
+
"""
|
|
2010
|
+
if _daemon_is_running():
|
|
2011
|
+
raise IPFSError(
|
|
2012
|
+
"Cannot write swarm.key while the daemon is running. "
|
|
2013
|
+
"Call create_private_network() before start() or any IPFS operation."
|
|
2014
|
+
)
|
|
2015
|
+
# os.urandom uses the OS CSPRNG (e.g. /dev/urandom on Linux, CryptGenRandom
|
|
2016
|
+
# on Windows) — safe for generating cryptographic key material.
|
|
2017
|
+
key_hex = os.urandom(32).hex() # 32 bytes → 64 hex chars → 256-bit key
|
|
2018
|
+
_write_swarm_key(key_hex)
|
|
2019
|
+
logger.info("Private network swarm key written to %s", _swarm_key_path())
|
|
2020
|
+
return key_hex
|
|
2021
|
+
|
|
2022
|
+
|
|
2023
|
+
def join_private_network(key: str) -> None:
|
|
2024
|
+
"""Join an existing private network using a swarm key from another node.
|
|
2025
|
+
|
|
2026
|
+
key: the 64-character hex string returned by create_private_network() on
|
|
2027
|
+
the founder's node. Treat it like a password — anyone with this key
|
|
2028
|
+
can join your network.
|
|
2029
|
+
|
|
2030
|
+
Writes swarm.key to the repo so that when the daemon starts it will only
|
|
2031
|
+
accept connections from peers sharing the same key.
|
|
2032
|
+
|
|
2033
|
+
Must be called BEFORE start() / the first IPFS operation.
|
|
2034
|
+
"""
|
|
2035
|
+
if _daemon_is_running():
|
|
2036
|
+
raise IPFSError(
|
|
2037
|
+
"Cannot write swarm.key while the daemon is running. "
|
|
2038
|
+
"Call join_private_network() before start() or any IPFS operation."
|
|
2039
|
+
)
|
|
2040
|
+
key = key.strip().lower()
|
|
2041
|
+
# Validate: must be exactly 64 lowercase hex characters (32 bytes).
|
|
2042
|
+
if len(key) != 64 or not all(c in "0123456789abcdef" for c in key):
|
|
2043
|
+
raise ValueError(
|
|
2044
|
+
"key must be a 64-character hex string, as returned by "
|
|
2045
|
+
"create_private_network(). "
|
|
2046
|
+
f"Got {len(key)!r} characters."
|
|
2047
|
+
)
|
|
2048
|
+
_write_swarm_key(key)
|
|
2049
|
+
logger.info("Private network swarm key written to %s", _swarm_key_path())
|
|
2050
|
+
|
|
2051
|
+
|
|
2052
|
+
def get_private_network_key() -> str | None:
|
|
2053
|
+
"""Return this node's swarm key as a hex string, or None if not configured.
|
|
2054
|
+
|
|
2055
|
+
Use this to retrieve the key from the founder node so you can share it
|
|
2056
|
+
with new members who need to call join_private_network().
|
|
2057
|
+
"""
|
|
2058
|
+
path = _swarm_key_path()
|
|
2059
|
+
if not path.exists():
|
|
2060
|
+
return None
|
|
2061
|
+
for line in path.read_text().splitlines():
|
|
2062
|
+
line = line.strip()
|
|
2063
|
+
# The key line is the only line that doesn't start with '/'.
|
|
2064
|
+
if line and not line.startswith("/"):
|
|
2065
|
+
return line
|
|
2066
|
+
return None
|
|
2067
|
+
|
|
2068
|
+
|
|
2069
|
+
def is_private_network() -> bool:
|
|
2070
|
+
"""Return True if this node is configured to run as a private network."""
|
|
2071
|
+
return _swarm_key_path().exists()
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def rotate_identity(oldkey_name: str = "previous-self") -> str:
|
|
2075
|
+
"""Generate a new libp2p keypair, severing peer-ID linkability with prior runs.
|
|
2076
|
+
|
|
2077
|
+
The old key is preserved under ``oldkey_name`` (default
|
|
2078
|
+
``"previous-self"``) inside the keystore so it can be exported if you
|
|
2079
|
+
ever need to re-sign IPNS records published under the old identity.
|
|
2080
|
+
Pins, config, and ``swarm.key`` are untouched.
|
|
2081
|
+
|
|
2082
|
+
The daemon must NOT be running. Call before the first IPFS operation,
|
|
2083
|
+
or stop a running daemon first (Python process exit stops it cleanly).
|
|
2084
|
+
|
|
2085
|
+
Returns the new peer ID. Raises :class:`IPFSError` if the daemon is
|
|
2086
|
+
running, the repo is missing, or the underlying ``ipfs key rotate``
|
|
2087
|
+
fails.
|
|
2088
|
+
"""
|
|
2089
|
+
if _daemon_is_running():
|
|
2090
|
+
raise IPFSError(
|
|
2091
|
+
"Cannot rotate identity while the daemon is running. "
|
|
2092
|
+
"Stop it first (let the Python process exit, or call stop())."
|
|
2093
|
+
)
|
|
2094
|
+
repo = _get_repo_path()
|
|
2095
|
+
if not (repo / "config").exists():
|
|
2096
|
+
raise IPFSError(
|
|
2097
|
+
f"No IPFS repo at {repo}. rotate_identity() requires an existing "
|
|
2098
|
+
"repo — run an operation once to initialise it, then rotate."
|
|
2099
|
+
)
|
|
2100
|
+
binary = _find_or_install_kubo()
|
|
2101
|
+
env = os.environ.copy()
|
|
2102
|
+
env["IPFS_PATH"] = str(repo)
|
|
2103
|
+
try:
|
|
2104
|
+
subprocess.run(
|
|
2105
|
+
[
|
|
2106
|
+
str(binary), "key", "rotate",
|
|
2107
|
+
f"--oldkey={oldkey_name}",
|
|
2108
|
+
"--type=ed25519",
|
|
2109
|
+
],
|
|
2110
|
+
env=env, check=True, capture_output=True, text=True, timeout=30,
|
|
2111
|
+
)
|
|
2112
|
+
except subprocess.CalledProcessError as exc:
|
|
2113
|
+
raise IPFSError(
|
|
2114
|
+
f"Failed to rotate identity: {exc.stderr.strip()}"
|
|
2115
|
+
) from exc
|
|
2116
|
+
try:
|
|
2117
|
+
config = json.loads((repo / "config").read_text())
|
|
2118
|
+
new_id: str = config["Identity"]["PeerID"]
|
|
2119
|
+
except (OSError, KeyError, json.JSONDecodeError) as exc:
|
|
2120
|
+
raise IPFSError(f"Could not read new PeerID after rotation: {exc}") from exc
|
|
2121
|
+
logger.info(
|
|
2122
|
+
"Identity rotated. New peer ID: %s. Old key preserved as %r.",
|
|
2123
|
+
new_id, oldkey_name,
|
|
2124
|
+
)
|
|
2125
|
+
return new_id
|
|
2126
|
+
|
|
2127
|
+
|
|
2128
|
+
def lockdown_mode() -> None:
|
|
2129
|
+
"""Configure this node to never talk to the public IPFS network.
|
|
2130
|
+
|
|
2131
|
+
Sets, on the on-disk repo:
|
|
2132
|
+
|
|
2133
|
+
- ``Routing.Type = "none"`` — no DHT participation, no provider records.
|
|
2134
|
+
- ``Bootstrap = []`` — no auto-dial of Protocol Labs / Cloudflare nodes.
|
|
2135
|
+
- ``Swarm.Addresses`` restricted to loopback (``127.0.0.1`` + ``::1``).
|
|
2136
|
+
- ``Gateway.NoFetch = true`` — local gateway will not fetch remote CIDs.
|
|
2137
|
+
|
|
2138
|
+
Call BEFORE :func:`start` / the first IPFS operation. The repo must
|
|
2139
|
+
already exist (run any operation once to initialise it, or call
|
|
2140
|
+
:func:`create_private_network` first if you want a private network
|
|
2141
|
+
overlaid).
|
|
2142
|
+
|
|
2143
|
+
This is the right starting point for: encrypted-content stores, local
|
|
2144
|
+
caches, air-gapped pipelines, or any workflow where contacting third
|
|
2145
|
+
parties is unacceptable.
|
|
2146
|
+
"""
|
|
2147
|
+
if _daemon_is_running():
|
|
2148
|
+
raise IPFSError(
|
|
2149
|
+
"Cannot apply lockdown_mode while the daemon is running. "
|
|
2150
|
+
"Call lockdown_mode() before start() / the first IPFS operation."
|
|
2151
|
+
)
|
|
2152
|
+
repo = _get_repo_path()
|
|
2153
|
+
if not (repo / "config").exists():
|
|
2154
|
+
raise IPFSError(
|
|
2155
|
+
f"No IPFS repo at {repo}. lockdown_mode() requires an existing "
|
|
2156
|
+
"repo — run an operation once to initialise it, or call "
|
|
2157
|
+
"create_private_network() first."
|
|
2158
|
+
)
|
|
2159
|
+
binary = _find_or_install_kubo()
|
|
2160
|
+
env = os.environ.copy()
|
|
2161
|
+
env["IPFS_PATH"] = str(repo)
|
|
2162
|
+
|
|
2163
|
+
settings: list[tuple[list[str], str]] = [
|
|
2164
|
+
(["config", "Routing.Type", "none"], "Routing.Type"),
|
|
2165
|
+
(["config", "--json", "Bootstrap", "[]"], "Bootstrap"),
|
|
2166
|
+
(
|
|
2167
|
+
[
|
|
2168
|
+
"config", "--json", "Addresses.Swarm",
|
|
2169
|
+
json.dumps(["/ip4/127.0.0.1/tcp/4001", "/ip6/::1/tcp/4001"]),
|
|
2170
|
+
],
|
|
2171
|
+
"Addresses.Swarm",
|
|
2172
|
+
),
|
|
2173
|
+
(["config", "--json", "Gateway.NoFetch", "true"], "Gateway.NoFetch"),
|
|
2174
|
+
]
|
|
2175
|
+
for argv, label in settings:
|
|
2176
|
+
try:
|
|
2177
|
+
subprocess.run(
|
|
2178
|
+
[str(binary), *argv],
|
|
2179
|
+
env=env, check=True, capture_output=True, text=True, timeout=15,
|
|
2180
|
+
)
|
|
2181
|
+
except subprocess.CalledProcessError as exc:
|
|
2182
|
+
raise IPFSError(
|
|
2183
|
+
f"Failed to set {label} during lockdown_mode: {exc.stderr.strip()}"
|
|
2184
|
+
) from exc
|
|
2185
|
+
logger.info("lockdown_mode applied — node will not talk to public IPFS.")
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
# ---------------------------------------------------------------------------
|
|
2189
|
+
# Long-name aliases — so code written against OpenMarket-Desktop's
|
|
2190
|
+
# ipfs_manager.py keeps working after `pip install basic-ipfs`
|
|
2191
|
+
# ---------------------------------------------------------------------------
|
|
2192
|
+
|
|
2193
|
+
add_to_ipfs = add
|
|
2194
|
+
announce_to_ipfs = announce
|
|
2195
|
+
get_from_ipfs = get
|
|
2196
|
+
ipfs_pin = pin
|
|
2197
|
+
ipfs_stop_pinning = unpin
|
|
2198
|
+
get_pinned_cids = get_all_pins
|
|
2199
|
+
start_ipfs_node = start
|
|
2200
|
+
stop_ipfs_node = stop
|
|
2201
|
+
get_ipfs_status = status
|
|
2202
|
+
|
|
2203
|
+
__all__ = [
|
|
2204
|
+
# short API
|
|
2205
|
+
"add", "announce", "add_folder", "compute_cid_locally",
|
|
2206
|
+
"get", "pin", "unpin", "get_all_pins", "is_pinned",
|
|
2207
|
+
"exists", "garbage_collection",
|
|
2208
|
+
"peers", "connect_to_node", "connect_to_nodes", "my_node_multiaddress",
|
|
2209
|
+
"create_private_network", "join_private_network",
|
|
2210
|
+
"get_private_network_key", "is_private_network",
|
|
2211
|
+
"rotate_identity", "lockdown_mode",
|
|
2212
|
+
"status", "start", "stop", "Node", "node",
|
|
2213
|
+
# long aliases
|
|
2214
|
+
"add_to_ipfs", "announce_to_ipfs", "get_from_ipfs", "ipfs_pin", "ipfs_stop_pinning",
|
|
2215
|
+
"get_pinned_cids", "start_ipfs_node", "stop_ipfs_node", "get_ipfs_status",
|
|
2216
|
+
# exceptions
|
|
2217
|
+
"IPFSError", "IPFSBinaryNotFound", "IPFSDaemonTimeout", "IPFSOperationError",
|
|
2218
|
+
"IPFSPortInUse", "IPFSRepoLocked", "IPFSRepoCorrupt",
|
|
2219
|
+
# types
|
|
2220
|
+
"StatusDict",
|
|
2221
|
+
# power-user internals
|
|
2222
|
+
"IPFSManager",
|
|
2223
|
+
]
|
|
2224
|
+
|
|
2225
|
+
# Annotations are lazy (PEP 563), so these names exist only for type checkers
|
|
2226
|
+
# and don't need to live in the module namespace at runtime. Hide them so
|
|
2227
|
+
# `basic_ipfs.<TAB>` doesn't surface them. (Path / TypedDict / overload are
|
|
2228
|
+
# kept — they're used at runtime.)
|
|
2229
|
+
del Any, Iterable, Union
|