basic-ipfs 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
basic_ipfs/__init__.py ADDED
@@ -0,0 +1,2229 @@
1
+ """
2
+ basic_ipfs — stupid-simple IPFS for Python.
3
+
4
+ Runs a real Kubo node under the hood. Auto-downloads the binary on first use.
5
+
6
+ Quick start:
7
+
8
+ import basic_ipfs
9
+
10
+ cid = basic_ipfs.add("photo.jpg") # or bytes — adds AND pins
11
+ cid = basic_ipfs.announce("photo.jpg") # adds without pinning (may be GC'd)
12
+ data = basic_ipfs.get(cid) # → bytes
13
+ basic_ipfs.get(cid, "copy.jpg") # or write to disk
14
+
15
+ basic_ipfs.pin(cid) # keep forever
16
+ basic_ipfs.unpin(cid) # let GC reclaim
17
+
18
+ The daemon starts lazily on the first call and stops cleanly on process exit.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import atexit
24
+ import datetime
25
+ import glob as _glob
26
+ import hashlib
27
+ import hmac
28
+ import io
29
+ import ipaddress
30
+ import json
31
+ import logging
32
+ import os
33
+ import platform
34
+ import re
35
+ import shutil
36
+ import signal
37
+ import socket
38
+ import stat
39
+ import subprocess
40
+ import sys
41
+ import tarfile
42
+ import threading
43
+ import time
44
+ import zipfile
45
+ from collections.abc import Iterable
46
+ from pathlib import Path
47
+ from typing import Any, TypedDict, Union, overload
48
+ from urllib.parse import urlparse
49
+
50
+ import requests
51
+ from platformdirs import user_data_dir
52
+
53
+ from . import kubo_checksums
54
+
55
+ # Single source of truth: the version that pip resolved when installing this
56
+ # wheel. Avoids drift between pyproject.toml and a hardcoded literal — if
57
+ # __version__ ever lies about which fix is deployed, security advisories
58
+ # become very hard to reason about.
59
+ try:
60
+ from importlib.metadata import PackageNotFoundError
61
+ from importlib.metadata import version as _pkg_version
62
+
63
+ __version__ = _pkg_version("basic-ipfs")
64
+ except PackageNotFoundError:
65
+ # Running from a source checkout that was never installed (e.g. tests
66
+ # invoked with `pytest` against the working tree). Stay obviously fake.
67
+ __version__ = "0+unknown"
68
+
69
+ logger = logging.getLogger(__name__)
70
+
71
+ # ---------------------------------------------------------------------------
72
+ # Configuration — assign before the first API call if you want to override
73
+ # ---------------------------------------------------------------------------
74
+
75
+ APP_NAME = "basic_ipfs"
76
+
77
+ # Pinned Kubo version — downloaded automatically if not already present.
78
+ KUBO_VERSION = "v0.40.1"
79
+ KUBO_DIST_BASE = "https://dist.ipfs.tech/kubo"
80
+
81
+ # Hostnames the download is permitted to redirect to. TLS still catches a
82
+ # MitM, but pinning the origin means a 30x to attacker.example also fails
83
+ # instead of relying on the SHA-512 check alone.
84
+ _ALLOWED_DOWNLOAD_HOSTS = ("dist.ipfs.tech",)
85
+
86
+ # Where the IPFS repo lives. None = platformdirs.user_data_dir(APP_NAME)
87
+ REPO_PATH: Path | None = None
88
+
89
+ # Local API the daemon listens on
90
+ API_HOST = "127.0.0.1"
91
+ API_PORT = 5001
92
+
93
+ # Local HTTP gateway address. Kubo defaults to 127.0.0.1:8080 — only
94
+ # override if you need to run a second basic_ipfs node on the same host.
95
+ GATEWAY_HOST = "127.0.0.1"
96
+ GATEWAY_PORT = 8080
97
+
98
+ # Optional override for ``Addresses.Swarm``. When ``None``, Kubo's defaults
99
+ # apply (listens on 0.0.0.0:4001 etc.). Set to a list of multiaddrs to pin
100
+ # the swarm listener — e.g. ``["/ip4/127.0.0.1/tcp/4001"]`` for
101
+ # loopback-only on a multi-tenant host. Applied before daemon launch.
102
+ SWARM_ADDRESSES: list[str] | None = None
103
+
104
+ # Repo size cap. Empty string = unlimited.
105
+ STORAGE_MAX = "50GB"
106
+
107
+ # Seconds to wait for the daemon to come up on startup
108
+ DAEMON_STARTUP_TIMEOUT = 60
109
+
110
+ # Free-space buffer required before auto-download (~115 MB archive + extracted).
111
+ _INSTALL_FREE_BYTES = 300 * 1024 * 1024
112
+
113
+ # How many CIDs to send per batch pin/unpin HTTP call. Kubo's URL is the limit;
114
+ # 500 × ~60-byte CIDs ≈ 30 KB — comfortable headroom under typical 8 KB+ limits.
115
+ _PIN_BATCH_SIZE = 500
116
+
117
+ # Kubo daemon log: kept inside the repo for easy bug reports.
118
+ _DAEMON_LOG_NAME = "basic_ipfs_daemon.log"
119
+ _DAEMON_LOG_MAX_BYTES = 5 * 1024 * 1024 # rotate when the previous log exceeds this
120
+
121
+ # ---------------------------------------------------------------------------
122
+ # Exceptions
123
+ # ---------------------------------------------------------------------------
124
+
125
+
126
+ class IPFSError(RuntimeError):
127
+ """Base class for everything this package raises."""
128
+
129
+
130
+ class IPFSBinaryNotFound(IPFSError):
131
+ """Kubo binary missing and auto-download failed."""
132
+
133
+
134
+ class IPFSDaemonTimeout(IPFSError):
135
+ """Daemon did not become ready in time."""
136
+
137
+
138
+ class IPFSOperationError(IPFSError):
139
+ """An IPFS API call returned an error.
140
+
141
+ Attributes mirror Kubo's JSON error envelope so callers can branch on
142
+ the structured fields instead of pattern-matching the message string.
143
+ Any of them may be ``None`` if the response was not a well-formed Kubo
144
+ error body (e.g. an HTML 502 from a reverse proxy).
145
+ """
146
+
147
+ def __init__(
148
+ self,
149
+ message: str,
150
+ *,
151
+ status_code: int | None = None,
152
+ kubo_code: int | None = None,
153
+ kubo_type: str | None = None,
154
+ kubo_message: str | None = None,
155
+ ) -> None:
156
+ super().__init__(message)
157
+ self.status_code = status_code
158
+ self.kubo_code = kubo_code
159
+ self.kubo_type = kubo_type
160
+ self.kubo_message = kubo_message
161
+ # Populated by pin/unpin batch fallbacks when a partial failure
162
+ # leaves some CIDs processed and others not. Callers can branch on
163
+ # ``isinstance(exc, IPFSOperationError) and exc.failed_cids`` to
164
+ # decide what to retry.
165
+ self.succeeded_cids: list[str] = []
166
+ self.failed_cids: list[str] = []
167
+
168
+ def is_not_pinned(self) -> bool:
169
+ """True iff this error is Kubo's ``<cid> is not pinned`` response.
170
+
171
+ Used to make pin-removal idempotent without swallowing real failures.
172
+ Requires both a Kubo-shaped error body (``Type == "error"``) and the
173
+ canonical ``not pinned`` substring — guards against accidentally
174
+ treating an unrelated 500 as a successful no-op.
175
+ """
176
+ if self.kubo_type != "error":
177
+ return False
178
+ msg = (self.kubo_message or "").lower()
179
+ return "not pinned" in msg
180
+
181
+
182
+ class IPFSPortInUse(IPFSError):
183
+ """The configured API port is already bound by something other than our daemon."""
184
+
185
+
186
+ class IPFSRepoLocked(IPFSError):
187
+ """The IPFS repo is locked by another running Kubo daemon."""
188
+
189
+
190
+ class IPFSRepoCorrupt(IPFSError):
191
+ """The IPFS repo on disk failed a sanity check."""
192
+
193
+
194
+ # ---------------------------------------------------------------------------
195
+ # TypedDicts
196
+ # ---------------------------------------------------------------------------
197
+
198
+
199
+ class StatusDict(TypedDict):
200
+ peer_id: str | None
201
+ agent_version: str | None
202
+ repo_size_bytes: int | None
203
+ num_objects: int | None
204
+ pinned_cids: int
205
+ addresses: list[str]
206
+
207
+
208
+ # ---------------------------------------------------------------------------
209
+ # Platform detection
210
+ # ---------------------------------------------------------------------------
211
+
212
+
213
+ def _base_dir() -> Path:
214
+ """Package directory — respects PyInstaller one-file bundles."""
215
+ if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
216
+ return Path(sys._MEIPASS)
217
+ return Path(__file__).resolve().parent
218
+
219
+
220
+ def _is_musl() -> bool:
221
+ """True on musl-libc Linux (e.g. Alpine). Kubo ships glibc only."""
222
+ if sys.platform != "linux":
223
+ return False
224
+ return bool(_glob.glob("/lib/ld-musl-*")) or bool(_glob.glob("/lib64/ld-musl-*"))
225
+
226
+
227
+ _SUPPORTED_TRIPLES = (
228
+ "linux-amd64", "linux-arm64", "linux-riscv64",
229
+ "darwin-amd64", "darwin-arm64",
230
+ "windows-amd64",
231
+ )
232
+
233
+
234
+ def _platform_key() -> str:
235
+ system = platform.system().lower()
236
+ machine = platform.machine().lower()
237
+
238
+ if system == "linux":
239
+ if _is_musl():
240
+ manual = Path(user_data_dir(APP_NAME)) / "bin" / f"linux-{machine}" / "ipfs"
241
+ raise IPFSBinaryNotFound(
242
+ "Detected Alpine/musl Linux. Kubo publishes glibc binaries only.\n"
243
+ " - With `apk add gcompat` the glibc build *may* run, but it isn't officially supported.\n"
244
+ " - Otherwise, place a musl-compatible `ipfs` binary at:\n"
245
+ f" {manual}"
246
+ )
247
+ if machine in ("x86_64", "amd64"):
248
+ return "linux-amd64"
249
+ if machine in ("aarch64", "arm64"):
250
+ return "linux-arm64"
251
+ if machine in ("riscv64",):
252
+ return "linux-riscv64"
253
+ if machine in ("armv7l", "armv6l", "armhf", "arm"):
254
+ raise IPFSBinaryNotFound(
255
+ f"32-bit ARM ({machine}) is not supported by current Kubo releases. "
256
+ "Run a 64-bit OS (aarch64) on Raspberry Pi 4/5 or similar."
257
+ )
258
+ if system == "darwin":
259
+ if machine in ("arm64", "aarch64"):
260
+ return "darwin-arm64"
261
+ if machine in ("x86_64", "amd64"):
262
+ return "darwin-amd64"
263
+ if system == "windows":
264
+ if machine in ("amd64", "x86_64"):
265
+ return "windows-amd64"
266
+
267
+ manual = Path(user_data_dir(APP_NAME)) / "bin" / f"{system}-{machine}" / "ipfs"
268
+ raise IPFSBinaryNotFound(
269
+ f"Unsupported platform: {system}/{machine}.\n"
270
+ f" Supported: {', '.join(_SUPPORTED_TRIPLES)}.\n"
271
+ f" To use an unsupported platform, place an ipfs binary manually at:\n"
272
+ f" {manual}"
273
+ )
274
+
275
+
276
+ def _binary_name() -> str:
277
+ return "ipfs.exe" if sys.platform == "win32" else "ipfs"
278
+
279
+
280
+ def _bundled_binary_path() -> Path:
281
+ """Path inside the installed wheel — used for pre-placed binaries
282
+ (PyInstaller / Briefcase / air-gapped deploys)."""
283
+ return _base_dir() / "bin" / _platform_key() / _binary_name()
284
+
285
+
286
+ def _user_binary_path() -> Path:
287
+ """Per-user install path. The auto-downloader writes here so we never
288
+ mutate site-packages — site-packages is a trust boundary owned by the
289
+ package manager, not by runtime code, and on a multi-user host any
290
+ other user with write access there could swap the binary."""
291
+ return Path(user_data_dir(APP_NAME)) / "bin" / _platform_key() / _binary_name()
292
+
293
+
294
+ def _binary_path() -> Path:
295
+ """Back-compat alias. Returns the user-install path that the downloader
296
+ writes to. Lookup order is implemented in _find_or_install_kubo()."""
297
+ return _user_binary_path()
298
+
299
+
300
+ # ---------------------------------------------------------------------------
301
+ # Kubo auto-download
302
+ # ---------------------------------------------------------------------------
303
+
304
+
305
+ # Kubo version strings look like ``v0.40.1`` or ``v0.41.0-rc1``. We
306
+ # interpolate KUBO_VERSION into a download URL, so anything outside this
307
+ # shape (path traversal, query injection, accidental whitespace) is
308
+ # rejected before it can reach _download(). Defence in depth: the redirect
309
+ # host pin and the baked-in SHA-512 table also block the attack, but a
310
+ # regex gate makes the failure mode obvious and local.
311
+ _KUBO_VERSION_RE = re.compile(r"^v\d+\.\d+\.\d+(-rc\d+)?$")
312
+
313
+
314
+ def _archive_info() -> tuple[str, str]:
315
+ if not _KUBO_VERSION_RE.match(KUBO_VERSION):
316
+ raise IPFSBinaryNotFound(
317
+ f"Refusing to download with malformed KUBO_VERSION {KUBO_VERSION!r}. "
318
+ f"Expected something like 'v0.40.1' or 'v0.41.0-rc1'."
319
+ )
320
+ key = _platform_key()
321
+ os_name, arch = key.split("-", 1)
322
+ ext = "zip" if os_name == "windows" else "tar.gz"
323
+ name = f"kubo_{KUBO_VERSION}_{os_name}-{arch}.{ext}"
324
+ return f"{KUBO_DIST_BASE}/{KUBO_VERSION}/{name}", ext
325
+
326
+
327
+ def _download_session() -> requests.Session:
328
+ """A session with retry+backoff and host-pinned redirects."""
329
+ try:
330
+ from urllib3.util.retry import Retry
331
+ except ImportError: # very old urllib3
332
+ from requests.packages.urllib3.util.retry import Retry # type: ignore
333
+
334
+ s = requests.Session()
335
+ retry = Retry(
336
+ total=3,
337
+ backoff_factor=2,
338
+ status_forcelist=(502, 503, 504, 520, 522, 524),
339
+ allowed_methods=frozenset(["GET", "HEAD"]),
340
+ raise_on_status=False,
341
+ )
342
+ adapter = _PinnedRedirectAdapter(max_retries=retry)
343
+ s.mount("http://", adapter)
344
+ s.mount("https://", adapter)
345
+ return s
346
+
347
+
348
+ def _check_redirect_origin(final_url: str) -> None:
349
+ host = (urlparse(final_url).hostname or "").lower()
350
+ for allowed in _ALLOWED_DOWNLOAD_HOSTS:
351
+ if host == allowed or host.endswith("." + allowed):
352
+ return
353
+ raise IPFSBinaryNotFound(
354
+ f"Refusing to follow redirect to unexpected host {host!r}. "
355
+ f"Allowed: {', '.join(_ALLOWED_DOWNLOAD_HOSTS)}."
356
+ )
357
+
358
+
359
+ class _PinnedRedirectAdapter(requests.adapters.HTTPAdapter):
360
+ """HTTP adapter that re-checks the host pin at every redirect hop.
361
+
362
+ Plain ``allow_redirects=True`` only lets us inspect the *final* URL —
363
+ a 30x chain through ``attacker.example`` (with valid TLS) would still
364
+ leak the request headers and User-Agent before landing back on
365
+ ``dist.ipfs.tech``. Override ``send`` to walk the chain manually,
366
+ enforcing _check_redirect_origin on each hop's destination before
367
+ we follow it. The hash check is still fail-closed against bad
368
+ bytes, but this closes the request-side-channel gap.
369
+ """
370
+
371
+ def send(self, request, **kwargs): # type: ignore[override]
372
+ # Disable urllib3-level redirect following so each hop bubbles up
373
+ # here for inspection. ``requests.Session.resolve_redirects`` will
374
+ # then iterate over them, and we override its behaviour by
375
+ # rebuilding the chain ourselves.
376
+ kwargs["allow_redirects"] = False
377
+ resp = super().send(request, **kwargs)
378
+ # Walk redirects up to a sane cap (matches requests' default of 30).
379
+ seen = 0
380
+ while resp.is_redirect and seen < 30:
381
+ seen += 1
382
+ next_url = resp.headers.get("Location", "")
383
+ if not next_url:
384
+ break
385
+ # Resolve relative redirects against the previous URL.
386
+ from urllib.parse import urljoin
387
+ next_url = urljoin(resp.url, next_url)
388
+ _check_redirect_origin(next_url)
389
+ # Drop the body and follow.
390
+ try:
391
+ resp.close()
392
+ except Exception:
393
+ pass
394
+ new_req = request.copy()
395
+ new_req.url = next_url
396
+ # 303 / 302 / 301 on POST → GET, but we only do GET here.
397
+ resp = super().send(new_req, **kwargs)
398
+ return resp
399
+
400
+
401
+ def _download(url: str, dest: Path, timeout: int = 600) -> str:
402
+ """Stream ``url`` into ``dest``, returning the SHA-512 hex digest.
403
+
404
+ The whole archive used to be buffered in memory, which let a hostile
405
+ origin (or any 30x to a misconfigured mirror) waste up to a gigabyte
406
+ of RAM before the SHA mismatch ever rejected it. Stream to disk
407
+ instead, hash incrementally, and abort early if the byte count
408
+ exceeds ``_MAX_DOWNLOAD_BYTES`` — the hash check is still
409
+ fail-closed, this just bounds the resource cost of a bad response.
410
+ """
411
+ logger.info("Downloading %s", url)
412
+ last_pct_logged = -1
413
+ hasher = hashlib.sha512()
414
+ received = 0
415
+ dest.parent.mkdir(parents=True, exist_ok=True)
416
+ with _download_session() as session, session.get(url, stream=True, timeout=timeout) as r:
417
+ _check_redirect_origin(r.url)
418
+ r.raise_for_status()
419
+ total = int(r.headers.get("Content-Length", 0))
420
+ with open(dest, "wb") as fh:
421
+ for chunk in r.iter_content(chunk_size=1 << 16):
422
+ if not chunk:
423
+ continue
424
+ received += len(chunk)
425
+ if received > _MAX_DOWNLOAD_BYTES:
426
+ raise IPFSBinaryNotFound(
427
+ f"Refusing to download more than {_MAX_DOWNLOAD_BYTES} bytes "
428
+ f"from {url} — got {received} bytes and counting. "
429
+ f"Either the origin is misbehaving or KUBO_VERSION points at "
430
+ f"something unexpected."
431
+ )
432
+ hasher.update(chunk)
433
+ fh.write(chunk)
434
+ if total:
435
+ pct = received * 100 // total
436
+ if pct >= last_pct_logged + 10:
437
+ logger.info(" %3d%% (%d / %d bytes)", pct, received, total)
438
+ last_pct_logged = pct
439
+ return hasher.hexdigest().lower()
440
+
441
+
442
+ def _expected_sha512(archive_url: str) -> str:
443
+ """Look up the SHA-512 we expect for this version+platform.
444
+
445
+ Only the baked-in table is trusted. Fetching the companion ``.sha512``
446
+ from the same origin is rejected: an attacker who controls
447
+ ``dist.ipfs.tech`` can swap both files together. New Kubo versions must
448
+ have their hash committed to ``kubo_checksums.py`` before they can be
449
+ installed.
450
+ """
451
+ baked = kubo_checksums.known_checksum(KUBO_VERSION, _platform_key())
452
+ if baked is not None:
453
+ return baked.lower()
454
+ raise IPFSBinaryNotFound(
455
+ f"No baked-in SHA-512 for Kubo {KUBO_VERSION}/{_platform_key()}. "
456
+ f"Refusing to fetch a digest from the same origin as the archive — "
457
+ f"that defeats the verification. Add the hash to "
458
+ f"basic_ipfs/kubo_checksums.py and reinstall, or pin "
459
+ f"basic_ipfs.KUBO_VERSION to a version already in the table."
460
+ )
461
+
462
+
463
+ # Hardening caps for archive extraction. Real Kubo binaries are ~115 MB
464
+ # inside a tarball with a single top-level ``kubo/`` directory, so anything
465
+ # materially larger or more deeply nested is suspicious — refuse rather
466
+ # than allocate.
467
+ _MAX_ARCHIVE_MEMBER_BYTES = 512 * 1024 * 1024 # 512 MB per entry
468
+ _MAX_ARCHIVE_PATH_DEPTH = 4
469
+
470
+ # Hard ceiling on the streamed download itself — independent of the per-member
471
+ # cap above, because streaming happens before extraction. A hostile origin (or
472
+ # a Content-Length that lies) cannot cost us more than this in disk + time.
473
+ # Twice the per-member cap leaves comfortable headroom for archive overhead
474
+ # while still bounding the worst case.
475
+ _MAX_DOWNLOAD_BYTES = 2 * _MAX_ARCHIVE_MEMBER_BYTES
476
+
477
+
478
+ # Default cap when ``get(cid)`` is asked to return bytes (no output_path).
479
+ # Without this, a 4 GB CID returned via `bytes` allocates 4 GB of process
480
+ # RSS — fine for a CLI tool, lethal for a long-running service. Callers
481
+ # who really want the entire payload in memory should write to disk and
482
+ # read back, or set ``MAX_GET_BYTES`` to ``None`` to opt out.
483
+ MAX_GET_BYTES: int | None = 256 * 1024 * 1024
484
+
485
+ # Default per-request timeout for ``get(cid)``. ``None`` would mean "wait
486
+ # forever," which means an unreachable CID hangs the calling thread until
487
+ # the process is killed — the worst possible failure mode for a library
488
+ # advertised for long-running services. 300 s is high enough that a slow
489
+ # but progressing fetch succeeds and low enough that a stuck swarm fetch
490
+ # fails with a useful exception. Callers who really want unbounded waits
491
+ # can pass ``timeout=None`` per call or override this global.
492
+ DEFAULT_GET_TIMEOUT: float | None = 300.0
493
+
494
+
495
+ def _safe_member_name(name: str) -> bool:
496
+ # Defence in depth: extraction never honours the member's path (we always
497
+ # write to a fixed `dest`), but reject obviously-hostile names anyway so
498
+ # the helper matches its name and a future refactor that does honour the
499
+ # path stays safe.
500
+ if not name or name.startswith(("/", "\\")):
501
+ return False
502
+ p = Path(name)
503
+ if p.is_absolute():
504
+ return False
505
+ parts = p.parts
506
+ if not 0 < len(parts) <= _MAX_ARCHIVE_PATH_DEPTH:
507
+ return False
508
+ return all(part not in ("..",) for part in parts)
509
+
510
+
511
+ def _extract_binary(archive_path: Path, ext: str, dest: Path) -> None:
512
+ dest.parent.mkdir(parents=True, exist_ok=True)
513
+ if ext == "tar.gz":
514
+ with tarfile.open(str(archive_path), mode="r:gz") as tf:
515
+ for member in tf.getmembers():
516
+ if not member.isfile() or Path(member.name).name != "ipfs":
517
+ continue
518
+ if not _safe_member_name(member.name):
519
+ raise IPFSBinaryNotFound(
520
+ f"Refusing archive member with unsafe path: {member.name!r}"
521
+ )
522
+ if member.size > _MAX_ARCHIVE_MEMBER_BYTES:
523
+ raise IPFSBinaryNotFound(
524
+ f"Archive member {member.name!r} is {member.size} bytes — "
525
+ f"exceeds the {_MAX_ARCHIVE_MEMBER_BYTES}-byte cap."
526
+ )
527
+ src = tf.extractfile(member)
528
+ if src is not None:
529
+ with open(dest, "wb") as out:
530
+ shutil.copyfileobj(src, out, length=1 << 16)
531
+ break
532
+ else:
533
+ with zipfile.ZipFile(str(archive_path)) as zf:
534
+ for info in zf.infolist():
535
+ if Path(info.filename).name != "ipfs.exe":
536
+ continue
537
+ if not _safe_member_name(info.filename):
538
+ raise IPFSBinaryNotFound(
539
+ f"Refusing archive member with unsafe path: {info.filename!r}"
540
+ )
541
+ if info.file_size > _MAX_ARCHIVE_MEMBER_BYTES:
542
+ raise IPFSBinaryNotFound(
543
+ f"Archive member {info.filename!r} is {info.file_size} bytes — "
544
+ f"exceeds the {_MAX_ARCHIVE_MEMBER_BYTES}-byte cap."
545
+ )
546
+ with zf.open(info.filename) as src, open(dest, "wb") as out:
547
+ shutil.copyfileobj(src, out, length=1 << 16)
548
+ break
549
+
550
+ if not dest.exists():
551
+ raise IPFSBinaryNotFound(
552
+ f"Could not find ipfs binary inside Kubo archive for {_platform_key()}"
553
+ )
554
+ # Owner-execute only — site-packages is typically owned by the install
555
+ # user, and a multi-user host has its own kubo install.
556
+ dest.chmod(dest.stat().st_mode | stat.S_IEXEC)
557
+
558
+
559
+ def _check_disk_space(dest: Path) -> None:
560
+ target = dest.parent
561
+ while not target.exists() and target != target.parent:
562
+ target = target.parent
563
+ try:
564
+ free = shutil.disk_usage(target).free
565
+ except OSError:
566
+ return # best-effort
567
+ if free < _INSTALL_FREE_BYTES:
568
+ raise IPFSBinaryNotFound(
569
+ f"Not enough disk space to install Kubo: {free // (1024 * 1024)} MB free at "
570
+ f"{target}, need at least {_INSTALL_FREE_BYTES // (1024 * 1024)} MB. "
571
+ f"Free up space or set basic_ipfs.REPO_PATH to a different disk."
572
+ )
573
+
574
+
575
+ def _write_provenance(binary_path: Path, url: str, sha512_hex: str) -> None:
576
+ """Audit trail next to the binary: where it came from and how we verified it."""
577
+ prov = binary_path.parent / ".provenance.json"
578
+ data = {
579
+ "version": KUBO_VERSION,
580
+ "url": url,
581
+ "sha512": sha512_hex,
582
+ "verification": "baked-in",
583
+ "installed_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
584
+ "platform": _platform_key(),
585
+ }
586
+ try:
587
+ prov.write_text(json.dumps(data, indent=2))
588
+ except OSError:
589
+ pass # non-fatal
590
+
591
+
592
+ def _auto_download_kubo(dest: Path) -> None:
593
+ archive_url, ext = _archive_info()
594
+ _check_disk_space(dest)
595
+ expected = _expected_sha512(archive_url)
596
+
597
+ dest.parent.mkdir(parents=True, exist_ok=True)
598
+ archive_tmp = dest.parent / (dest.name + ".archive.partial")
599
+ archive_tmp.unlink(missing_ok=True)
600
+ partial = dest.parent / (dest.name + ".partial")
601
+ partial.unlink(missing_ok=True)
602
+
603
+ try:
604
+ actual = _download(archive_url, archive_tmp)
605
+ if not hmac.compare_digest(actual, expected):
606
+ raise IPFSBinaryNotFound(
607
+ f"SHA-512 mismatch for Kubo {KUBO_VERSION} ({_platform_key()}): "
608
+ f"expected {expected}, got {actual}. Refusing to install an unverified binary."
609
+ )
610
+ # Extract to a sibling .partial path, then atomic rename. Prevents a
611
+ # half-written binary on interrupted installs.
612
+ _extract_binary(archive_tmp, ext, partial)
613
+ os.replace(partial, dest)
614
+ finally:
615
+ # Always clean up the streamed archive; never leave a half-written
616
+ # .partial executable around on a verification or extraction failure.
617
+ archive_tmp.unlink(missing_ok=True)
618
+ if partial.exists() and not dest.exists():
619
+ partial.unlink(missing_ok=True)
620
+
621
+ _write_provenance(dest, archive_url, expected)
622
+ logger.info("Kubo %s installed at %s (verified via baked-in SHA-512)",
623
+ KUBO_VERSION, dest)
624
+
625
+
626
+ def _find_or_install_kubo() -> Path:
627
+ """user_data_dir bin → bundled bin (PyInstaller / pre-placed) → system PATH → auto-download.
628
+
629
+ Resolution order matters: a binary the user explicitly dropped into
630
+ user_data_dir wins so air-gapped or pre-vetted installs are honoured;
631
+ a binary inside the wheel wins next so PyInstaller / Briefcase
632
+ bundles still work; only then do we fall back to whatever ``ipfs``
633
+ is on PATH or download a fresh copy. The downloader always writes
634
+ to the user_data_dir path — never into site-packages.
635
+ """
636
+ user = _user_binary_path()
637
+ if user.exists():
638
+ try:
639
+ user.chmod(user.stat().st_mode | stat.S_IEXEC)
640
+ except OSError as exc:
641
+ logger.warning("could not chmod %s: %s", user, exc)
642
+ return user
643
+
644
+ bundled = _bundled_binary_path()
645
+ if bundled.exists():
646
+ try:
647
+ bundled.chmod(bundled.stat().st_mode | stat.S_IEXEC)
648
+ except OSError as exc:
649
+ logger.warning("could not chmod %s: %s", bundled, exc)
650
+ return bundled
651
+
652
+ on_path = shutil.which("ipfs")
653
+ if on_path:
654
+ logger.info("Using system-installed Kubo at %s", on_path)
655
+ return Path(on_path)
656
+
657
+ logger.info("Kubo binary not found locally — downloading %s…", KUBO_VERSION)
658
+ try:
659
+ _auto_download_kubo(user)
660
+ except IPFSBinaryNotFound:
661
+ raise
662
+ except (requests.RequestException, OSError) as exc:
663
+ raise IPFSBinaryNotFound(
664
+ f"Failed to auto-download Kubo {KUBO_VERSION} for {_platform_key()}: {exc}\n"
665
+ f" Check your internet connection (or HTTPS_PROXY / REQUESTS_CA_BUNDLE), "
666
+ f" or place the binary manually at:\n {user}"
667
+ ) from exc
668
+ return user
669
+
670
+
671
+ # ---------------------------------------------------------------------------
672
+ # Repo path
673
+ # ---------------------------------------------------------------------------
674
+
675
+
676
+ def _get_repo_path() -> Path:
677
+ if REPO_PATH is not None:
678
+ return Path(REPO_PATH)
679
+ return Path(user_data_dir(APP_NAME)) / "ipfs_repo"
680
+
681
+
682
+ # ---------------------------------------------------------------------------
683
+ # Small helpers
684
+ # ---------------------------------------------------------------------------
685
+
686
+
687
+ def _as_str_list(arg: str | Iterable[str]) -> list[str]:
688
+ """Normalise a single string or an iterable of strings into a plain list."""
689
+ if isinstance(arg, str):
690
+ return [arg]
691
+ if isinstance(arg, (bytes, bytearray)):
692
+ raise TypeError("argument must be str or iterable of str, not bytes")
693
+ try:
694
+ items = list(arg)
695
+ except TypeError as exc:
696
+ raise TypeError(
697
+ f"argument must be str or iterable of str, not {type(arg).__name__}"
698
+ ) from exc
699
+ for c in items:
700
+ if not isinstance(c, str):
701
+ raise TypeError(f"every element must be str, got {type(c).__name__}")
702
+ return items
703
+
704
+
705
+ def _chunked(items: list[str], size: int) -> Iterable[list[str]]:
706
+ for i in range(0, len(items), size):
707
+ yield items[i:i + size]
708
+
709
+
710
+ def _addr_score(multiaddr: str) -> int:
711
+ """Score a multiaddr for sharing with others. Higher = better. -1 = exclude."""
712
+ try:
713
+ parts = multiaddr.split("/")
714
+ proto = parts[1] if len(parts) > 1 else ""
715
+ ip_str = parts[2] if len(parts) > 2 else ""
716
+ if proto == "ip4":
717
+ v4 = ipaddress.IPv4Address(ip_str)
718
+ if v4.is_loopback:
719
+ return -1
720
+ return 2 if v4.is_global else 1 # public > private/LAN
721
+ if proto == "ip6":
722
+ v6 = ipaddress.IPv6Address(ip_str)
723
+ if v6.is_loopback or v6.is_link_local:
724
+ return -1
725
+ return 2 if v6.is_global else 1
726
+ except Exception:
727
+ pass
728
+ return -1
729
+
730
+
731
+ def _quote_multipart_filename(name: str) -> str:
732
+ # Match what requests/urllib3 send: a literal filename inside double
733
+ # quotes, with backslash and double-quote backslash-escaped per RFC
734
+ # 7578 §4.2. CR/LF in a header value would forge a header injection,
735
+ # so refuse anything that contains them outright. Forward slashes are
736
+ # preserved — Kubo's add handler treats them as path separators when
737
+ # building the directory tree, which is exactly what add_folder()
738
+ # relies on.
739
+ if "\r" in name or "\n" in name:
740
+ raise ValueError(f"refusing CR/LF in multipart filename: {name!r}")
741
+ return name.replace("\\", "\\\\").replace('"', '\\"')
742
+
743
+
744
+ # A streaming-multipart "part": (filename, content_type, body) where body
745
+ # is bytes, an os.PathLike (opened lazily and streamed), or a file-like
746
+ # object supporting .read(n). The encoder reads in 64 KB chunks, so RAM
747
+ # usage is constant regardless of total upload size.
748
+ _MULTIPART_PART = tuple
749
+
750
+
751
+ def _iter_multipart(
752
+ parts: list[tuple[str, str, bytes | os.PathLike[str] | io.IOBase]],
753
+ ) -> tuple[str, Iterable[bytes]]:
754
+ """Build a streaming multipart/form-data body.
755
+
756
+ Returns the boundary string and a generator that yields the wire body
757
+ in chunks. Suitable for ``requests.post(data=gen, headers={...})`` —
758
+ requests will pass through the iterable without buffering, so a
759
+ 100 GB upload uses ~64 KB of RSS regardless of total size. (The
760
+ legacy ``files=`` path of requests reads every part into memory
761
+ via ``urllib3.filepost.encode_multipart_formdata``, which is the
762
+ OOM hazard this replaces.)
763
+ """
764
+ # Random boundary unlikely to appear in any input. 16 hex bytes is
765
+ # ample uniqueness; the leading dashes match what RFC 7578 examples use.
766
+ boundary = "----basic_ipfs_" + os.urandom(16).hex()
767
+ boundary_bytes = boundary.encode("ascii")
768
+
769
+ def gen() -> Iterable[bytes]:
770
+ for filename, content_type, body in parts:
771
+ quoted = _quote_multipart_filename(filename)
772
+ yield b"--" + boundary_bytes + b"\r\n"
773
+ yield (
774
+ f'Content-Disposition: form-data; name="file"; filename="{quoted}"\r\n'
775
+ f"Content-Type: {content_type}\r\n\r\n"
776
+ ).encode()
777
+ if isinstance(body, (bytes, bytearray, memoryview)):
778
+ yield bytes(body)
779
+ elif hasattr(body, "read"):
780
+ while True:
781
+ chunk = body.read(1 << 16)
782
+ if not chunk:
783
+ break
784
+ yield chunk
785
+ else:
786
+ with open(os.fspath(body), "rb") as fh:
787
+ while True:
788
+ chunk = fh.read(1 << 16)
789
+ if not chunk:
790
+ break
791
+ yield chunk
792
+ yield b"\r\n"
793
+ yield b"--" + boundary_bytes + b"--\r\n"
794
+
795
+ return boundary, gen()
796
+
797
+
798
+ def _build_api_session() -> requests.Session:
799
+ """Session for the local Kubo HTTP API.
800
+
801
+ Retries connection-level failures only — never status codes, since
802
+ most Kubo POSTs (`add`, `pin/add`, `pin/rm`) have observable side
803
+ effects per attempt and a server-side 5xx mid-write means the
804
+ operation may already have succeeded in part. Connection errors
805
+ (ECONNREFUSED, broken pipe during a graceful daemon restart) are
806
+ safe to redrive; they happen mostly when the daemon is migrating
807
+ its repo or rotating logs.
808
+ """
809
+ try:
810
+ from urllib3.util.retry import Retry
811
+ except ImportError: # very old urllib3
812
+ from requests.packages.urllib3.util.retry import Retry # type: ignore
813
+
814
+ s = requests.Session()
815
+ retry = Retry(
816
+ total=2,
817
+ connect=2,
818
+ read=0,
819
+ status=0,
820
+ backoff_factor=0.25,
821
+ # `False` means "no method restriction" in urllib3 — connection
822
+ # errors happen before any request is sent, so retrying every
823
+ # method is safe. urllib3's type stub for allowed_methods is too
824
+ # narrow to accept the documented sentinel.
825
+ allowed_methods=False, # type: ignore[arg-type]
826
+ raise_on_status=False,
827
+ )
828
+ adapter = requests.adapters.HTTPAdapter(max_retries=retry)
829
+ s.mount("http://", adapter)
830
+ s.mount("https://", adapter)
831
+ return s
832
+
833
+
834
+ def _is_port_in_use(host: str, port: int) -> bool:
835
+ """True if something is accepting on host:port."""
836
+ try:
837
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
838
+ s.settimeout(1.0)
839
+ s.connect((host, port))
840
+ return True
841
+ except OSError:
842
+ return False
843
+
844
+
845
+ def _secure_open(path: Path, mode: int = 0o600) -> int:
846
+ """Open ``path`` for writing with restrictive perms applied atomically.
847
+
848
+ Closes the TOCTOU window between ``write_text`` and ``chmod`` — without
849
+ this, the file briefly exists with the process umask before its mode is
850
+ tightened, so a co-tenant can race in and read a credential. Returns an
851
+ OS-level file descriptor; caller is responsible for closing it.
852
+ """
853
+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
854
+ if hasattr(os, "O_NOFOLLOW"):
855
+ flags |= os.O_NOFOLLOW # refuse to write through a symlink swap
856
+ return os.open(str(path), flags, mode)
857
+
858
+
859
+ def _secure_write_text(path: Path, text: str, mode: int = 0o600) -> None:
860
+ """Write ``text`` to ``path`` with ``mode`` applied at create time."""
861
+ fd = _secure_open(path, mode=mode)
862
+ try:
863
+ with os.fdopen(fd, "w") as fh:
864
+ fh.write(text)
865
+ except BaseException:
866
+ # Clean up a half-written file on any failure (KeyboardInterrupt
867
+ # included) so the next run starts from a known-empty state.
868
+ try:
869
+ path.unlink()
870
+ except OSError:
871
+ pass
872
+ raise
873
+
874
+
875
+ def _secure_mkdir(path: Path, mode: int = 0o700) -> None:
876
+ """Create directories up to ``path`` with restrictive ``mode``.
877
+
878
+ Each newly-created leaf is mkdir'd with the requested mode so an
879
+ attacker can't snapshot the directory's contents during the umask
880
+ window between ``mkdir`` and ``chmod``. Existing directories are left
881
+ alone — the operator may have intentionally widened them.
882
+ """
883
+ path = Path(path)
884
+ if path.exists():
885
+ return
886
+ parent = path.parent
887
+ if parent != path and not parent.exists():
888
+ _secure_mkdir(parent, mode=mode)
889
+ try:
890
+ os.mkdir(str(path), mode)
891
+ except FileExistsError:
892
+ return
893
+ # umask can still mask bits off; force them on for the leaf.
894
+ try:
895
+ os.chmod(str(path), mode)
896
+ except OSError:
897
+ pass
898
+
899
+
900
+ # ---------------------------------------------------------------------------
901
+ # IPFSManager — process-singleton daemon lifecycle (managed via module locks)
902
+ # ---------------------------------------------------------------------------
903
+
904
+
905
+ class IPFSManager:
906
+ """Manages the Kubo daemon subprocess for the lifetime of the process.
907
+
908
+ Public surface used by the module-level functions: ``start``, ``stop``,
909
+ ``add``, ``announce``, ``add_folder``, ``cat``, ``pin_add``, ``pin_rm``,
910
+ ``pin_ls``, ``pin_check``, ``block_exists``, ``repo_gc``, ``swarm_peers``,
911
+ ``swarm_connect``, ``my_addrs``, ``status``.
912
+ """
913
+
914
+ def __init__(self) -> None:
915
+ self._binary: Path | None = None
916
+ self._repo: Path | None = None
917
+ self._process: subprocess.Popen[bytes] | None = None
918
+ self._owns_daemon = False
919
+ self._api_url = f"http://{API_HOST}:{API_PORT}/api/v0"
920
+ self._session: requests.Session | None = None
921
+ self._log_path: Path | None = None
922
+ self._log_file: io.IOBase | None = None # writable handle for subprocess stderr
923
+ self._initialised = False
924
+
925
+ # ---------------- startup ----------------
926
+
927
+ def start(self) -> None:
928
+ if self._initialised:
929
+ return
930
+
931
+ self._binary = _find_or_install_kubo()
932
+ self._repo = _get_repo_path()
933
+ self._session = _build_api_session()
934
+ # Re-read config every start so callers can change API_HOST/API_PORT
935
+ # between stop() and start().
936
+ self._api_url = f"http://{API_HOST}:{API_PORT}/api/v0"
937
+
938
+ self._ensure_repo()
939
+
940
+ # Apply config to the on-disk repo, but only if no daemon is already
941
+ # running — we don't want to edit another process's live config out
942
+ # from under it.
943
+ if not self._is_api_up():
944
+ self._configure_api_address()
945
+ self._configure_gateway_address()
946
+ self._configure_storage_limit()
947
+ self._configure_swarm_addresses()
948
+
949
+ _swarm_key_warn_if_world_readable()
950
+ self._start_daemon()
951
+ self._wait_for_api()
952
+
953
+ # atexit registration is module-scoped (see _ensure_atexit_registered
954
+ # below) so successive stop()/start() cycles don't accumulate stale
955
+ # bound-method references on the atexit chain.
956
+ _ensure_atexit_registered()
957
+
958
+ self._initialised = True
959
+ logger.info("IPFS node ready — repo: %s", self._repo)
960
+
961
+ def _env(self) -> dict[str, str]:
962
+ env = os.environ.copy()
963
+ env["IPFS_PATH"] = str(self._repo)
964
+ return env
965
+
966
+ def _run_cli(
967
+ self,
968
+ *args: str,
969
+ check: bool = True,
970
+ timeout: int | None = 30,
971
+ ) -> subprocess.CompletedProcess[str]:
972
+ return subprocess.run(
973
+ [str(self._binary), *args],
974
+ env=self._env(),
975
+ capture_output=True,
976
+ text=True,
977
+ timeout=timeout,
978
+ check=check,
979
+ )
980
+
981
+ def _ensure_repo(self) -> None:
982
+ assert self._repo is not None
983
+ # Create with restrictive mode atomically so a multi-user host can't
984
+ # peek into a freshly-created repo before chmod runs. mkdir() applies
985
+ # the mode under the process umask; on POSIX we re-apply afterwards
986
+ # to defeat a 0o077 umask, but the directory was never world-traversable
987
+ # because mkdir(mode=0o700) clamps via the umask only for *additional*
988
+ # bits — owner bits stay set. Existing dirs are left as-is (operator
989
+ # may have widened them deliberately).
990
+ if os.name == "posix":
991
+ _secure_mkdir(self._repo, mode=0o700)
992
+ else:
993
+ self._repo.mkdir(parents=True, exist_ok=True)
994
+ config_path = self._repo / "config"
995
+
996
+ if not config_path.exists():
997
+ logger.info("Initialising IPFS repo at %s", self._repo)
998
+ try:
999
+ self._run_cli("init", "--profile", "server")
1000
+ except subprocess.CalledProcessError as exc:
1001
+ raise IPFSError(
1002
+ f"Failed to initialise IPFS repo at {self._repo}: {exc.stderr.strip()}"
1003
+ ) from exc
1004
+ return
1005
+
1006
+ # Cheap corruption check: config exists and parses as JSON.
1007
+ try:
1008
+ json.loads(config_path.read_text())
1009
+ except (json.JSONDecodeError, OSError) as exc:
1010
+ raise IPFSRepoCorrupt(
1011
+ f"IPFS repo config at {config_path} is unreadable: {exc}\n"
1012
+ f" If salvageable, fix it manually. Otherwise delete\n"
1013
+ f" {self._repo}\n"
1014
+ f" and let basic_ipfs recreate it (you will lose local pins)."
1015
+ ) from exc
1016
+
1017
+ def _configure_api_address(self) -> None:
1018
+ try:
1019
+ self._run_cli(
1020
+ "config", "Addresses.API",
1021
+ f"/ip4/{API_HOST}/tcp/{API_PORT}",
1022
+ )
1023
+ except subprocess.CalledProcessError as exc:
1024
+ logger.warning("Could not set API address: %s", exc.stderr)
1025
+
1026
+ def _configure_gateway_address(self) -> None:
1027
+ try:
1028
+ self._run_cli(
1029
+ "config", "Addresses.Gateway",
1030
+ f"/ip4/{GATEWAY_HOST}/tcp/{GATEWAY_PORT}",
1031
+ )
1032
+ except subprocess.CalledProcessError as exc:
1033
+ logger.warning("Could not set Gateway address: %s", exc.stderr)
1034
+
1035
+ def _configure_storage_limit(self) -> None:
1036
+ if not STORAGE_MAX:
1037
+ return
1038
+ try:
1039
+ self._run_cli("config", "Datastore.StorageMax", STORAGE_MAX)
1040
+ except subprocess.CalledProcessError as exc:
1041
+ logger.warning("Could not set storage limit: %s", exc.stderr)
1042
+
1043
+ def _configure_swarm_addresses(self) -> None:
1044
+ if SWARM_ADDRESSES is None:
1045
+ return
1046
+ try:
1047
+ self._run_cli(
1048
+ "config", "Addresses.Swarm",
1049
+ "--json", json.dumps(list(SWARM_ADDRESSES)),
1050
+ )
1051
+ except subprocess.CalledProcessError as exc:
1052
+ logger.warning("Could not set Addresses.Swarm: %s", exc.stderr)
1053
+
1054
+ def _open_daemon_log(self) -> None:
1055
+ """Open the daemon log file, rotating if the previous one is large."""
1056
+ assert self._repo is not None
1057
+ self._log_path = self._repo / _DAEMON_LOG_NAME
1058
+ try:
1059
+ if self._log_path.exists() and self._log_path.stat().st_size > _DAEMON_LOG_MAX_BYTES:
1060
+ old = self._log_path.with_suffix(self._log_path.suffix + ".old")
1061
+ if old.exists():
1062
+ try:
1063
+ old.unlink()
1064
+ except OSError:
1065
+ pass
1066
+ self._log_path.replace(old)
1067
+ except OSError:
1068
+ pass # rotation is best-effort
1069
+ # Buffered: stderr write rates from Kubo are low, no need for unbuffered.
1070
+ # Open via os.open with mode 0o600 on POSIX so the file is never
1071
+ # world-readable, even briefly. The daemon log captures CIDs, peer
1072
+ # IDs, multiaddrs — fingerprintable activity.
1073
+ if os.name == "posix":
1074
+ flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
1075
+ fd = os.open(str(self._log_path), flags, 0o600)
1076
+ self._log_file = os.fdopen(fd, "ab")
1077
+ # If the file pre-existed with a wider mode, narrow it.
1078
+ try:
1079
+ os.chmod(self._log_path, 0o600)
1080
+ except OSError as exc:
1081
+ logger.warning("could not restrict log perms at %s: %s", self._log_path, exc)
1082
+ else:
1083
+ self._log_file = open(self._log_path, "ab")
1084
+
1085
+ def _start_daemon(self) -> None:
1086
+ if self._is_api_up():
1087
+ logger.info("IPFS API already reachable — skipping daemon launch")
1088
+ return
1089
+
1090
+ # If something is bound to the port and isn't a Kubo daemon, fail loud.
1091
+ if _is_port_in_use(API_HOST, API_PORT):
1092
+ raise IPFSPortInUse(
1093
+ f"Port {API_HOST}:{API_PORT} is in use but does not look like an IPFS daemon. "
1094
+ f"Set basic_ipfs.API_PORT to a different port, or stop whatever is using it."
1095
+ )
1096
+
1097
+ self._open_daemon_log()
1098
+
1099
+ cmd = [
1100
+ str(self._binary), "daemon",
1101
+ "--migrate=true",
1102
+ ]
1103
+ # Critical: never pass PIPE for stdout/stderr on a long-lived process;
1104
+ # the OS pipe buffer (~64 KB) fills and the daemon blocks on its next
1105
+ # write. Discard stdout, send stderr to a rotating file.
1106
+ popen_kwargs: dict[str, Any] = {
1107
+ "env": self._env(),
1108
+ "stdout": subprocess.DEVNULL,
1109
+ "stderr": self._log_file,
1110
+ }
1111
+ if sys.platform == "win32":
1112
+ popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
1113
+
1114
+ self._process = subprocess.Popen(cmd, **popen_kwargs)
1115
+ self._owns_daemon = True
1116
+ logger.info("Kubo daemon started (pid %d)", self._process.pid)
1117
+
1118
+ def _read_log_tail(self, n_bytes: int = 4096) -> str:
1119
+ if not self._log_path or not self._log_path.exists():
1120
+ return "(no daemon log yet)"
1121
+ try:
1122
+ with open(self._log_path, "rb") as fh:
1123
+ fh.seek(0, os.SEEK_END)
1124
+ size = fh.tell()
1125
+ fh.seek(max(0, size - n_bytes))
1126
+ return fh.read().decode(errors="replace")
1127
+ except OSError:
1128
+ return "(log unreadable)"
1129
+
1130
+ def _wait_for_api(self) -> None:
1131
+ deadline = time.monotonic() + DAEMON_STARTUP_TIMEOUT
1132
+ while time.monotonic() < deadline:
1133
+ if self._is_api_up():
1134
+ return
1135
+ if self._process and self._process.poll() is not None:
1136
+ tail = self._read_log_tail()
1137
+ lower = tail.lower()
1138
+ if "lock" in lower and "repo" in lower:
1139
+ raise IPFSRepoLocked(
1140
+ f"The IPFS repo at {self._repo} is locked by another process. "
1141
+ "Only one Kubo daemon can use a repo at a time. Stop the other "
1142
+ "process, or set basic_ipfs.REPO_PATH to a different path.\n"
1143
+ f"--- daemon log tail ---\n{tail}"
1144
+ )
1145
+ raise IPFSError(
1146
+ f"Kubo daemon exited unexpectedly (code {self._process.returncode}):\n{tail}"
1147
+ )
1148
+ time.sleep(0.5)
1149
+ raise IPFSDaemonTimeout(
1150
+ f"Kubo daemon did not become ready within {DAEMON_STARTUP_TIMEOUT}s. "
1151
+ f"See log: {self._log_path}"
1152
+ )
1153
+
1154
+ # ---------------- shutdown ----------------
1155
+
1156
+ def stop(self) -> None:
1157
+ if not self._initialised:
1158
+ return
1159
+
1160
+ if not self._owns_daemon:
1161
+ # We piggy-backed on a daemon another process started. Don't
1162
+ # shut it down — that'd pull the rug out from under them.
1163
+ self._close_session_and_log()
1164
+ self._initialised = False
1165
+ return
1166
+
1167
+ logger.info("Shutting down IPFS daemon…")
1168
+ try:
1169
+ self._post("shutdown", timeout=10)
1170
+ except Exception:
1171
+ pass
1172
+
1173
+ if self._process:
1174
+ try:
1175
+ self._process.wait(timeout=10)
1176
+ except subprocess.TimeoutExpired:
1177
+ if sys.platform == "win32":
1178
+ self._process.terminate()
1179
+ else:
1180
+ self._process.send_signal(signal.SIGTERM)
1181
+ try:
1182
+ self._process.wait(timeout=5)
1183
+ except subprocess.TimeoutExpired:
1184
+ self._process.kill()
1185
+
1186
+ self._close_session_and_log()
1187
+ self._owns_daemon = False
1188
+ self._initialised = False
1189
+
1190
+ def _close_session_and_log(self) -> None:
1191
+ if self._session is not None:
1192
+ try:
1193
+ self._session.close()
1194
+ except Exception:
1195
+ pass
1196
+ self._session = None
1197
+ if self._log_file is not None:
1198
+ try:
1199
+ self._log_file.close()
1200
+ except Exception:
1201
+ pass
1202
+ self._log_file = None
1203
+
1204
+ # ---------------- HTTP helpers ----------------
1205
+
1206
+ def _is_api_up(self) -> bool:
1207
+ # Confirm the listener on API_HOST:API_PORT is actually a Kubo
1208
+ # daemon, not a stray service that happens to return 200 on POST.
1209
+ # Without this check, a port collision on a shared dev host could
1210
+ # silently route add/pin calls into an unrelated service.
1211
+ if self._session is None:
1212
+ return False
1213
+ try:
1214
+ r = self._session.post(f"{self._api_url}/version", timeout=2)
1215
+ except Exception:
1216
+ return False
1217
+ if r.status_code != 200:
1218
+ return False
1219
+ try:
1220
+ body = r.json()
1221
+ except Exception:
1222
+ return False
1223
+ return isinstance(body, dict) and isinstance(body.get("Version"), str)
1224
+
1225
+ def _post(
1226
+ self,
1227
+ endpoint: str,
1228
+ *,
1229
+ params: dict[str, Any] | list[tuple[str, Any]] | None = None,
1230
+ files: Any | None = None,
1231
+ data: Any | None = None,
1232
+ headers: dict[str, str] | None = None,
1233
+ stream: bool = False,
1234
+ timeout: float | None = 60,
1235
+ ) -> requests.Response:
1236
+ assert self._session is not None
1237
+ url = f"{self._api_url}/{endpoint}"
1238
+ try:
1239
+ resp = self._session.post(
1240
+ url, params=params, files=files, data=data, headers=headers,
1241
+ stream=stream, timeout=timeout,
1242
+ )
1243
+ except requests.ConnectionError as exc:
1244
+ raise IPFSOperationError(
1245
+ f"Cannot reach IPFS daemon at {url}. Is it running?"
1246
+ ) from exc
1247
+
1248
+ if not resp.ok:
1249
+ kubo_code: int | None = None
1250
+ kubo_type: str | None = None
1251
+ kubo_message: str | None = None
1252
+ try:
1253
+ body = resp.json()
1254
+ except Exception:
1255
+ body = None
1256
+ if isinstance(body, dict):
1257
+ kubo_message = body.get("Message")
1258
+ raw_code = body.get("Code")
1259
+ if isinstance(raw_code, int):
1260
+ kubo_code = raw_code
1261
+ raw_type = body.get("Type")
1262
+ if isinstance(raw_type, str):
1263
+ kubo_type = raw_type
1264
+ # Truncate raw bodies aggressively — a misconfigured reverse
1265
+ # proxy can return multi-MB HTML pages that would otherwise
1266
+ # land in the exception message and the daemon log.
1267
+ raw = resp.text or ""
1268
+ if len(raw) > 1024:
1269
+ raw = raw[:1024] + f"... [truncated {len(raw) - 1024} bytes]"
1270
+ detail = kubo_message if kubo_message is not None else raw
1271
+ raise IPFSOperationError(
1272
+ f"IPFS API error [{resp.status_code}] {endpoint}: {detail}",
1273
+ status_code=resp.status_code,
1274
+ kubo_code=kubo_code,
1275
+ kubo_type=kubo_type,
1276
+ kubo_message=kubo_message,
1277
+ )
1278
+ return resp
1279
+
1280
+ # ---------------- core operations ----------------
1281
+
1282
+ def _add(
1283
+ self,
1284
+ source: str | bytes | os.PathLike[str],
1285
+ pin: bool,
1286
+ provide: bool = True,
1287
+ only_hash: bool = False,
1288
+ timeout: float | None = None,
1289
+ ) -> str:
1290
+ # Build a single multipart part that streams from disk (or from
1291
+ # bytes wrapped in BytesIO). _iter_multipart() ensures the upload
1292
+ # never holds more than 64 KB at a time in RAM, so even a multi-GB
1293
+ # add() is RSS-flat.
1294
+ body: bytes | io.IOBase
1295
+ if isinstance(source, (bytes, bytearray, memoryview)):
1296
+ body = bytes(source)
1297
+ filename = "data"
1298
+ elif isinstance(source, (str, os.PathLike)):
1299
+ path = Path(os.fspath(source))
1300
+ if not path.exists():
1301
+ raise FileNotFoundError(f"File not found: {path}")
1302
+ if path.is_dir():
1303
+ raise IsADirectoryError(
1304
+ f"{path} is a directory — use add_folder() for directories."
1305
+ )
1306
+ # Hand the open handle to the streaming generator; it closes
1307
+ # after EOF. We intentionally do not use a context manager here
1308
+ # because the generator owns the read lifecycle.
1309
+ body = open(path, "rb")
1310
+ filename = path.name
1311
+ else:
1312
+ raise TypeError(
1313
+ f"source must be str, PathLike, or bytes, not {type(source).__name__}"
1314
+ )
1315
+
1316
+ params: list[tuple[str, str]] = [
1317
+ ("pin", "true" if pin else "false"),
1318
+ ("cid-version", "1"),
1319
+ ]
1320
+ if not provide:
1321
+ # Skip DHT announce. Content is still locally addressable; peers
1322
+ # who already know the CID can fetch via bitswap, but no provider
1323
+ # record is published.
1324
+ params.append(("provide", "false"))
1325
+ if only_hash:
1326
+ # Compute the CID without writing a block to the repo, pinning,
1327
+ # or announcing anything. Pin/provide flags are silently ignored
1328
+ # by Kubo when only-hash is set; we still send the same shape so
1329
+ # the request is uniform with the regular add path.
1330
+ params.append(("only-hash", "true"))
1331
+
1332
+ boundary, body_iter = _iter_multipart(
1333
+ [(filename, "application/octet-stream", body)]
1334
+ )
1335
+ try:
1336
+ resp = self._post(
1337
+ "add",
1338
+ params=params,
1339
+ data=body_iter,
1340
+ headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
1341
+ timeout=timeout,
1342
+ )
1343
+ finally:
1344
+ if hasattr(body, "close"):
1345
+ try:
1346
+ body.close()
1347
+ except Exception:
1348
+ pass
1349
+
1350
+ lines = [ln for ln in resp.text.strip().splitlines() if ln.strip()]
1351
+ cid: str = json.loads(lines[-1])["Hash"]
1352
+ return cid
1353
+
1354
+ def add(
1355
+ self,
1356
+ source: str | bytes | os.PathLike[str],
1357
+ provide: bool = True,
1358
+ *,
1359
+ timeout: float | None = None,
1360
+ ) -> str:
1361
+ return self._add(source, pin=True, provide=provide, timeout=timeout)
1362
+
1363
+ def announce(
1364
+ self,
1365
+ source: str | bytes | os.PathLike[str],
1366
+ provide: bool = True,
1367
+ *,
1368
+ timeout: float | None = None,
1369
+ ) -> str:
1370
+ return self._add(source, pin=False, provide=provide, timeout=timeout)
1371
+
1372
+ def compute_cid_locally(
1373
+ self,
1374
+ source: str | bytes | os.PathLike[str],
1375
+ *,
1376
+ timeout: float | None = None,
1377
+ ) -> str:
1378
+ # only-hash=true is Kubo's "chunk and hash, do not write" mode. No
1379
+ # block is stored in the repo, no pin is created, no provider record
1380
+ # is announced, no peer ever learns the CID exists.
1381
+ return self._add(
1382
+ source, pin=False, provide=False, only_hash=True, timeout=timeout
1383
+ )
1384
+
1385
+ def add_folder(
1386
+ self,
1387
+ path: str | os.PathLike[str],
1388
+ *,
1389
+ timeout: float | None = None,
1390
+ ) -> str:
1391
+ src = Path(os.fspath(path)).resolve()
1392
+ if not src.is_dir():
1393
+ raise NotADirectoryError(f"Not a directory: {src}")
1394
+
1395
+ # Build the parts list eagerly (cheap — just (name, type, path)
1396
+ # tuples) but defer file opens to _iter_multipart, which opens
1397
+ # each file as it streams it and closes it on EOF. This keeps
1398
+ # the open-FD count at 1 throughout the upload regardless of
1399
+ # how many files are in the tree, dodging both RAM and ulimit
1400
+ # exhaustion that the previous open-everything-up-front
1401
+ # implementation hit on large trees.
1402
+ parts: list[tuple[str, str, bytes | os.PathLike[str] | io.IOBase]] = []
1403
+ for current_dir, dirnames, filenames in os.walk(src):
1404
+ dirnames.sort()
1405
+ filenames.sort()
1406
+ current_path = Path(current_dir)
1407
+ rel = current_path.relative_to(src)
1408
+ dir_name = src.name if rel == Path(".") else (Path(src.name) / rel).as_posix()
1409
+ parts.append((dir_name, "application/x-directory", b""))
1410
+ for fname in filenames:
1411
+ rel_file = (Path(src.name) / rel / fname).as_posix()
1412
+ parts.append((rel_file, "application/octet-stream", current_path / fname))
1413
+
1414
+ boundary, body_iter = _iter_multipart(parts)
1415
+ resp = self._post(
1416
+ "add",
1417
+ params=[
1418
+ ("recursive", "true"),
1419
+ ("wrap-with-directory", "false"),
1420
+ ("pin", "true"),
1421
+ ("cid-version", "1"),
1422
+ ("quieter", "true"),
1423
+ ],
1424
+ data=body_iter,
1425
+ headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
1426
+ timeout=timeout,
1427
+ )
1428
+
1429
+ # quieter=true emits only the root entry — but parse defensively in
1430
+ # case Kubo decides to be chatty in a future version.
1431
+ lines = [ln for ln in resp.text.strip().splitlines() if ln.strip()]
1432
+ if not lines:
1433
+ raise IPFSOperationError("ipfs add returned no output")
1434
+ cid: str = json.loads(lines[-1])["Hash"]
1435
+ return cid
1436
+
1437
+ def cat(
1438
+ self,
1439
+ cid: str,
1440
+ output_path: str | os.PathLike[str] | None = None,
1441
+ *,
1442
+ timeout: float | None = None,
1443
+ max_bytes: int | None = -1,
1444
+ ) -> bytes | None:
1445
+ # Streaming path: writing to disk is unbounded — the caller chose to
1446
+ # spend disk, not RAM. The in-memory path is bounded by
1447
+ # ``max_bytes`` (or the module-level ``MAX_GET_BYTES`` default), so a
1448
+ # malicious / oversized CID can't OOM a long-running service.
1449
+ resp = self._post("cat", params={"arg": cid}, stream=True, timeout=timeout)
1450
+ if output_path is not None:
1451
+ with open(os.fspath(output_path), "wb") as fh:
1452
+ for chunk in resp.iter_content(chunk_size=1 << 16):
1453
+ fh.write(chunk)
1454
+ return None
1455
+ # Sentinel ``-1`` means "use the module default"; explicit ``None``
1456
+ # means "no cap" so callers who really want unbounded bytes can opt
1457
+ # out without monkey-patching the module global.
1458
+ cap = MAX_GET_BYTES if max_bytes == -1 else max_bytes
1459
+ buf = bytearray()
1460
+ for chunk in resp.iter_content(chunk_size=1 << 16):
1461
+ if not chunk:
1462
+ continue
1463
+ buf.extend(chunk)
1464
+ if cap is not None and len(buf) > cap:
1465
+ # Abort the stream rather than continue reading; close the
1466
+ # connection so we don't keep the daemon writing.
1467
+ resp.close()
1468
+ raise IPFSOperationError(
1469
+ f"Refusing to load CID {cid} into memory: exceeds "
1470
+ f"{cap} bytes (already received {len(buf)}). "
1471
+ f"Pass an output_path to stream to disk, or set "
1472
+ f"basic_ipfs.MAX_GET_BYTES = None / a larger value."
1473
+ )
1474
+ return bytes(buf)
1475
+
1476
+ def pin_add(
1477
+ self,
1478
+ cids: str | Iterable[str],
1479
+ *,
1480
+ timeout: float | None = None,
1481
+ ) -> None:
1482
+ cid_list = _as_str_list(cids)
1483
+ for chunk in _chunked(cid_list, _PIN_BATCH_SIZE):
1484
+ params = [("arg", c) for c in chunk]
1485
+ params.append(("recursive", "true"))
1486
+ self._post("pin/add", params=params, timeout=timeout)
1487
+
1488
+ def pin_rm(
1489
+ self,
1490
+ cids: str | Iterable[str],
1491
+ *,
1492
+ timeout: float | None = 60,
1493
+ ) -> None:
1494
+ cid_list = _as_str_list(cids)
1495
+ succeeded: list[str] = []
1496
+ failed: list[tuple[str, str]] = []
1497
+ for chunk in _chunked(cid_list, _PIN_BATCH_SIZE):
1498
+ try:
1499
+ params = [("arg", c) for c in chunk]
1500
+ params.append(("recursive", "true"))
1501
+ self._post("pin/rm", params=params, timeout=timeout)
1502
+ succeeded.extend(chunk)
1503
+ except IPFSOperationError as exc:
1504
+ if not exc.is_not_pinned():
1505
+ # Hard failure on the batch — record what was processed
1506
+ # so the caller can retry the rest, then re-raise with
1507
+ # the partial-success context attached. The whole
1508
+ # current chunk is the "failed" set since Kubo aborts
1509
+ # an erroring batch atomically.
1510
+ wrapped = IPFSOperationError(
1511
+ f"pin/rm failed mid-batch after {len(succeeded)} "
1512
+ f"successful unpins: {exc}",
1513
+ status_code=exc.status_code,
1514
+ kubo_code=exc.kubo_code,
1515
+ kubo_type=exc.kubo_type,
1516
+ kubo_message=exc.kubo_message,
1517
+ )
1518
+ wrapped.succeeded_cids = succeeded
1519
+ wrapped.failed_cids = list(chunk)
1520
+ raise wrapped from exc
1521
+ # One or more CIDs in the batch weren't pinned. Kubo aborts
1522
+ # the whole batch in that case, so fall back to per-CID
1523
+ # removal to preserve idempotency. Track per-CID outcomes
1524
+ # so a hard failure mid-fallback surfaces the partial state
1525
+ # to the caller instead of silently leaving half the chunk
1526
+ # unpinned.
1527
+ for c in chunk:
1528
+ try:
1529
+ self._post(
1530
+ "pin/rm",
1531
+ params={"arg": c, "recursive": "true"},
1532
+ timeout=timeout,
1533
+ )
1534
+ succeeded.append(c)
1535
+ except IPFSOperationError as per_exc:
1536
+ if per_exc.is_not_pinned():
1537
+ succeeded.append(c) # idempotent no-op counts as success
1538
+ continue
1539
+ failed.append((c, str(per_exc)))
1540
+ if failed:
1541
+ err = IPFSOperationError(
1542
+ f"unpin partially succeeded: {len(succeeded)} ok, "
1543
+ f"{len(failed)} failed; first failure: {failed[0][1]}"
1544
+ )
1545
+ err.failed_cids = [c for c, _ in failed]
1546
+ err.succeeded_cids = succeeded
1547
+ raise err
1548
+
1549
+ def pin_ls(self) -> list[str]:
1550
+ resp = self._post("pin/ls", params={"type": "recursive"}, timeout=10).json()
1551
+ return list(resp.get("Keys", {}).keys())
1552
+
1553
+ def pin_check(self, cid: str) -> bool:
1554
+ # Use type=recursive (matches pin_ls). Indirect pins (folder children)
1555
+ # are intentionally NOT counted — the only pins basic_ipfs creates are
1556
+ # recursive ones via add(), so this is the consistent answer.
1557
+ try:
1558
+ self._post("pin/ls", params={"arg": cid, "type": "recursive"}, timeout=10)
1559
+ return True
1560
+ except IPFSOperationError as exc:
1561
+ if exc.is_not_pinned():
1562
+ return False
1563
+ raise
1564
+
1565
+ def block_exists(self, cid: str) -> bool:
1566
+ # offline=true is critical: without it, Kubo will try to fetch the
1567
+ # block from the swarm if it isn't local, which can hang for the
1568
+ # full timeout and contradicts what exists() promises.
1569
+ try:
1570
+ self._post(
1571
+ "block/stat",
1572
+ params={"arg": cid, "offline": "true"},
1573
+ timeout=5,
1574
+ )
1575
+ return True
1576
+ except (IPFSOperationError, requests.exceptions.Timeout):
1577
+ return False
1578
+
1579
+ def repo_gc(self, *, timeout: float | None = None) -> None:
1580
+ self._post("repo/gc", timeout=timeout)
1581
+
1582
+ def swarm_peers(self) -> list[str]:
1583
+ resp = self._post("swarm/peers", timeout=10).json()
1584
+ result = []
1585
+ for p in resp.get("Peers", []) or []:
1586
+ addr = p.get("Addr", "")
1587
+ peer = p.get("Peer", "")
1588
+ if peer and "/p2p/" not in addr:
1589
+ addr = f"{addr}/p2p/{peer}"
1590
+ if addr:
1591
+ result.append(addr)
1592
+ return result
1593
+
1594
+ def swarm_connect(self, multiaddrs: str | Iterable[str]) -> None:
1595
+ for addr in _as_str_list(multiaddrs):
1596
+ self._post("swarm/connect", params={"arg": addr}, timeout=30)
1597
+
1598
+ def my_addrs(self) -> list[str]:
1599
+ resp = self._post("id", timeout=5).json()
1600
+ return resp.get("Addresses", []) or []
1601
+
1602
+ def status(self) -> StatusDict:
1603
+ ver = self._post("version", timeout=5).json()
1604
+ id_info = self._post("id", timeout=5).json()
1605
+ repo_stat = self._post("repo/stat", timeout=10).json()
1606
+ pins = self._post("pin/ls", params={"type": "recursive"}, timeout=10).json()
1607
+ return {
1608
+ "peer_id": id_info.get("ID"),
1609
+ "agent_version": ver.get("Version"),
1610
+ "repo_size_bytes": repo_stat.get("RepoSize"),
1611
+ "num_objects": repo_stat.get("NumObjects"),
1612
+ "pinned_cids": len(pins.get("Keys", {})),
1613
+ "addresses": id_info.get("Addresses", []) or [],
1614
+ }
1615
+
1616
+
1617
+ # ---------------------------------------------------------------------------
1618
+ # Module-level singleton — lazy-started on first use
1619
+ # ---------------------------------------------------------------------------
1620
+
1621
+ _manager: IPFSManager | None = None
1622
+ _start_lock = threading.Lock()
1623
+ _atexit_registered = False
1624
+
1625
+
1626
+ def _atexit_stop() -> None:
1627
+ """Module-scoped atexit handler — always stops the *current* manager.
1628
+
1629
+ Registered exactly once per process so a long-running supervisor that
1630
+ cycles stop()/start() many times does not accumulate dead bound-method
1631
+ references on the atexit chain.
1632
+ """
1633
+ global _manager
1634
+ if _manager is not None:
1635
+ try:
1636
+ _manager.stop()
1637
+ except Exception:
1638
+ pass
1639
+ _manager = None
1640
+
1641
+
1642
+ def _ensure_atexit_registered() -> None:
1643
+ global _atexit_registered
1644
+ if not _atexit_registered:
1645
+ atexit.register(_atexit_stop)
1646
+ _atexit_registered = True
1647
+
1648
+
1649
+ def _get_manager() -> IPFSManager:
1650
+ global _manager
1651
+ if _manager is None:
1652
+ with _start_lock:
1653
+ if _manager is None:
1654
+ m = IPFSManager()
1655
+ m.start()
1656
+ # Publish only after start() completes, so other threads
1657
+ # never observe a half-initialised manager.
1658
+ _manager = m
1659
+ return _manager
1660
+
1661
+
1662
+ # ---------------------------------------------------------------------------
1663
+ # Public API — the functions you actually call
1664
+ # ---------------------------------------------------------------------------
1665
+
1666
+
1667
+ def add(
1668
+ source: str | bytes | os.PathLike[str],
1669
+ provide: bool = True,
1670
+ *,
1671
+ timeout: float | None = None,
1672
+ ) -> str:
1673
+ """Add a file (path or bytes) to IPFS and pin it. Returns the CID.
1674
+
1675
+ Pass ``provide=False`` to skip the DHT announce — the content is still
1676
+ addressable by CID, but its presence on this node is not advertised to
1677
+ the public network. Useful for staging private data before deciding
1678
+ whether to publish.
1679
+
1680
+ ``timeout`` (seconds) bounds the upload. Default ``None`` waits as
1681
+ long as the daemon needs (right for huge files).
1682
+ """
1683
+ return _get_manager().add(source, provide=provide, timeout=timeout)
1684
+
1685
+
1686
+ def announce(
1687
+ source: str | bytes | os.PathLike[str],
1688
+ provide: bool = True,
1689
+ *,
1690
+ timeout: float | None = None,
1691
+ ) -> str:
1692
+ """Add a file (path or bytes) to IPFS without pinning. Returns the CID.
1693
+
1694
+ Content is available on the network but may be garbage-collected unless
1695
+ pinned later. Pass ``provide=False`` to skip the DHT announce.
1696
+ ``timeout`` (seconds) bounds the upload; default is no timeout.
1697
+ """
1698
+ return _get_manager().announce(source, provide=provide, timeout=timeout)
1699
+
1700
+
1701
+ _GET_TIMEOUT_DEFAULT = object()
1702
+
1703
+
1704
+ @overload
1705
+ def get(cid: str) -> bytes: ...
1706
+ @overload
1707
+ def get(cid: str, output_path: str | os.PathLike[str]) -> None: ...
1708
+
1709
+
1710
+ def get(
1711
+ cid: str,
1712
+ output_path: str | os.PathLike[str] | None = None,
1713
+ *,
1714
+ timeout: Any = _GET_TIMEOUT_DEFAULT,
1715
+ max_bytes: int | None = -1,
1716
+ ) -> bytes | None:
1717
+ """Retrieve content by CID.
1718
+
1719
+ Returns bytes unless ``output_path`` is given (then writes to disk and
1720
+ returns ``None``).
1721
+
1722
+ ``timeout`` (seconds) bounds how long the daemon may spend fetching the
1723
+ content. The default uses ``basic_ipfs.DEFAULT_GET_TIMEOUT`` (300 s
1724
+ out of the box) so an unreachable CID raises rather than hanging the
1725
+ calling thread forever. Pass an explicit ``timeout=None`` to opt
1726
+ out and wait indefinitely.
1727
+
1728
+ ``max_bytes`` caps the size of the in-memory return value. The default
1729
+ sentinel ``-1`` uses ``basic_ipfs.MAX_GET_BYTES`` (256 MB out of the
1730
+ box). Pass ``None`` to opt out of the cap entirely (e.g. a CLI tool
1731
+ that's fine letting the OS OOM-kill it). Ignored when ``output_path``
1732
+ is set — streaming to disk is unbounded by design.
1733
+ """
1734
+ effective_timeout = (
1735
+ DEFAULT_GET_TIMEOUT if timeout is _GET_TIMEOUT_DEFAULT else timeout
1736
+ )
1737
+ return _get_manager().cat(
1738
+ cid, output_path, timeout=effective_timeout, max_bytes=max_bytes
1739
+ )
1740
+
1741
+
1742
+ def pin(cids: str | Iterable[str], *, timeout: float | None = None) -> None:
1743
+ """Recursively pin one or many CIDs so they are never garbage-collected.
1744
+
1745
+ Accepts a single CID string or any iterable of CID strings.
1746
+ ``timeout`` (seconds) bounds each batch HTTP call to the daemon.
1747
+ """
1748
+ _get_manager().pin_add(cids, timeout=timeout)
1749
+
1750
+
1751
+ def unpin(cids: str | Iterable[str], *, timeout: float | None = None) -> None:
1752
+ """Unpin one or many CIDs. Accepts a single CID string or any iterable.
1753
+
1754
+ Idempotent — already-unpinned CIDs are silently skipped.
1755
+ ``timeout`` (seconds) bounds each batch HTTP call to the daemon.
1756
+ """
1757
+ _get_manager().pin_rm(cids, timeout=timeout)
1758
+
1759
+
1760
+ def get_all_pins() -> list[str]:
1761
+ """Return a list of all currently pinned CIDs (recursive pins only)."""
1762
+ return _get_manager().pin_ls()
1763
+
1764
+
1765
+ def is_pinned(cid: str) -> bool:
1766
+ """Return True if cid has a recursive pin (matches get_all_pins())."""
1767
+ return _get_manager().pin_check(cid)
1768
+
1769
+
1770
+ def exists(cid: str) -> bool:
1771
+ """Return True if cid is available in the local node."""
1772
+ return _get_manager().block_exists(cid)
1773
+
1774
+
1775
+ def garbage_collection(*, timeout: float | None = None) -> None:
1776
+ """Trigger Kubo GC, freeing storage used by unpinned content.
1777
+
1778
+ ``timeout`` (seconds) bounds the GC call; default is no timeout
1779
+ because GC over a large repo is legitimately slow.
1780
+ """
1781
+ _get_manager().repo_gc(timeout=timeout)
1782
+
1783
+
1784
+ def add_folder(
1785
+ path: str | os.PathLike[str],
1786
+ *,
1787
+ timeout: float | None = None,
1788
+ ) -> str:
1789
+ """Add a directory to IPFS recursively and pin it. Returns the root CID.
1790
+
1791
+ ``timeout`` (seconds) bounds the upload; default is no timeout.
1792
+ """
1793
+ return _get_manager().add_folder(path, timeout=timeout)
1794
+
1795
+
1796
+ def compute_cid_locally(
1797
+ source: str | bytes | os.PathLike[str],
1798
+ *,
1799
+ timeout: float | None = None,
1800
+ ) -> str:
1801
+ """Return the CID a file or bytes payload *would* have, without
1802
+ publishing anything.
1803
+
1804
+ Chunks and hashes the data using Kubo's ``add --only-hash`` mode:
1805
+ no block is written to the repo, no pin is created, no provider
1806
+ record is announced to the DHT, no peer ever learns the CID exists.
1807
+ Useful for previewing the CID of content you have not yet decided
1808
+ to publish.
1809
+
1810
+ Note: this routes through the local Kubo daemon's HTTP API, so the
1811
+ daemon must be running (and the binary must be installed). It does
1812
+ not by itself initiate any IPFS-network traffic — Kubo computes the
1813
+ CID locally and never advertises it. ``timeout`` (seconds) bounds
1814
+ the local hash computation; default is no timeout.
1815
+
1816
+ Output matches what ``add()`` / ``announce()`` would return for the
1817
+ same bytes (CIDv1, default chunker).
1818
+ """
1819
+ return _get_manager().compute_cid_locally(source, timeout=timeout)
1820
+
1821
+
1822
+ def peers() -> list[str]:
1823
+ """Return the full multiaddrs of all currently connected peers."""
1824
+ return _get_manager().swarm_peers()
1825
+
1826
+
1827
+ def connect_to_node(multiaddress: str) -> None:
1828
+ """Connect to a peer by multiaddr (e.g. /ip4/1.2.3.4/tcp/4001/p2p/12D3KooW...)."""
1829
+ _get_manager().swarm_connect(multiaddress)
1830
+
1831
+
1832
+ def connect_to_nodes(multiaddresses: Iterable[str]) -> None:
1833
+ """Connect to multiple peers. Errors raise on the first failed address."""
1834
+ _get_manager().swarm_connect(multiaddresses)
1835
+
1836
+
1837
+ def my_node_multiaddress() -> tuple[str | None, str | None]:
1838
+ """Return (ipv4_multiaddr, ipv6_multiaddr) for this node.
1839
+
1840
+ Filters out loopback and link-local addresses and prefers public IPs.
1841
+ Either value may be None if that address family isn't available.
1842
+ Both strings are in the exact format connect_to_node() accepts.
1843
+ """
1844
+ all_addrs = _get_manager().my_addrs()
1845
+ scored = [(a, _addr_score(a)) for a in all_addrs]
1846
+ usable = [(a, s) for a, s in scored if s >= 0]
1847
+
1848
+ ipv4 = next(
1849
+ (a for a, _ in sorted(
1850
+ ((a, s) for a, s in usable if a.startswith("/ip4/")),
1851
+ key=lambda x: x[1], reverse=True,
1852
+ )),
1853
+ None,
1854
+ )
1855
+ ipv6 = next(
1856
+ (a for a, _ in sorted(
1857
+ ((a, s) for a, s in usable if a.startswith("/ip6/")),
1858
+ key=lambda x: x[1], reverse=True,
1859
+ )),
1860
+ None,
1861
+ )
1862
+ return ipv4, ipv6
1863
+
1864
+
1865
+ def status() -> StatusDict:
1866
+ """Return peer ID, repo size, pinned count, etc."""
1867
+ return _get_manager().status()
1868
+
1869
+
1870
+ def start() -> None:
1871
+ """Explicitly start the daemon. Also happens lazily on the first call."""
1872
+ _get_manager()
1873
+
1874
+
1875
+ def stop() -> None:
1876
+ """Explicitly stop the daemon. Also happens automatically on process exit."""
1877
+ global _manager
1878
+ with _start_lock:
1879
+ if _manager is not None:
1880
+ _manager.stop()
1881
+ _manager = None
1882
+
1883
+
1884
+ class Node:
1885
+ """Context manager: start on enter, stop on exit."""
1886
+
1887
+ def __enter__(self) -> Node:
1888
+ start()
1889
+ return self
1890
+
1891
+ def __exit__(self, *_: Any) -> None:
1892
+ stop()
1893
+
1894
+
1895
+ # Lowercase alias kept permanently for the documented `with basic_ipfs.node():` form.
1896
+ node = Node
1897
+
1898
+
1899
+ # ---------------------------------------------------------------------------
1900
+ # Private network support
1901
+ #
1902
+ # By default every IPFS node joins the *public* IPFS network: anyone can
1903
+ # connect to you, and your CIDs are discoverable by the whole world.
1904
+ #
1905
+ # A private network changes this completely. Every node in your group shares
1906
+ # a secret 256-bit key stored in a file called `swarm.key` inside the IPFS
1907
+ # repo. When Kubo starts with that file present it enters isolation mode:
1908
+ #
1909
+ # - It will ONLY complete the libp2p handshake with peers that present the
1910
+ # same key. Peers without it are silently dropped before any data is
1911
+ # exchanged — they can't even tell a node is there.
1912
+ # - It does NOT join the public DHT. Your group maintains its own private
1913
+ # routing table. The outside world cannot discover your peers or CIDs.
1914
+ # - XSalsa20 stream encryption is applied on top of every connection using
1915
+ # the shared key, so traffic between members is also encrypted in transit.
1916
+ #
1917
+ # The practical result: connect_to_nodes() + a shared swarm key gives you a
1918
+ # completely isolated peer-to-peer network. Only nodes you give the key to
1919
+ # can participate — this is your private DHT.
1920
+ #
1921
+ # ── How to set up a private network ─────────────────────────────────────────
1922
+ #
1923
+ # One node (the "founder") runs this BEFORE starting the daemon:
1924
+ #
1925
+ # import basic_ipfs
1926
+ # key = basic_ipfs.create_private_network() # generates key, writes swarm.key
1927
+ # basic_ipfs.start() # daemon starts in isolation mode
1928
+ # ipv4, ipv6 = basic_ipfs.my_node_multiaddress()
1929
+ # print(key) # share this secret string with trusted nodes
1930
+ # print(ipv4) # share your address so they can connect to you
1931
+ #
1932
+ # Every other node runs this BEFORE starting their daemon:
1933
+ #
1934
+ # import basic_ipfs
1935
+ # basic_ipfs.join_private_network("<key from founder>")
1936
+ # basic_ipfs.start()
1937
+ # basic_ipfs.connect_to_node("<founder's ipv4 or ipv6 multiaddr>")
1938
+ # # now fully peered — content flows directly between members
1939
+ #
1940
+ # IMPORTANT: swarm.key must exist before `ipfs daemon` launches. These
1941
+ # functions raise IPFSError if called while the daemon is already running.
1942
+ # ---------------------------------------------------------------------------
1943
+
1944
+
1945
+ def _swarm_key_path() -> Path:
1946
+ # Kubo looks for swarm.key directly inside $IPFS_PATH (the repo root).
1947
+ return _get_repo_path() / "swarm.key"
1948
+
1949
+
1950
+ def _write_swarm_key(key_hex: str) -> None:
1951
+ # The swarm.key file format is defined by go-libp2p-pnet. It is a
1952
+ # three-line text file:
1953
+ #
1954
+ # /key/swarm/psk/1.0.0/ ← protocol identifier — must be this exact string
1955
+ # /base16/ ← encoding of the key material — base16 = hex
1956
+ # <64 hex chars> ← 32 random bytes = 256-bit key
1957
+ #
1958
+ # Kubo reads this at startup and refuses connections from any peer whose
1959
+ # swarm.key does not contain the same 64 hex chars.
1960
+ path = _swarm_key_path()
1961
+ if os.name == "posix":
1962
+ _secure_mkdir(path.parent, mode=0o700)
1963
+ else:
1964
+ path.parent.mkdir(parents=True, exist_ok=True)
1965
+ body = f"/key/swarm/psk/1.0.0/\n/base16/\n{key_hex}\n"
1966
+ if os.name == "posix":
1967
+ # Atomic create with mode 0o600 — closes the TOCTOU window between
1968
+ # write_text() and chmod() that would otherwise leak this credential
1969
+ # to any local user during the umask gap. swarm.key is a 256-bit
1970
+ # symmetric secret; a single read leaks permanent network access.
1971
+ _secure_write_text(path, body, mode=0o600)
1972
+ else:
1973
+ path.write_text(body)
1974
+
1975
+
1976
+ def _swarm_key_warn_if_world_readable() -> None:
1977
+ if os.name != "posix":
1978
+ return
1979
+ path = _swarm_key_path()
1980
+ if not path.exists():
1981
+ return
1982
+ try:
1983
+ mode = path.stat().st_mode & 0o777
1984
+ except OSError:
1985
+ return
1986
+ if mode & 0o077:
1987
+ logger.warning(
1988
+ "swarm.key at %s is readable by other users (mode 0o%o). "
1989
+ "Run: chmod 600 %s", path, mode, path,
1990
+ )
1991
+
1992
+
1993
+ def _daemon_is_running() -> bool:
1994
+ return _manager is not None and getattr(_manager, "_initialised", False)
1995
+
1996
+
1997
+ def create_private_network() -> str:
1998
+ """Create a new private IPFS network by generating a random swarm key.
1999
+
2000
+ Writes swarm.key to the repo so that when the daemon starts it will only
2001
+ accept connections from peers that share the same key.
2002
+
2003
+ Returns the key as a 64-character hex string. Share this string (securely)
2004
+ with every node that should join your network — they pass it to
2005
+ join_private_network() before starting their daemon.
2006
+
2007
+ Must be called BEFORE start() / the first IPFS operation.
2008
+ Safe to call multiple times — overwrites any existing swarm.key.
2009
+ """
2010
+ if _daemon_is_running():
2011
+ raise IPFSError(
2012
+ "Cannot write swarm.key while the daemon is running. "
2013
+ "Call create_private_network() before start() or any IPFS operation."
2014
+ )
2015
+ # os.urandom uses the OS CSPRNG (e.g. /dev/urandom on Linux, CryptGenRandom
2016
+ # on Windows) — safe for generating cryptographic key material.
2017
+ key_hex = os.urandom(32).hex() # 32 bytes → 64 hex chars → 256-bit key
2018
+ _write_swarm_key(key_hex)
2019
+ logger.info("Private network swarm key written to %s", _swarm_key_path())
2020
+ return key_hex
2021
+
2022
+
2023
+ def join_private_network(key: str) -> None:
2024
+ """Join an existing private network using a swarm key from another node.
2025
+
2026
+ key: the 64-character hex string returned by create_private_network() on
2027
+ the founder's node. Treat it like a password — anyone with this key
2028
+ can join your network.
2029
+
2030
+ Writes swarm.key to the repo so that when the daemon starts it will only
2031
+ accept connections from peers sharing the same key.
2032
+
2033
+ Must be called BEFORE start() / the first IPFS operation.
2034
+ """
2035
+ if _daemon_is_running():
2036
+ raise IPFSError(
2037
+ "Cannot write swarm.key while the daemon is running. "
2038
+ "Call join_private_network() before start() or any IPFS operation."
2039
+ )
2040
+ key = key.strip().lower()
2041
+ # Validate: must be exactly 64 lowercase hex characters (32 bytes).
2042
+ if len(key) != 64 or not all(c in "0123456789abcdef" for c in key):
2043
+ raise ValueError(
2044
+ "key must be a 64-character hex string, as returned by "
2045
+ "create_private_network(). "
2046
+ f"Got {len(key)!r} characters."
2047
+ )
2048
+ _write_swarm_key(key)
2049
+ logger.info("Private network swarm key written to %s", _swarm_key_path())
2050
+
2051
+
2052
+ def get_private_network_key() -> str | None:
2053
+ """Return this node's swarm key as a hex string, or None if not configured.
2054
+
2055
+ Use this to retrieve the key from the founder node so you can share it
2056
+ with new members who need to call join_private_network().
2057
+ """
2058
+ path = _swarm_key_path()
2059
+ if not path.exists():
2060
+ return None
2061
+ for line in path.read_text().splitlines():
2062
+ line = line.strip()
2063
+ # The key line is the only line that doesn't start with '/'.
2064
+ if line and not line.startswith("/"):
2065
+ return line
2066
+ return None
2067
+
2068
+
2069
+ def is_private_network() -> bool:
2070
+ """Return True if this node is configured to run as a private network."""
2071
+ return _swarm_key_path().exists()
2072
+
2073
+
2074
+ def rotate_identity(oldkey_name: str = "previous-self") -> str:
2075
+ """Generate a new libp2p keypair, severing peer-ID linkability with prior runs.
2076
+
2077
+ The old key is preserved under ``oldkey_name`` (default
2078
+ ``"previous-self"``) inside the keystore so it can be exported if you
2079
+ ever need to re-sign IPNS records published under the old identity.
2080
+ Pins, config, and ``swarm.key`` are untouched.
2081
+
2082
+ The daemon must NOT be running. Call before the first IPFS operation,
2083
+ or stop a running daemon first (Python process exit stops it cleanly).
2084
+
2085
+ Returns the new peer ID. Raises :class:`IPFSError` if the daemon is
2086
+ running, the repo is missing, or the underlying ``ipfs key rotate``
2087
+ fails.
2088
+ """
2089
+ if _daemon_is_running():
2090
+ raise IPFSError(
2091
+ "Cannot rotate identity while the daemon is running. "
2092
+ "Stop it first (let the Python process exit, or call stop())."
2093
+ )
2094
+ repo = _get_repo_path()
2095
+ if not (repo / "config").exists():
2096
+ raise IPFSError(
2097
+ f"No IPFS repo at {repo}. rotate_identity() requires an existing "
2098
+ "repo — run an operation once to initialise it, then rotate."
2099
+ )
2100
+ binary = _find_or_install_kubo()
2101
+ env = os.environ.copy()
2102
+ env["IPFS_PATH"] = str(repo)
2103
+ try:
2104
+ subprocess.run(
2105
+ [
2106
+ str(binary), "key", "rotate",
2107
+ f"--oldkey={oldkey_name}",
2108
+ "--type=ed25519",
2109
+ ],
2110
+ env=env, check=True, capture_output=True, text=True, timeout=30,
2111
+ )
2112
+ except subprocess.CalledProcessError as exc:
2113
+ raise IPFSError(
2114
+ f"Failed to rotate identity: {exc.stderr.strip()}"
2115
+ ) from exc
2116
+ try:
2117
+ config = json.loads((repo / "config").read_text())
2118
+ new_id: str = config["Identity"]["PeerID"]
2119
+ except (OSError, KeyError, json.JSONDecodeError) as exc:
2120
+ raise IPFSError(f"Could not read new PeerID after rotation: {exc}") from exc
2121
+ logger.info(
2122
+ "Identity rotated. New peer ID: %s. Old key preserved as %r.",
2123
+ new_id, oldkey_name,
2124
+ )
2125
+ return new_id
2126
+
2127
+
2128
+ def lockdown_mode() -> None:
2129
+ """Configure this node to never talk to the public IPFS network.
2130
+
2131
+ Sets, on the on-disk repo:
2132
+
2133
+ - ``Routing.Type = "none"`` — no DHT participation, no provider records.
2134
+ - ``Bootstrap = []`` — no auto-dial of Protocol Labs / Cloudflare nodes.
2135
+ - ``Swarm.Addresses`` restricted to loopback (``127.0.0.1`` + ``::1``).
2136
+ - ``Gateway.NoFetch = true`` — local gateway will not fetch remote CIDs.
2137
+
2138
+ Call BEFORE :func:`start` / the first IPFS operation. The repo must
2139
+ already exist (run any operation once to initialise it, or call
2140
+ :func:`create_private_network` first if you want a private network
2141
+ overlaid).
2142
+
2143
+ This is the right starting point for: encrypted-content stores, local
2144
+ caches, air-gapped pipelines, or any workflow where contacting third
2145
+ parties is unacceptable.
2146
+ """
2147
+ if _daemon_is_running():
2148
+ raise IPFSError(
2149
+ "Cannot apply lockdown_mode while the daemon is running. "
2150
+ "Call lockdown_mode() before start() / the first IPFS operation."
2151
+ )
2152
+ repo = _get_repo_path()
2153
+ if not (repo / "config").exists():
2154
+ raise IPFSError(
2155
+ f"No IPFS repo at {repo}. lockdown_mode() requires an existing "
2156
+ "repo — run an operation once to initialise it, or call "
2157
+ "create_private_network() first."
2158
+ )
2159
+ binary = _find_or_install_kubo()
2160
+ env = os.environ.copy()
2161
+ env["IPFS_PATH"] = str(repo)
2162
+
2163
+ settings: list[tuple[list[str], str]] = [
2164
+ (["config", "Routing.Type", "none"], "Routing.Type"),
2165
+ (["config", "--json", "Bootstrap", "[]"], "Bootstrap"),
2166
+ (
2167
+ [
2168
+ "config", "--json", "Addresses.Swarm",
2169
+ json.dumps(["/ip4/127.0.0.1/tcp/4001", "/ip6/::1/tcp/4001"]),
2170
+ ],
2171
+ "Addresses.Swarm",
2172
+ ),
2173
+ (["config", "--json", "Gateway.NoFetch", "true"], "Gateway.NoFetch"),
2174
+ ]
2175
+ for argv, label in settings:
2176
+ try:
2177
+ subprocess.run(
2178
+ [str(binary), *argv],
2179
+ env=env, check=True, capture_output=True, text=True, timeout=15,
2180
+ )
2181
+ except subprocess.CalledProcessError as exc:
2182
+ raise IPFSError(
2183
+ f"Failed to set {label} during lockdown_mode: {exc.stderr.strip()}"
2184
+ ) from exc
2185
+ logger.info("lockdown_mode applied — node will not talk to public IPFS.")
2186
+
2187
+
2188
+ # ---------------------------------------------------------------------------
2189
+ # Long-name aliases — so code written against OpenMarket-Desktop's
2190
+ # ipfs_manager.py keeps working after `pip install basic-ipfs`
2191
+ # ---------------------------------------------------------------------------
2192
+
2193
+ add_to_ipfs = add
2194
+ announce_to_ipfs = announce
2195
+ get_from_ipfs = get
2196
+ ipfs_pin = pin
2197
+ ipfs_stop_pinning = unpin
2198
+ get_pinned_cids = get_all_pins
2199
+ start_ipfs_node = start
2200
+ stop_ipfs_node = stop
2201
+ get_ipfs_status = status
2202
+
2203
+ __all__ = [
2204
+ # short API
2205
+ "add", "announce", "add_folder", "compute_cid_locally",
2206
+ "get", "pin", "unpin", "get_all_pins", "is_pinned",
2207
+ "exists", "garbage_collection",
2208
+ "peers", "connect_to_node", "connect_to_nodes", "my_node_multiaddress",
2209
+ "create_private_network", "join_private_network",
2210
+ "get_private_network_key", "is_private_network",
2211
+ "rotate_identity", "lockdown_mode",
2212
+ "status", "start", "stop", "Node", "node",
2213
+ # long aliases
2214
+ "add_to_ipfs", "announce_to_ipfs", "get_from_ipfs", "ipfs_pin", "ipfs_stop_pinning",
2215
+ "get_pinned_cids", "start_ipfs_node", "stop_ipfs_node", "get_ipfs_status",
2216
+ # exceptions
2217
+ "IPFSError", "IPFSBinaryNotFound", "IPFSDaemonTimeout", "IPFSOperationError",
2218
+ "IPFSPortInUse", "IPFSRepoLocked", "IPFSRepoCorrupt",
2219
+ # types
2220
+ "StatusDict",
2221
+ # power-user internals
2222
+ "IPFSManager",
2223
+ ]
2224
+
2225
+ # Annotations are lazy (PEP 563), so these names exist only for type checkers
2226
+ # and don't need to live in the module namespace at runtime. Hide them so
2227
+ # `basic_ipfs.<TAB>` doesn't surface them. (Path / TypedDict / overload are
2228
+ # kept — they're used at runtime.)
2229
+ del Any, Iterable, Union