flowstatemcp 2.5.0__py3-none-any.whl

flowstate/offsite_crypto.py

@@ -0,0 +1,272 @@
+"""Encryption for the off-site GitHub copy (2.4.4, data security audit D8).
+
+The nightly copy is encrypted on this computer before it is pushed; GitHub only ever holds the
+encrypted file. AES-256-GCM from the `cryptography` package (it comes with `mcp`; FlowState lists it as its
+`offsite` extra and checks for it in tests; the relay uses it too). Without it the push is
+refused: an unencrypted copy is never pushed.
+
+The key: 32 random bytes made on this computer the first time, kept in the OS credential store
+(Windows Credential Manager `flowstate:offsite-key`, macOS Keychain `flowstate-offsite-key`,
+Linux a 0600 file `~/.config/flowstate/offsite-key`). The recovery code is the same key written
+for people (`flowstate offsite key`): save it in a password manager. Without it, a lost or
+replaced computer can't read the copy; that is the point.
+
+File format (`.fsenc`): MAGIC, 8-byte key id (start of SHA-256 of the key), 8-byte random nonce
+prefix, 4-byte chunk size, and (format 2, MAGIC2) the 8-byte time the copy was made (Unix
+seconds); then 1 MiB chunks, each sealed with nonce = prefix + 4-byte counter and the header +
+counter + a last-chunk flag as associated data, so chunks can't be reordered, dropped or cut off
+without the check failing, and the time in the header can't be changed. A file with no chunk at
+all (cut down to its header) is refused. Format-1 files (no time) are still read.
+
+The key is made only when the store says definitely "there is none"; a store that can't be
+read stops the push, and an existing key is never written over (re-audit N-F6).
+"""
+
+import base64
+import hashlib
+import os
+import secrets
+import struct
+from pathlib import Path
+from typing import Optional
+
+MAGIC = b"FSENC1\0"
+MAGIC2 = b"FSENC2\0"   # + the time the copy was made, sealed with the chunks
+CHUNK = 1 << 20
+_HEADER = len(MAGIC) + 8 + 8 + 4
+_HEADER2 = _HEADER + 8
+SUFFIX = ".fsenc"
+
+
+class CryptoError(Exception):
+    pass
+
+
+def _aes(key: bytes):
+    try:
+        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+    except ImportError:
+        raise CryptoError("encrypting the off-site copy needs the `cryptography` package "
+                          "(one of FlowState's own packages). Nothing was "
+                          "pushed; reinstall FlowState's packages and try again.") from None
+    return AESGCM(key)
+
+
+def key_id(key: bytes) -> str:
+    return hashlib.sha256(key).hexdigest()[:16]
+
+
+# ---------------------------------------------------------------- the key
+def _store():
+    from . import secret_store as SS
+    if os.environ.get("FLOWSTATE_TOKEN_STORE_FILE"):   # tests and sandboxes: a private file
+        return SS.FileStore(Path(os.environ["FLOWSTATE_TOKEN_STORE_FILE"]).expanduser()
+                            .with_name("offsite-key"))
+    import platform
+    system = platform.system()
+    if system == "Windows":
+        return SS.WindowsStore(target="flowstate:offsite-key")
+    if system == "Darwin":
+        return SS.MacStore(service="flowstate-offsite-key")
+    from . import paths
+    return SS.FileStore(paths.config_dir() / "offsite-key")
+
+
+_DAMAGED = ("the off-site key stored on this computer is damaged, so FlowState can't use it. "
+            "It was not replaced (that would lose your earlier copies); nothing was {doing}. If "
+            "you saved its recovery code, you can still restore those copies with it.")
+
+
+def _empty(store, doing: str = "pushed") -> str:
+    where = {"macOS Keychain": "the Keychain item flowstate-offsite-key (Keychain Access)",
+             "Windows Credential Manager": "the Credential Manager item flowstate:offsite-key"}
+    how = where.get(getattr(store, "name", ""), f"the file {getattr(store, 'path', 'offsite-key')}")
+    return (f"the off-site key on this computer is empty: {how} exists but holds no key, so "
+            f"FlowState can't use it. It was not replaced; nothing was {doing}. If you never "
+            f"pushed an encrypted off-site copy, delete that empty item and push again (a new "
+            f"key is made then). If you did, keep it and restore those copies with the "
+            f"recovery code you saved (task 56 Low).")
+
+
+def _decode(text: str, doing: str = "pushed") -> bytes:
+    try:
+        t = text.strip()
+        raw = base64.urlsafe_b64decode(t + "=" * (-len(t) % 4))
+    except (ValueError, TypeError):
+        raise CryptoError(_DAMAGED.format(doing=doing)) from None
+    if len(raw) != 32:
+        raise CryptoError(_DAMAGED.format(doing=doing))
+    return raw
+
+
+def get_key(store=None, doing: str = "pushed") -> Optional[bytes]:
+    """The key, or None only when the store says there is none. A store that can't be read
+    raises CryptoError (never "no key"). `doing`: what didn't happen, for its message
+    ("pushed"; "restored" for a restore, "set up" for offsite setup: task 56 Info)."""
+    store = store or _store()
+    try:
+        v = store.get()
+    except Exception as e:  # noqa: BLE001 - locked Keychain, denied prompt, unreadable file
+        raise CryptoError(
+            f"couldn't read the off-site key from {getattr(store, 'name', 'the credential store')} "
+            f"({e if str(e) else type(e).__name__}). Nothing was {doing} and no new key was "
+            f"made; unlock it (or allow FlowState) and try again.") from None
+    if v is None:
+        return None
+    if not str(v).strip():   # the item is there but empty: not "no key", never replaced
+        raise CryptoError(_empty(store, doing))
+    return _decode(v, doing)
+
+
+def get_or_create_key(store=None) -> tuple:
+    """(key, made_now). A new key only after a definite "not found", and only with an
+    add-only write: an existing item is never overwritten."""
+    store = store or _store()
+    key = get_key(store)
+    if key is not None:
+        return key, False
+    # one first-time key at a time on this computer: Credential Manager has no add-only write,
+    # so two pushes at the same moment could each make one, and the first copy's key be lost
+    from . import paths
+    from .privatefs import private_dir
+    private_dir(paths.config_dir())
+    try:
+        with paths._lock(paths.config_dir() / "offsite-key"):
+            return _create_key(store)
+    except TimeoutError:
+        raise CryptoError("another FlowState is making the off-site key right now; nothing was "
+                          "pushed. Try again in a moment.") from None
+
+
+def _create_key(store) -> tuple:
+    from . import secret_store as SS
+    key = get_key(store)   # made by another FlowState while this one waited
+    if key is not None:
+        return key, False
+    key = secrets.token_bytes(32)
+    text = base64.urlsafe_b64encode(key).decode().rstrip("=")
+    try:
+        if hasattr(store, "create"):
+            store.create(text)
+        else:   # a store without an add-only write: check right before writing
+            if store.get() is not None:
+                raise SS.AlreadyStored("an off-site key appeared meanwhile; it was not replaced")
+            store.set(text)
+    except SS.AlreadyStored:
+        raise CryptoError("an off-site key was already stored (made by another FlowState at "
+                          "the same moment?); it was not replaced. Nothing was pushed; try "
+                          "again.") from None
+    except CryptoError:
+        raise
+    except Exception as e:  # noqa: BLE001
+        raise CryptoError(f"the credential store refused the new off-site key "
+                          f"({type(e).__name__}); nothing was pushed") from None
+    if get_key(store) != key:
+        raise CryptoError("the credential store didn't keep the new off-site key; nothing was "
+                          "pushed")
+    return key, True
+
+
+def recovery_code(key: bytes) -> str:
+    """The key for people: 52 letters and digits in groups of four."""
+    b32 = base64.b32encode(key).decode().rstrip("=")
+    return "-".join(b32[i:i + 4] for i in range(0, len(b32), 4))
+
+
+def key_from_code(code: str) -> bytes:
+    s = "".join(ch for ch in (code or "").upper() if ch.isalnum())
+    try:
+        key = base64.b32decode(s + "=" * (-len(s) % 8))
+    except ValueError:
+        raise CryptoError("that recovery code isn't valid (letters A-Z and digits 2-7)") from None
+    if len(key) != 32:
+        raise CryptoError("that recovery code is the wrong length")
+    return key
+
+
+# ---------------------------------------------------------------- files
+def _aad(header: bytes, n: int, last: bool) -> bytes:
+    return header + struct.pack(">I?", n, last)
+
+
+def encrypt_file(src: Path, dst: Path, key: bytes, created: Optional[int] = None) -> None:
+    """`created`: when the copy was made (Unix seconds; default now), sealed in the header."""
+    import time
+    aes = _aes(key)
+    prefix = secrets.token_bytes(8)
+    when = int(time.time()) if created is None else int(created)
+    header = (MAGIC2 + bytes.fromhex(key_id(key)) + prefix + struct.pack(">I", CHUNK)
+              + struct.pack(">Q", when))
+    from .privatefs import private_file
+    private_file(dst)
+    with open(src, "rb") as f, open(dst, "wb") as out:
+        out.write(header)
+        n, chunk = 0, f.read(CHUNK)
+        while True:
+            nxt = f.read(CHUNK)
+            last = not nxt
+            out.write(aes.encrypt(prefix + struct.pack(">I", n), chunk, _aad(header, n, last)))
+            if last:
+                break
+            n, chunk = n + 1, nxt
+
+
+def is_encrypted(path: Path) -> bool:
+    try:
+        with open(path, "rb") as f:
+            return f.read(len(MAGIC)) in (MAGIC, MAGIC2)
+    except OSError:
+        return False
+
+
+def file_key_id(path: Path) -> str:
+    with open(path, "rb") as f:
+        head = f.read(_HEADER)
+    return head[len(MAGIC):len(MAGIC) + 8].hex()
+
+
+def decrypt_file(src: Path, dst: Path, key: bytes) -> dict:
+    """Decrypt into `dst` (private). Returns {"created": Unix seconds the copy was made, from
+    the sealed header, or None for a format-1 file}; only trustworthy because it returned."""
+    from cryptography.exceptions import InvalidTag
+    aes = _aes(key)
+    with open(src, "rb") as f:
+        header = f.read(_HEADER)
+        magic = header[:len(MAGIC)]
+        if len(header) != _HEADER or magic not in (MAGIC, MAGIC2):
+            raise CryptoError("not a FlowState encrypted backup")
+        created = None
+        if magic == MAGIC2:
+            extra = f.read(8)
+            if len(extra) != 8:
+                raise CryptoError("this encrypted copy is cut short or damaged (its header is "
+                                  "incomplete); nothing was restored")
+            header += extra
+            created = struct.unpack(">Q", extra)[0]
+        if header[len(MAGIC):len(MAGIC) + 8].hex() != key_id(key):
+            raise CryptoError("this backup was encrypted with a different key (another "
+                              "computer's, or one from before a key reset): use its recovery code")
+        prefix = header[len(MAGIC) + 8:len(MAGIC) + 16]
+        size = struct.unpack(">I", header[len(MAGIC) + 16:len(MAGIC) + 20])[0] + 16
+        from .privatefs import private_file
+        dst.unlink(missing_ok=True)
+        private_file(dst)
+        try:
+            with open(dst, "wb") as out:
+                n, chunk = 0, f.read(size)
+                if not chunk:   # cut down to its header: not an empty database
+                    raise CryptoError("this encrypted copy is cut short or damaged (it has no "
+                                      "data after its header); nothing was restored")
+                while chunk:
+                    nxt = f.read(size)
+                    try:
+                        out.write(aes.decrypt(prefix + struct.pack(">I", n), chunk,
+                                              _aad(header, n, not nxt)))
+                    except InvalidTag:
+                        raise CryptoError("the encrypted backup is damaged or was changed "
+                                          "(its check failed); nothing was restored") from None
+                    n, chunk = n + 1, nxt
+        except Exception:
+            dst.unlink(missing_ok=True)
+            raise
+    return {"created": created}

flowstate/paths.py

@@ -0,0 +1,265 @@
+"""Per-OS folders FlowState uses besides the database itself. Standard library only.
+
+    backups:  FLOWSTATE_BACKUP_DIR, else Windows %LOCALAPPDATA%\\flowstate\\backups,
+              macOS ~/Library/Application Support/flowstate/backups,
+              Linux ~/.local/share/flowstate/backups
+    config:   FLOWSTATE_CONFIG_DIR, else Windows %APPDATA%\\flowstate,
+              macOS ~/Library/Application Support/flowstate, Linux ~/.config/flowstate
+
+None of these is inside the (possibly git-synced) data folder unless the user says so.
+"""
+
+import json
+import os
+import platform
+import threading
+from pathlib import Path
+from typing import Optional
+
+
+def _system() -> str:
+    return platform.system()
+
+
+def data_root(home: Optional[Path] = None, system: Optional[str] = None) -> Path:
+    system, h = system or _system(), Path(home) if home else Path.home()
+    if system == "Windows":
+        base = Path(os.environ["LOCALAPPDATA"]) if (not home and os.environ.get("LOCALAPPDATA")) \
+            else h / "AppData" / "Local"
+        return base / "flowstate"
+    if system == "Darwin":
+        return h / "Library" / "Application Support" / "flowstate"
+    return h / ".local" / "share" / "flowstate"
+
+
+def backup_dir(home: Optional[Path] = None, system: Optional[str] = None) -> Path:
+    if not home and os.environ.get("FLOWSTATE_BACKUP_DIR"):
+        return Path(os.environ["FLOWSTATE_BACKUP_DIR"]).expanduser()
+    return data_root(home, system) / "backups"
+
+
+def config_dir(home: Optional[Path] = None, system: Optional[str] = None) -> Path:
+    if not home and os.environ.get("FLOWSTATE_CONFIG_DIR"):
+        return Path(os.environ["FLOWSTATE_CONFIG_DIR"]).expanduser()
+    system, h = system or _system(), Path(home) if home else Path.home()
+    if system == "Windows":
+        base = Path(os.environ["APPDATA"]) if (not home and os.environ.get("APPDATA")) \
+            else h / "AppData" / "Roaming"
+        return base / "flowstate"
+    if system == "Darwin":
+        return h / "Library" / "Application Support" / "flowstate"
+    return h / ".config" / "flowstate"
+
+
+# ---------------------------------------------------------------- settings file
+def config_file() -> Path:
+    return config_dir() / "config.json"
+
+
+class ConfigUnreadable(Exception):
+    """config.json exists but isn't a JSON object FlowState can read (a hand-edit typo, or a
+    file another program is half-way through writing)."""
+
+    def __init__(self, path: Path):
+        super().__init__(f"FlowState's settings file {path} can't be read (it isn't valid JSON). "
+                         f"Nothing was changed. Fix or delete that file, then try again.")
+        self.path = path
+
+
+def _read_config(p: Path) -> dict:
+    """The settings, {} when there is no file; ConfigUnreadable when there is one that doesn't
+    parse. On Windows a reader can be refused for a moment while a writer replaces the file,
+    so a refused read is tried again briefly first."""
+    import time
+    deadline = time.monotonic() + 1.0
+    while True:
+        try:
+            text = p.read_text(encoding="utf-8")
+            break
+        except FileNotFoundError:
+            return {}
+        except PermissionError:
+            if time.monotonic() > deadline:
+                raise ConfigUnreadable(p) from None
+            time.sleep(0.02)
+        except OSError:
+            raise ConfigUnreadable(p) from None
+    try:
+        data = json.loads(text)
+    except ValueError:
+        raise ConfigUnreadable(p) from None
+    if not isinstance(data, dict):
+        raise ConfigUnreadable(p)
+    return data
+
+
+def load_config(strict: bool = False) -> dict:
+    """FlowState's own settings (no secrets ever go in here). A file that can't be read counts
+    as no settings, unless `strict` (then ConfigUnreadable): a setting that must fail safe
+    (updates.self_update_allowed, the name sent to the relay) and every writer use strict."""
+    try:
+        return _read_config(config_file())
+    except ConfigUnreadable:
+        if strict:
+            raise
+        return {}
+
+
+def replace_file(tmp: Path, p: Path, wait: float = 2.0) -> None:
+    """os.replace(tmp, p), the last step of an atomic write. On Windows the replace is refused
+    (access denied) while another process has `p` open, e.g. `flowstate relay status` reading
+    the file the relay agent rewrites; readers hold it for moments, so try again briefly."""
+    import time
+    deadline = time.monotonic() + wait
+    while True:
+        try:
+            os.replace(tmp, p)
+            return
+        except PermissionError:
+            if os.name != "nt" or time.monotonic() > deadline:
+                raise
+            time.sleep(0.02)
+
+
+def save_config(data: dict) -> Path:
+    """Write the whole settings file atomically (a temp file of its own, then replace). Use
+    update_config to change one setting: it keeps the others."""
+    import secrets
+    from .privatefs import private_dir, write_private
+    p = config_file()
+    private_dir(p.parent)
+    tmp = p.with_name(f"{p.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp")
+    try:
+        write_private(tmp, json.dumps(data, indent=2) + "\n")   # owner-only (audit D9)
+        replace_file(tmp, p)
+    finally:
+        if tmp.exists():
+            tmp.unlink()
+    return p
+
+
+LOCK_WAIT = 5.0      # seconds a settings writer waits for another one
+
+
+def _try_lock(fd: int) -> bool:
+    """An OS lock on the open lock file, without waiting: True if we have it, False if another
+    process (or thread) holds it. The OS drops it when the holder exits, so a writer that
+    crashed never blocks the next one. A filesystem without locks counts as "have it"."""
+    import errno
+    try:
+        if os.name == "nt":
+            import msvcrt
+            os.lseek(fd, 0, os.SEEK_SET)
+            msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
+        else:
+            import fcntl
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return True
+    except OSError as e:
+        if e.errno in (errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK, errno.EDEADLK):
+            return False
+        return True   # e.g. ENOLCK / ENOTSUP on an odd filesystem: go ahead as before 2.4.4
+
+
+def _unlock(fd: int) -> None:
+    try:
+        if os.name == "nt":
+            import msvcrt
+            os.lseek(fd, 0, os.SEEK_SET)
+            msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
+        else:
+            import fcntl
+            fcntl.flock(fd, fcntl.LOCK_UN)
+    except OSError:
+        pass
+
+
+_thread_lock = threading.Lock()   # threads of one process take turns too (and wait less)
+
+
+def _quiet_unlink(f: Path) -> None:
+    try:
+        f.unlink()
+    except OSError:
+        pass
+
+
+def _lock(p: Path):
+    """An OS file lock (flock / msvcrt) on a lock file next to config.json, held while one
+    writer reads, changes and writes the settings, so two writers at once can't lose each
+    other's change. Nothing to go stale: the OS releases it when its holder exits (audit
+    N-F23: a lock left by a crashed writer no longer blocks for 30 s, and two waiting
+    writers can't both "break" one). After locking, the file must still be the one at the
+    path (the last holder removes it when done), else it starts over."""
+    import contextlib
+    import time
+
+    @contextlib.contextmanager
+    def held():
+        lock = p.with_name(p.name + ".lock")
+        deadline = time.monotonic() + LOCK_WAIT
+
+        def busy():
+            raise TimeoutError(f"Another FlowState command is changing {p}; try again "
+                               f"in a moment.") from None
+        if not _thread_lock.acquire(timeout=LOCK_WAIT):
+            busy()
+        try:
+            while True:
+                # Windows refuses the open (and the stat) while the last holder's removal of
+                # the file is still going through: a busy lock, not an error.
+                try:
+                    fd = os.open(str(lock), os.O_CREAT | os.O_RDWR, 0o600)
+                except PermissionError:
+                    fd = None
+                if fd is not None:
+                    if _try_lock(fd):
+                        try:
+                            same = os.path.samestat(os.fstat(fd), os.stat(str(lock)))
+                        except (FileNotFoundError, PermissionError):
+                            same = False
+                        if same:
+                            break
+                        _unlock(fd)   # its holder removed it as we opened it: take the new one
+                    os.close(fd)
+                if time.monotonic() > deadline:
+                    busy()
+                time.sleep(0.02)
+            try:
+                yield
+            finally:
+                # POSIX: remove it while still holding it, so whoever has it open next sees a
+                # file that is no longer at the path and starts over. Windows can't remove an
+                # open file: unlock and close first; if another writer opened it meanwhile, the
+                # removal is refused and that writer keeps the same file.
+                if os.name != "nt":
+                    _quiet_unlink(lock)
+                _unlock(fd)
+                os.close(fd)
+                if os.name == "nt":
+                    _quiet_unlink(lock)
+        finally:
+            _thread_lock.release()
+    return held()
+
+
+def update_config(change) -> Path:
+    """Change the settings: `change(cfg)` edits the dict in place, under the lock, on a strict
+    read (a file that exists but can't be read raises ConfigUnreadable and is left as it is,
+    so one typo can't wipe the other settings)."""
+    from .privatefs import private_dir
+    p = config_file()
+    private_dir(p.parent)   # owner-only when FlowState makes it (audit D9)
+    with _lock(p):
+        cfg = _read_config(p)
+        change(cfg)
+        return save_config(cfg)
+
+
+def inside_git_worktree(path: Path) -> Optional[Path]:
+    """The git work tree containing `path`, if any (checks for a .git entry upwards)."""
+    p = Path(path).resolve()
+    for d in [p] + list(p.parents):
+        if (d / ".git").exists():
+            return d
+    return None