rssd-fs 0.1.0__py3-none-any.whl

rssd/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

rssd/cli.py

@@ -0,0 +1,278 @@
+"""Command line surface."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import shutil
+import sys
+from pathlib import Path
+
+from .config import Config, Limits
+from .models import Subscription
+
+#: The reference feeds from SPEC §14. Between them they exercise every code
+#: path: high churn, rich Atom content, thin RSS, an image-only body, and one
+#: feed that sends no cache validators at all.
+REFERENCE_FEEDS: list[tuple[str, str, str]] = [
+    ("lobsters", "https://lobste.rs/rss", "high churn, ETag + Last-Modified"),
+    ("rust-blog", "https://blog.rust-lang.org/feed.xml", "Atom, full content, xml:base"),
+    ("simonw", "https://simonwillison.net/atom/everything/", "rich escaped HTML"),
+    ("hn", "https://hnrss.org/frontpage", "title + link only, no body"),
+    ("xkcd", "https://xkcd.com/rss.xml", "body is a single image"),
+    ("godev", "https://go.dev/blog/feed.atom", "no cache validators at all"),
+]
+
+MUTABLE_FIXTURE = """<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0">
+  <channel>
+    <title>rssd demo feed</title>
+    <link>http://127.0.0.1:8765/</link>
+    <description>A feed you can edit on purpose.</description>
+    <item>
+      <title>The entry that gets revised</title>
+      <link>http://127.0.0.1:8765/demo/1</link>
+      <guid isPermaLink="false">rssd-demo-1</guid>
+      <pubDate>Mon, 15 Sep 2026 12:00:00 GMT</pubDate>
+      <description>&lt;p&gt;Revision {n}. Every version is kept on disk.&lt;/p&gt;</description>
+    </item>
+  </channel>
+</rss>
+"""
+
+
+def subscription_xml(name: str, url: str, comment: str, *, fulltext: bool = False) -> str:
+    return (
+        '<?xml version="1.0" encoding="utf-8"?>\n'
+        f"<!-- {comment} -->\n"
+        "<subscription>\n"
+        f"  <url>{url}</url>\n"
+        f"  <name>{name}</name>\n"
+        f"  <fulltext>{'true' if fulltext else 'false'}</fulltext>\n"
+        "</subscription>\n"
+    )
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    root = Path(args.root)
+    config = Config(root=root)
+    config.ensure_dirs()
+
+    base = args.fixture_base.rstrip("/") if args.fixture_mode else None
+    for name, url, comment in REFERENCE_FEEDS:
+        target = config.feeds_d / f"{name}.xml"
+        if target.exists() and not args.force:
+            continue
+        effective = f"{base}/{name}.xml" if base else url
+        target.write_text(subscription_xml(name, effective, comment))
+
+    fixtures = root / "fixtures"
+    fixtures.mkdir(parents=True, exist_ok=True)
+    mutable = fixtures / "mutable.xml"
+    if not mutable.exists():
+        mutable.write_text(MUTABLE_FIXTURE.format(n=1))
+    if args.fixture_mode:
+        src = Path(__file__).resolve().parent.parent.parent / "tests" / "fixtures" / "feeds"
+        for name, _, _ in REFERENCE_FEEDS:
+            candidate = src / f"{name}.xml"
+            if candidate.exists():
+                shutil.copy2(candidate, fixtures / f"{name}.xml")
+        target = config.feeds_d / "mutable.xml"
+        if not target.exists():
+            target.write_text(
+                subscription_xml("mutable", f"{base}/mutable.xml", "editable demo feed")
+            )
+
+    print(f"initialised {root}")
+    print(f"  feeds.d/   {len(list(config.feeds_d.glob('*.xml')))} subscriptions")
+    print(f"  store/     (empty until first poll)")
+    print(f"  var/       events.jsonl appears on first run")
+    print()
+    print(f"next:  rssd daemon --root {root} --poll-now")
+    return 0
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    from .subscriptions import load_dir
+
+    config = _config(args)
+    subs, errors = load_dir(config.feeds_d)
+    for name in sorted(subs):
+        sub = subs[name]
+        extra = "  [fulltext]" if sub.fulltext else ""
+        print(f"  ok      {name:<14} {sub.url}{extra}")
+    for path, message in errors:
+        print(f"  INVALID {path.name:<14} {message}")
+    print(f"\n{len(subs)} valid, {len(errors)} invalid")
+    return 1 if errors else 0
+
+
+def cmd_prune(args: argparse.Namespace) -> int:
+    """Delete retired feed folders. Explicit, never automatic.
+
+    The daemon itself will not remove harvested data under any circumstance;
+    that decision belongs to a human who typed this command.
+    """
+    from .state import load_state
+
+    config = _config(args)
+    removed = 0
+    for feed_dir in sorted(config.store.iterdir()) if config.store.exists() else []:
+        if not feed_dir.is_dir():
+            continue
+        state = load_state(config, feed_dir.name)
+        if state.health != "retired":
+            continue
+        if not args.yes:
+            print(f"  would remove {feed_dir}")
+            removed += 1
+            continue
+        shutil.rmtree(feed_dir)
+        config.state_path(feed_dir.name).unlink(missing_ok=True)
+        print(f"  removed {feed_dir}")
+        removed += 1
+    if removed and not args.yes:
+        print(f"\n{removed} retired feed(s). Re-run with --yes to delete.")
+    elif not removed:
+        print("nothing to prune")
+    return 0
+
+
+def cmd_serve_fixtures(args: argparse.Namespace) -> int:
+    from . import fixtures
+
+    directory = Path(args.dir) if args.dir else fixtures.FIXTURE_DIR
+    server = fixtures.serve(args.port, directory)
+    print(f"serving {directory} on http://127.0.0.1:{args.port}")
+    for path in sorted(directory.glob("*.xml")):
+        print(f"  /{path.name}")
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.shutdown()
+    return 0
+
+
+def cmd_demo_mutate(args: argparse.Namespace) -> int:
+    """Force a revision on demand.
+
+    Waiting for a real publisher to fix a typo is not a demo. This edits the
+    served fixture so the next poll sees changed content for an entry it has
+    already stored, producing a .r2 file and an entry.revised event.
+    """
+    import re
+
+    path = Path(args.root) / "fixtures" / "mutable.xml"
+    if not path.exists():
+        print(f"rssd: {path} not found; run `rssd init --fixture-mode` first")
+        return 1
+    text = path.read_text()
+    match = re.search(r"Revision (\d+)\.", text)
+    n = int(match.group(1)) + 1 if match else 2
+    path.write_text(MUTABLE_FIXTURE.format(n=n))
+    print(f"mutable.xml now at revision {n} — next poll will write .r{n}.xml")
+    return 0
+
+
+def cmd_daemon(args: argparse.Namespace) -> int:
+    from .daemon import run_daemon
+
+    return asyncio.run(run_daemon(_config(args)))
+
+
+def cmd_once(args: argparse.Namespace) -> int:
+    from .daemon import run_once
+
+    return asyncio.run(run_once(_config(args)))
+
+
+def cmd_poll(args: argparse.Namespace) -> int:
+    from .daemon import poll_one
+
+    return asyncio.run(poll_one(_config(args), args.feed))
+
+
+def cmd_tui(args: argparse.Namespace) -> int:
+    try:
+        from .tui.app import run_tui
+    except ImportError:
+        print("rssd: the TUI needs the [tui] extra — run `uv sync --extra tui`")
+        return 1
+    return run_tui(_config(args))
+
+
+def _config(args: argparse.Namespace) -> Config:
+    return Config(
+        root=Path(args.root),
+        limits=Limits(),
+        poll_now=getattr(args, "poll_now", False),
+        fixture_mode=getattr(args, "fixture_mode", False),
+        no_fsync=getattr(args, "no_fsync", False),
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="rssd", description="A file-based RSS daemon. The filesystem is the API."
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def with_root(p: argparse.ArgumentParser) -> argparse.ArgumentParser:
+        p.add_argument("--root", default=".", help="instance root directory")
+        return p
+
+    p_init = sub.add_parser("init", help="scaffold a new instance")
+    p_init.add_argument("root", nargs="?", default=".")
+    p_init.add_argument("--force", action="store_true", help="overwrite existing subscriptions")
+    p_init.add_argument("--fixture-mode", action="store_true",
+                        help="point subscriptions at the local fixture server")
+    p_init.add_argument("--fixture-base", default="http://127.0.0.1:8765")
+    p_init.set_defaults(func=cmd_init)
+
+    p_daemon = with_root(sub.add_parser("daemon", help="run the daemon"))
+    p_daemon.add_argument("--poll-now", action="store_true", help="skip the startup stagger")
+    p_daemon.add_argument("--fixture-mode", action="store_true")
+    p_daemon.add_argument("--no-fsync", action="store_true", help="unsafe; for slow filesystems")
+    p_daemon.set_defaults(func=cmd_daemon)
+
+    p_once = with_root(sub.add_parser("once", help="one poll pass, then exit"))
+    p_once.add_argument("--no-fsync", action="store_true")
+    p_once.set_defaults(func=cmd_once)
+
+    p_poll = with_root(sub.add_parser("poll", help="force-poll a single feed"))
+    p_poll.add_argument("feed")
+    p_poll.set_defaults(func=cmd_poll)
+
+    with_root(sub.add_parser("tui", help="terminal visualiser")).set_defaults(func=cmd_tui)
+    with_root(sub.add_parser("validate", help="check every subscription parses")).set_defaults(
+        func=cmd_validate
+    )
+
+    p_prune = with_root(sub.add_parser("prune", help="delete retired feed folders"))
+    p_prune.add_argument("--retired", action="store_true", default=True)
+    p_prune.add_argument("--yes", action="store_true", help="actually delete")
+    p_prune.set_defaults(func=cmd_prune)
+
+    p_serve = sub.add_parser("serve-fixtures", help="local HTTP server over fixtures")
+    p_serve.add_argument("--port", type=int, default=8765)
+    p_serve.add_argument("--dir", default=None)
+    p_serve.set_defaults(func=cmd_serve_fixtures)
+
+    p_mutate = with_root(sub.add_parser("demo-mutate", help="force a revision on demand"))
+    p_mutate.set_defaults(func=cmd_demo_mutate)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rssd/config.py

@@ -0,0 +1,151 @@
+"""Root paths, defaults and limits.
+
+Every path in rssd is derived from a single root, so a whole instance is one
+directory you can move, tar, or throw away.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+#: Feed folder names double as directory names and appear in every event, so
+#: they are kept boring on purpose.
+NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
+
+USER_AGENT = "rssd/0.1 (+https://github.com/allie/rssd)"
+
+ACCEPT = (
+    "application/atom+xml, application/rss+xml, application/rdf+xml;q=0.9, "
+    "application/xml;q=0.8, text/xml;q=0.7, */*;q=0.1"
+)
+
+#: XML namespace for entry documents.
+NS = "https://rssd.dev/entry/1"
+
+#: Temp files carry BOTH a dot prefix and a non-.xml suffix, so that neither
+#: `ls`, nor a `*.xml` glob, nor `find -name '*.xml'` can ever see one.
+TMP_PREFIX = ".rssd-tmp-"
+
+
+@dataclass(frozen=True, slots=True)
+class Limits:
+    #: Interval bounds. Nothing polls faster than a minute or slower than 6h.
+    min_interval: int = 60
+    max_interval: int = 6 * 3600
+    default_interval: int = 900
+    #: Feeds that send no ETag/Last-Modified can't be checked cheaply, so they
+    #: are checked less often (SPEC §10.4).
+    no_validator_floor: int = 1800
+
+    #: A broken feed must not be able to write ten thousand files mid-demo.
+    max_new_entries_per_poll: int = 100
+    max_revisions_per_entry_per_day: int = 8
+    max_tracked_entries: int = 50_000
+
+    #: Rotating-guid detection thresholds (SPEC §9.4).
+    churn_new_ratio: float = 0.8
+    churn_hash_overlap: float = 0.5
+    churn_min_polls: int = 2
+
+    max_body_bytes: int = 5 * 1024 * 1024
+    request_timeout: float = 30.0
+    max_redirects: int = 5
+
+    global_concurrency: int = 6
+    per_host_concurrency: int = 2
+
+    max_failures_before_failing: int = 10
+    backoff_cap: int = 6 * 3600
+
+    event_log_max_bytes: int = 16 * 1024 * 1024
+    event_log_keep: int = 10
+
+    #: Startup stagger so the first minute isn't a thundering herd.
+    stagger_step: float = 0.75
+    stagger_cap: float = 30.0
+
+    #: Debounce for feeds.d/ changes, and the grace window that stops a vim
+    #: `:w` (unlink-then-create) from retiring a feed.
+    watch_debounce: float = 0.3
+    unlink_grace: float = 5.0
+
+    slug_max_len: int = 48
+    #: Entries dated further ahead than this are clamped (SPEC §4.2).
+    future_clamp: int = 24 * 3600
+
+
+@dataclass(frozen=True, slots=True)
+class Config:
+    root: Path
+    limits: Limits = field(default_factory=Limits)
+    #: Collapse the startup stagger; for demos.
+    poll_now: bool = False
+    #: Point every subscription at the local fixture server instead of the
+    #: real internet. Demo insurance.
+    fixture_mode: bool = False
+    fixture_base: str = "http://127.0.0.1:8765"
+    #: Escape hatch for slow filesystems; never use in production.
+    no_fsync: bool = False
+
+    # ── derived paths ────────────────────────────────────────────────────
+
+    @property
+    def feeds_d(self) -> Path:
+        return self.root / "feeds.d"
+
+    @property
+    def store(self) -> Path:
+        return self.root / "store"
+
+    @property
+    def var(self) -> Path:
+        return self.root / "var"
+
+    @property
+    def state_dir(self) -> Path:
+        return self.var / "state"
+
+    @property
+    def events_path(self) -> Path:
+        return self.var / "events.jsonl"
+
+    @property
+    def seq_path(self) -> Path:
+        return self.var / "seq"
+
+    @property
+    def lock_path(self) -> Path:
+        return self.var / "rssd.lock"
+
+    def feed_dir(self, name: str) -> Path:
+        return self.store / name
+
+    def entries_dir(self, name: str) -> Path:
+        return self.store / name / "entries"
+
+    def feed_xml(self, name: str) -> Path:
+        return self.store / name / "feed.xml"
+
+    def status_xml(self, name: str) -> Path:
+        return self.store / name / "status.xml"
+
+    def state_path(self, name: str) -> Path:
+        return self.state_dir / f"{name}.json"
+
+    def relative(self, path: Path) -> str:
+        """Paths in events are always relative to the root, so a consumer can
+        resolve them regardless of where the instance lives."""
+        try:
+            return str(path.relative_to(self.root))
+        except ValueError:
+            return str(path)
+
+    def ensure_dirs(self) -> None:
+        for d in (self.feeds_d, self.store, self.var, self.state_dir):
+            d.mkdir(parents=True, exist_ok=True)
+
+
+def valid_name(name: str) -> bool:
+    return bool(NAME_RE.match(name))

rssd/daemon.py

@@ -0,0 +1,168 @@
+"""Process lifecycle: the lock, the signals, and the recovery pass."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import fcntl
+import os
+import signal
+from pathlib import Path
+
+from .config import Config
+from .events import EventLog
+from .scheduler import Scheduler
+from .store import sweep_temps
+from .watcher import Reconciler, watch
+
+
+class LockedError(RuntimeError):
+    pass
+
+
+class RootLock:
+    """One daemon per root.
+
+    Two daemons sharing a tree would interleave revisions and fight over state
+    files, and the damage would be silent. An exclusive flock is cheap enough
+    that there is no reason not to.
+    """
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._fd: int | None = None
+
+    def acquire(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o644)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as exc:
+            os.close(fd)
+            raise LockedError(
+                f"another rssd daemon holds {self.path}; refusing to start"
+            ) from exc
+        os.ftruncate(fd, 0)
+        os.write(fd, f"{os.getpid()}\n".encode())
+        os.fsync(fd)
+        self._fd = fd
+
+    def release(self) -> None:
+        if self._fd is not None:
+            with contextlib.suppress(OSError):
+                fcntl.flock(self._fd, fcntl.LOCK_UN)
+                os.close(self._fd)
+            self._fd = None
+        with contextlib.suppress(OSError):
+            self.path.unlink()
+
+    def __enter__(self) -> RootLock:
+        self.acquire()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.release()
+
+
+async def run_daemon(config: Config) -> int:
+    config.ensure_dirs()
+    lock = RootLock(config.lock_path)
+    try:
+        lock.acquire()
+    except LockedError as exc:
+        print(f"rssd: {exc}")
+        return 1
+
+    events = EventLog(config)
+    events.open()
+    stop = asyncio.Event()
+
+    try:
+        # A SIGKILL mid-write can only ever leave temp files behind; a real
+        # entry path is never partial. Clear them before anything reads the tree.
+        swept = sweep_temps(config.store)
+        events.emit("daemon.started", None, pid=os.getpid(), root=str(config.root),
+                    swept_temps=swept)
+
+        scheduler = Scheduler(config, events)
+        reconciler = Reconciler(config=config, scheduler=scheduler, events=events)
+        reconciler.reconcile(initial=True)
+
+        tracked = sum(len(r.seen) for r in scheduler.runners.values())
+        events.emit(
+            "daemon.reconciled", None,
+            feeds=len(scheduler.runners), entries=tracked,
+        )
+
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, stop.set)
+
+        tasks = [
+            asyncio.create_task(scheduler.run(), name="scheduler"),
+            asyncio.create_task(watch(config, reconciler, stop), name="watcher"),
+            asyncio.create_task(stop.wait(), name="stop"),
+        ]
+        await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+
+        events.emit("daemon.stopping", None)
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+        await scheduler.stop()
+        return 0
+    finally:
+        events.close()
+        lock.release()
+
+
+async def run_once(config: Config) -> int:
+    """One poll pass over every subscription, then exit.
+
+    Useful in cron-shaped deployments, and it is what the integration tests
+    drive so they never have to reason about timers.
+    """
+    config.ensure_dirs()
+    events = EventLog(config)
+    events.open()
+    try:
+        sweep_temps(config.store)
+        events.emit("daemon.started", None, pid=os.getpid(), mode="once")
+        scheduler = Scheduler(config, events)
+        reconciler = Reconciler(config=config, scheduler=scheduler, events=events)
+        reconciler.reconcile(initial=True)
+        runners = list(scheduler.runners.values())
+        sem = asyncio.Semaphore(config.limits.global_concurrency)
+
+        async def _one(runner):
+            async with sem:
+                try:
+                    await runner.poll()
+                except Exception as exc:
+                    events.emit("feed.poll-failed", runner.name, error=f"internal: {exc!r}")
+
+        await asyncio.gather(*(_one(r) for r in runners))
+        events.emit("daemon.stopping", None)
+        await scheduler.client.aclose()
+        return 0
+    finally:
+        events.close()
+
+
+async def poll_one(config: Config, name: str) -> int:
+    config.ensure_dirs()
+    events = EventLog(config)
+    events.open()
+    try:
+        scheduler = Scheduler(config, events)
+        reconciler = Reconciler(config=config, scheduler=scheduler, events=events)
+        reconciler.reconcile(initial=True)
+        runner = scheduler.runners.get(name)
+        if runner is None:
+            print(f"rssd: no such feed: {name}")
+            return 1
+        await runner.poll()
+        await scheduler.client.aclose()
+        return 0
+    finally:
+        events.close()