traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/sources/file_poll.py

@@ -0,0 +1,136 @@
+"""File polling source for append-only log files."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from pathlib import Path
+from types import TracebackType
+from typing import Literal
+
+from traceforge.sources.base import RawRecord, Source
+
+logger = logging.getLogger(__name__)
+
+
+class FilePollSource(Source):
+    """Poll a file for new appended content at a fixed interval.
+
+    Detects truncation/rotation via size comparison and reopens from the
+    beginning. Suitable for files that are not watched via OS events (e.g.
+    network mounts where inotify is unavailable).
+    """
+
+    def __init__(
+        self,
+        path: str | Path,
+        name: str,
+        interval: float = 2.0,
+        encoding: str = "utf-8",
+        missing: Literal["wait", "error"] = "wait",
+    ) -> None:
+        if interval < 0:
+            raise ValueError("interval must be non-negative")
+        self.path = Path(path).resolve()
+        self.name = name
+        self.interval = interval
+        self.encoding = encoding
+        self.missing = missing
+        self._offset = 0
+        self._buffer = ""
+        self._sequence = 0
+        self._iterating = False
+        self._inode: int | None = None
+
+    async def __aenter__(self) -> "FilePollSource":
+        if self.missing == "error" and not self.path.exists():
+            raise FileNotFoundError(f"FilePollSource target does not exist: {self.path}")
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self._iterating = False
+
+    async def _iter_records(self) -> AsyncIterator[RawRecord]:
+        if self._iterating:
+            raise RuntimeError("FilePollSource does not support concurrent iteration")
+        self._iterating = True
+        try:
+            while True:
+                async for record in self._poll_once():
+                    yield record
+                await asyncio.sleep(self.interval)
+        finally:
+            self._iterating = False
+
+    def __aiter__(self) -> AsyncIterator[RawRecord]:
+        return self._iter_records()
+
+    async def _poll_once(self) -> AsyncIterator[RawRecord]:
+        try:
+            stat = await asyncio.to_thread(self.path.stat)
+        except (FileNotFoundError, OSError):
+            return
+
+        current_inode = self._get_inode(stat)
+        rotated = self._inode is not None and current_inode != self._inode
+        truncated = stat.st_size < self._offset
+
+        if rotated or truncated:
+            logger.info("FilePollSource %s: file rotated/truncated, resetting", self.name)
+            self._offset = 0
+            self._buffer = ""
+        self._inode = current_inode
+
+        try:
+            text = await asyncio.to_thread(self._read_from_offset)
+        except (FileNotFoundError, OSError):
+            self._offset = 0
+            self._buffer = ""
+            self._inode = None
+            return
+
+        if not text:
+            return
+
+        self._buffer += text
+        lines = self._buffer.splitlines(keepends=True)
+        if lines and not lines[-1].endswith(("\n", "\r")):
+            self._buffer = lines.pop()
+        else:
+            self._buffer = ""
+
+        for line in lines:
+            yield self._make_record(line.rstrip("\r\n"))
+
+    def _read_from_offset(self) -> str:
+        with self.path.open("r", encoding=self.encoding, errors="replace", newline="") as handle:
+            handle.seek(self._offset)
+            data = handle.read()
+            self._offset = handle.tell()
+            return data
+
+    @staticmethod
+    def _get_inode(stat: os.stat_result) -> int:
+        """Get file identity for rotation detection."""
+        if stat.st_ino != 0:
+            return stat.st_ino
+        return hash((stat.st_dev, stat.st_ctime_ns))
+
+    def _make_record(self, payload: str) -> RawRecord:
+        record = RawRecord(
+            payload=payload,
+            source_name=self.name,
+            mode="poll",
+            sequence=self._sequence,
+            received_at=datetime.now(timezone.utc),
+        )
+        self._sequence += 1
+        return record

traceforge/sources/file_watch.py

@@ -0,0 +1,221 @@
+"""File watching source using watchdog for OS-native filesystem events."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from pathlib import Path
+from types import TracebackType
+from typing import Literal, TextIO
+
+from watchdog.events import (
+    FileCreatedEvent,
+    FileModifiedEvent,
+    FileMovedEvent,
+    FileSystemEventHandler,
+)
+from watchdog.observers import Observer
+
+from traceforge.sources.base import RawRecord, Source
+
+
+class _FileHandler(FileSystemEventHandler):
+    """Watchdog handler that signals on file creation, modification, or move."""
+
+    def __init__(self, target_path: Path, loop: asyncio.AbstractEventLoop) -> None:
+        super().__init__()
+        self._target = str(target_path)
+        self._loop = loop
+        self._stopped = False
+        self.changed = asyncio.Event()
+
+    def stop(self) -> None:
+        self._stopped = True
+
+    def _signal(self) -> None:
+        if self._stopped:
+            return
+        try:
+            self._loop.call_soon_threadsafe(self.changed.set)
+        except RuntimeError:
+            pass
+
+    def on_modified(self, event: FileModifiedEvent) -> None:  # type: ignore[override]
+        if event.is_directory:
+            return
+        if str(Path(event.src_path).resolve()) == self._target:
+            self._signal()
+
+    def on_created(self, event: FileCreatedEvent) -> None:  # type: ignore[override]
+        if event.is_directory:
+            return
+        if str(Path(event.src_path).resolve()) == self._target:
+            self._signal()
+
+    def on_moved(self, event: FileMovedEvent) -> None:  # type: ignore[override]
+        if event.is_directory:
+            return
+        if str(Path(event.dest_path).resolve()) == self._target:
+            self._signal()
+
+
+class FileWatchSource(Source):
+    """Tail a file using watchdog OS-native events, yielding complete lines.
+
+    Handles file rotation, truncation, and creation. When the file is replaced
+    (new inode) or truncated, the source reopens from the beginning.
+    """
+
+    def __init__(
+        self,
+        path: str | Path,
+        name: str,
+        start_at: Literal["beginning", "end"] = "end",
+        encoding: str = "utf-8",
+    ) -> None:
+        self.path = Path(path).resolve()
+        self.name = name
+        self.start_at = start_at
+        self.encoding = encoding
+        self._file: TextIO | None = None
+        self._buffer = ""
+        self._sequence = 0
+        self._observer: Observer | None = None
+        self._handler: _FileHandler | None = None
+        self._inode: int | None = None
+        self._last_size: int = 0
+        self._iterating = False
+
+    async def __aenter__(self) -> "FileWatchSource":
+        self._open_file(start_at=self.start_at)
+        loop = asyncio.get_running_loop()
+        self._handler = _FileHandler(self.path, loop)
+        self._observer = Observer()
+        self._observer.schedule(self._handler, str(self.path.parent), recursive=False)
+        self._observer.start()
+        # Signal immediately so _iter_records drains any pre-existing content
+        if self.start_at == "beginning" and self._file is not None:
+            self._handler.changed.set()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._handler is not None:
+            self._handler.stop()
+        if self._observer is not None:
+            self._observer.stop()
+            self._observer.join(timeout=5)
+            self._observer = None
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+        self._buffer = ""
+        self._handler = None
+        self._iterating = False
+
+    async def _iter_records(self) -> AsyncIterator[RawRecord]:
+        if self._handler is None:
+            raise RuntimeError("FileWatchSource must be entered before iteration")
+        if self._iterating:
+            raise RuntimeError("FileWatchSource does not support concurrent iteration")
+        self._iterating = True
+        try:
+            while True:
+                await self._handler.changed.wait()
+                self._handler.changed.clear()
+
+                self._check_rotation()
+
+                if self._file is None:
+                    self._open_file(start_at="beginning")
+                    if self._file is None:
+                        continue
+
+                chunk = self._file.read()
+                if not chunk:
+                    continue
+
+                self._buffer += chunk
+                lines = self._buffer.splitlines(keepends=True)
+                if lines and not lines[-1].endswith(("\n", "\r")):
+                    self._buffer = lines.pop()
+                else:
+                    self._buffer = ""
+
+                for line in lines:
+                    yield self._make_record(line.rstrip("\r\n"))
+        finally:
+            self._iterating = False
+
+    def __aiter__(self) -> AsyncIterator[RawRecord]:
+        return self._iter_records()
+
+    def _check_rotation(self) -> None:
+        """Detect file rotation (new inode) or truncation and reopen."""
+        try:
+            stat = self.path.stat()
+        except FileNotFoundError:
+            if self._file is not None:
+                self._file.close()
+                self._file = None
+            self._inode = None
+            self._last_size = 0
+            self._buffer = ""
+            return
+
+        current_inode = self._get_inode(stat)
+        truncated = stat.st_size < self._last_size
+        rotated = self._inode is not None and current_inode != self._inode
+
+        if truncated or rotated:
+            self._open_file(start_at="beginning")
+        else:
+            self._last_size = stat.st_size
+
+    def _open_file(self, start_at: Literal["beginning", "end"]) -> None:
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+        try:
+            self._file = self.path.open("r", encoding=self.encoding, errors="replace", newline="")
+        except FileNotFoundError:
+            self._file = None
+            self._inode = None
+            self._last_size = 0
+            return
+        if start_at == "end":
+            self._file.seek(0, 2)
+        else:
+            self._file.seek(0)
+        self._buffer = ""
+        try:
+            stat = self.path.stat()
+            self._inode = self._get_inode(stat)
+            self._last_size = stat.st_size
+        except FileNotFoundError:
+            self._inode = None
+            self._last_size = 0
+
+    @staticmethod
+    def _get_inode(stat: os.stat_result) -> int:
+        """Get file identity. On Windows st_ino may be 0; fall back to creation time."""
+        if stat.st_ino != 0:
+            return stat.st_ino
+        return hash((stat.st_dev, stat.st_ctime_ns))
+
+    def _make_record(self, payload: str) -> RawRecord:
+        record = RawRecord(
+            payload=payload,
+            source_name=self.name,
+            mode="file_watch",
+            sequence=self._sequence,
+            received_at=datetime.now(timezone.utc),
+        )
+        self._sequence += 1
+        return record

traceforge/sources/http_poll.py

@@ -0,0 +1,141 @@
+"""HTTP polling source with conditional request support."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from types import TracebackType
+
+import httpx
+
+from traceforge.sources.base import RawRecord, Source
+
+logger = logging.getLogger(__name__)
+
+
+class HttpPollSource(Source):
+    """Poll an HTTP endpoint using conditional requests (ETag/Last-Modified).
+
+    Emits the full response body as a single record each time new content
+    is available. Supports cursor-based pagination via a custom header.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        name: str,
+        interval: float = 10.0,
+        cursor_header: str | None = None,
+        headers: dict[str, str] | None = None,
+        max_retries: int = 3,
+    ) -> None:
+        self.url = url
+        self.name = name
+        self.interval = interval
+        self.cursor_header = cursor_header
+        self.headers = dict(headers or {})
+        self.max_retries = max_retries
+        self._etag: str | None = None
+        self._last_modified: str | None = None
+        self._cursor: str | None = None
+        self._sequence = 0
+        self._client: httpx.AsyncClient | None = None
+        self._iterating = False
+
+    async def __aenter__(self) -> "HttpPollSource":
+        self._client = httpx.AsyncClient()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+        self._iterating = False
+
+    async def _iter_records(self) -> AsyncIterator[RawRecord]:
+        if self._client is None:
+            raise RuntimeError("HttpPollSource must be entered before iteration")
+        if self._iterating:
+            raise RuntimeError("HttpPollSource does not support concurrent iteration")
+        self._iterating = True
+        try:
+            while True:
+                record = await self._poll_once()
+                if record is not None:
+                    yield record
+                await asyncio.sleep(self.interval)
+        finally:
+            self._iterating = False
+
+    def __aiter__(self) -> AsyncIterator[RawRecord]:
+        return self._iter_records()
+
+    async def _poll_once(self) -> RawRecord | None:
+        """Execute one poll cycle with retry on transient errors."""
+        for attempt in range(self.max_retries + 1):
+            try:
+                return await self._do_request()
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:
+                if attempt >= self.max_retries:
+                    logger.error(
+                        "HttpPollSource %s: poll failed after %d retries: %s",
+                        self.name,
+                        self.max_retries,
+                        exc,
+                    )
+                    return None
+                delay = min(2**attempt, 30)
+                logger.warning(
+                    "HttpPollSource %s: transient error (attempt %d/%d): %s",
+                    self.name,
+                    attempt + 1,
+                    self.max_retries,
+                    exc,
+                )
+                await asyncio.sleep(delay)
+        return None
+
+    async def _do_request(self) -> RawRecord | None:
+        assert self._client is not None
+        headers = dict(self.headers)
+        if self._etag:
+            headers["If-None-Match"] = self._etag
+        if self._last_modified:
+            headers["If-Modified-Since"] = self._last_modified
+        if self.cursor_header and self._cursor:
+            headers[self.cursor_header] = self._cursor
+
+        response = await self._client.get(self.url, headers=headers)
+        if response.status_code == 304:
+            return None
+        response.raise_for_status()
+
+        self._etag = response.headers.get("ETag", self._etag)
+        self._last_modified = response.headers.get("Last-Modified", self._last_modified)
+        if self.cursor_header:
+            self._cursor = response.headers.get(self.cursor_header, self._cursor)
+
+        text = response.text
+        if not text:
+            return None
+        return self._make_record(text)
+
+    def _make_record(self, payload: str) -> RawRecord:
+        record = RawRecord(
+            payload=payload,
+            source_name=self.name,
+            mode="poll",
+            sequence=self._sequence,
+            received_at=datetime.now(timezone.utc),
+        )
+        self._sequence += 1
+        return record

traceforge/sources/replay.py

@@ -0,0 +1,63 @@
+"""Finite replay source for reading an existing file from start to end."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+from pathlib import Path
+from types import TracebackType
+
+from traceforge.sources.base import RawRecord, Source
+
+
+class ReplaySource(Source):
+    """Replay a file line-by-line from beginning to end.
+
+    Reads the file in a background thread to avoid blocking the event loop.
+    Validates that the file exists on entry.
+    """
+
+    def __init__(self, path: str | Path, name: str, encoding: str = "utf-8") -> None:
+        self.path = Path(path)
+        self.name = name
+        self.encoding = encoding
+        self._iterating = False
+
+    async def __aenter__(self) -> "ReplaySource":
+        if not self.path.exists():
+            raise FileNotFoundError(f"ReplaySource target does not exist: {self.path}")
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self._iterating = False
+
+    async def _iter_records(self) -> AsyncIterator[RawRecord]:
+        if self._iterating:
+            raise RuntimeError("ReplaySource does not support concurrent iteration")
+        self._iterating = True
+        try:
+            lines = await asyncio.to_thread(self._read_lines)
+            for index, line in enumerate(lines):
+                yield RawRecord(
+                    payload=line,
+                    source_name=self.name,
+                    mode="replay",
+                    sequence=index,
+                    received_at=datetime.now(timezone.utc),
+                )
+        finally:
+            self._iterating = False
+
+    def __aiter__(self) -> AsyncIterator[RawRecord]:
+        return self._iter_records()
+
+    def _read_lines(self) -> list[str]:
+        """Read file lines in a background thread."""
+        with self.path.open("r", encoding=self.encoding) as fh:
+            return [line.rstrip("\r\n") for line in fh]