fetchkit-agents 0.1.0__py3-none-any.whl

fetchkit/__init__.py

@@ -0,0 +1,91 @@
+"""
+fetchkit — a YAML-configured data fetching library for agentic applications.
+
+Fetch posts and comments from sources (Hacker News, RSS/Atom, arXiv, GitHub,
+Lobsters) into a single canonical ``Post`` model, with deduplication, sorting,
+and a typed config. Designed as a data-collection layer for agentic / LLM
+applications.
+
+Quick start::
+
+    from fetchkit import load_config, collect_all
+
+    config = load_config("config.yaml")
+    result = collect_all(config)
+    for post in result.posts:
+        print(post.title, post.url)
+
+See the README for the full configuration reference and the guide to adding
+fetchers.
+"""
+
+__version__ = "0.1.0"
+
+from fetchkit.collector import collect_all
+from fetchkit.config_loader import load_config, ConfigError
+from fetchkit.fetchers.base import Fetcher, FetcherResult
+from fetchkit.fetchers.registry import register_fetcher, get_fetcher
+from fetchkit.http import (
+    HttpClient,
+    RateLimiter,
+    get_default_client,
+    set_default_client,
+    use_client,
+)
+from fetchkit.schemas.post import Post, Comment, Source
+from fetchkit.schemas.fetcher import (
+    SortOrder,
+    PostFetchConfig,
+    CommentFetchConfig,
+    FetcherBase,
+    HackerNewsFetchConfig,
+    RSSFeedDescriptor,
+    RSSFetchConfig,
+    ArxivFetchConfig,
+    GitHubFetchConfig,
+    LobstersFetchConfig,
+    FetcherConfig,
+)
+from fetchkit.schemas.collector import CollectorResult
+from fetchkit.schemas.config import FetchKitConfig, HttpConfig
+from fetchkit.utils.time import resolve_window, parse_duration
+
+__all__ = [
+    "__version__",
+    # Orchestration
+    "collect_all",
+    "load_config",
+    "ConfigError",
+    # Fetcher protocol & registry
+    "Fetcher",
+    "FetcherResult",
+    "register_fetcher",
+    "get_fetcher",
+    # HTTP
+    "HttpClient",
+    "RateLimiter",
+    "get_default_client",
+    "set_default_client",
+    "use_client",
+    # Schemas
+    "Post",
+    "Comment",
+    "Source",
+    "SortOrder",
+    "PostFetchConfig",
+    "CommentFetchConfig",
+    "FetcherBase",
+    "HackerNewsFetchConfig",
+    "RSSFeedDescriptor",
+    "RSSFetchConfig",
+    "ArxivFetchConfig",
+    "GitHubFetchConfig",
+    "LobstersFetchConfig",
+    "FetcherConfig",
+    "CollectorResult",
+    "FetchKitConfig",
+    "HttpConfig",
+    # Time helpers
+    "resolve_window",
+    "parse_duration",
+]

fetchkit/__main__.py

@@ -0,0 +1,8 @@
+"""Enable ``python -m fetchkit`` as an alias for the ``fetchkit`` CLI."""
+
+import sys
+
+from fetchkit.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

fetchkit/cli.py

@@ -0,0 +1,169 @@
+"""Command-line interface for fetchkit.
+
+Designed so an agent (or a shell script) can fetch news deterministically from a
+YAML spec and parse structured JSON from stdout::
+
+    fetchkit run config.yaml                 # JSON: {"posts": [...], "errors": [...]}
+    fetchkit run config.yaml -o out.json     # write JSON to a file instead of stdout
+    fetchkit run config.yaml --fail-on-error # exit 1 if any source failed
+    fetchkit validate config.yaml            # check a config without fetching
+
+stdout carries only JSON (for ``run``) so it is safe to pipe into ``jq`` or parse
+programmatically. Diagnostics and ``--verbose`` logging go to stderr.
+
+Exit codes::
+
+    0  success
+    1  fetch completed but a source failed (only with --fail-on-error)
+    2  configuration error (missing file, invalid YAML, validation failure)
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Optional, Sequence
+
+from fetchkit import __version__
+from fetchkit.collector import collect_all
+from fetchkit.config_loader import ConfigError, load_config
+from fetchkit.utils.time import resolve_window
+
+
+def _emit_json(payload: Any, indent: Optional[int], output: Optional[str]) -> None:
+    """Write a JSON document to ``output`` (a file path) or stdout.
+
+    When writing to a file, stdout stays empty so the file is the sole result
+    artifact; a one-line confirmation is logged to stderr.
+    """
+    text = json.dumps(payload, indent=indent, ensure_ascii=False)
+    if output is None:
+        sys.stdout.write(text + "\n")
+        return
+    Path(output).expanduser().write_text(text + "\n", encoding="utf-8")
+    print(f"Wrote {payload['count']} post(s) to {output}", file=sys.stderr)
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    """Load a config, collect posts, and emit a JSON result document on stdout."""
+    try:
+        config = load_config(args.config)
+    except ConfigError as exc:
+        print(str(exc), file=sys.stderr)
+        return 2
+
+    # The window can be set two ways: in the YAML config itself (a `window:` key,
+    # mutually exclusive with start_time/end_time), or here on the command line.
+    # --window is a runtime override: it replaces whatever bounds the config
+    # resolved to, so you can reuse one config across different time ranges.
+    if args.window:
+        try:
+            config.start_time, config.end_time = resolve_window(args.window)
+        except ValueError as exc:
+            print(f"Invalid --window: {exc}", file=sys.stderr)
+            return 2
+
+    # Keep stdout pure JSON: progress/verbosity is logged to stderr, never printed.
+    result = collect_all(config)
+
+    payload = {
+        "count": len(result.posts),
+        "posts": [post.model_dump(mode="json") for post in result.posts],
+        "errors": [
+            {"source": source, "error": str(err)} for source, err in result.errors
+        ],
+    }
+    indent = None if args.compact else args.indent
+    _emit_json(payload, indent, args.output)
+
+    if args.fail_on_error and result.has_errors:
+        return 1
+    return 0
+
+
+def _cmd_validate(args: argparse.Namespace) -> int:
+    """Load and validate a config without fetching; report the result."""
+    try:
+        config = load_config(args.config)
+    except ConfigError as exc:
+        print(str(exc), file=sys.stderr)
+        return 2
+
+    enabled = sum(1 for f in config.fetchers if f.enabled)
+    print(
+        f"OK: {args.config} is valid "
+        f"({len(config.fetchers)} fetcher(s), {enabled} enabled).",
+        file=sys.stderr,
+    )
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Construct the top-level argument parser with ``run`` and ``validate``."""
+    parser = argparse.ArgumentParser(
+        prog="fetchkit",
+        description="Fetch news/posts deterministically from a YAML spec.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"fetchkit {__version__}"
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true",
+        help="Enable INFO-level logging to stderr.",
+    )
+
+    # Define the two subcommands (`fetchkit run ...` / `fetchkit validate ...`).
+    # `dest="command"` records which one was chosen; `required=True` makes picking
+    # a subcommand mandatory (running bare `fetchkit` errors with usage help).
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run = sub.add_parser("run", help="Fetch posts and print JSON to stdout.")
+    run.add_argument("config", help="Path to the YAML config file.")
+    run.add_argument(
+        "-o", "--output", default=None, metavar="PATH",
+        help="Write JSON to this file instead of stdout.",
+    )
+    run.add_argument(
+        "--window", default=None, metavar="SPEC",
+        help="Override the config time window, e.g. 'last 6 hours', 'yesterday', '7d'.",
+    )
+    run.add_argument(
+        "--indent", type=int, default=2,
+        help="JSON indentation for pretty output (default: 2).",
+    )
+    run.add_argument(
+        "--compact", action="store_true",
+        help="Emit single-line JSON (overrides --indent).",
+    )
+    run.add_argument(
+        "--fail-on-error", action="store_true",
+        help="Exit 1 if any source reported an error.",
+    )
+    run.set_defaults(func=_cmd_run)
+
+    validate = sub.add_parser("validate", help="Validate a config without fetching.")
+    validate.add_argument("config", help="Path to the YAML config file.")
+    validate.set_defaults(func=_cmd_validate)
+
+    return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    """CLI entry point. Returns a process exit code."""
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.verbose:
+        logging.basicConfig(
+            level=logging.INFO,
+            stream=sys.stderr,
+            format="%(levelname)s %(name)s: %(message)s",
+        )
+
+    exit_code: int = args.func(args)
+    return exit_code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

fetchkit/collector.py

@@ -0,0 +1,116 @@
+"""
+Unified fetcher orchestrator.
+
+Orchestrates fetching from multiple data sources (Hacker News, RSS, etc.)
+using a single top-level configuration. Preserves three core invariants:
+
+1. Per-fetcher time windows inherit the global window when omitted.
+2. Posts are de-duplicated by ``(source, id)`` (first occurrence wins).
+3. Posts are sorted descending by ``(created_at or UTC_MIN, id)`` for determinism.
+"""
+
+import logging
+from contextlib import nullcontext
+from typing import Optional
+
+from fetchkit.schemas.post import Post
+from fetchkit.schemas.config import FetchKitConfig
+from fetchkit.fetchers.registry import get_fetcher
+from fetchkit.utils.time import UTC_MIN
+from fetchkit.schemas.collector import CollectorResult
+from fetchkit.http import HttpClient, use_client
+
+logger = logging.getLogger(__name__)
+
+
+def collect_all(
+    config: FetchKitConfig,
+    verbose: bool = False,
+    http_client: Optional[HttpClient] = None,
+) -> CollectorResult:
+    """
+    Collect posts from all enabled data sources.
+
+    Args:
+        config: Top-level FetchKitConfig with global time window and fetcher settings.
+        verbose: When True, print per-source progress to stdout (off by default for
+            library use).
+        http_client: Optional shared HTTP client to install for this run. When
+            provided, it becomes the active default client used by fetchers for the
+            duration of the call. When None and ``config.http`` is set, a client is
+            built from it; otherwise the built-in default client is used.
+
+    Returns:
+        CollectorResult containing posts and any errors that occurred.
+        Callers should check result.has_errors to detect partial failures.
+    """
+    all_posts: list[Post] = []
+    errors: list[tuple[str, Exception]] = []
+
+    # Resolve the HTTP client for this run: the explicit override, else one built
+    # from config.http, else None (fall back to the ambient default). When we do
+    # have a run-specific client, install it thread-locally for the duration of
+    # the run via use_client() so concurrent collect_all() calls never clobber
+    # each other or a caller's set_default_client().
+    run_client: Optional[HttpClient] = http_client
+    if run_client is None and config.http is not None:
+        run_client = HttpClient(config.http)
+    client_ctx = use_client(run_client) if run_client is not None else nullcontext()
+
+    def _log(msg: str) -> None:
+        if verbose:
+            print(msg)
+
+    def _record_error(source: str, err: Exception) -> None:
+        """Track a source failure and emit matching log/console diagnostics."""
+        errors.append((source, err))
+        logger.error("Failed to fetch from %s: %s", source, err)
+        _log(f"Error: Failed to fetch from {source}: {err}")
+
+    with client_ctx:
+        for fetcher_config in config.fetchers:
+            if not fetcher_config.enabled:
+                continue
+
+            source_name = fetcher_config.type
+
+            try:
+                # Inherit global time window if not set
+                f_config = fetcher_config.model_copy()
+                if f_config.start_time is None:
+                    f_config.start_time = config.start_time
+                if f_config.end_time is None:
+                    f_config.end_time = config.end_time
+
+                fetcher = get_fetcher(f_config.type)
+                result = fetcher(f_config)
+
+                msg_name = f_config.name if f_config.name else f_config.type
+                _log(f"Fetched {len(result.posts)} posts from {msg_name}")
+
+                all_posts.extend(result.posts)
+
+                for err in result.errors:
+                    _record_error(source_name, err)
+
+            except Exception as e:
+                _record_error(source_name, e)
+
+    # Dedup by (source, id) for maximum safety
+    seen_keys = set()
+    unique_posts = []
+
+    for post in all_posts:
+        key = (post.source, post.id)
+        if key not in seen_keys:
+            seen_keys.add(key)
+            unique_posts.append(post)
+
+    # Sort by created_at DESC, then by source-specific ID for determinism
+    sorted_posts = sorted(
+        unique_posts,
+        key=lambda x: (x.created_at or UTC_MIN, x.id),
+        reverse=True,
+    )
+
+    return CollectorResult(posts=sorted_posts, errors=errors)

fetchkit/config_loader.py

@@ -0,0 +1,59 @@
+"""Load and validate YAML fetcher configuration into typed fetchkit models."""
+
+import yaml
+from pathlib import Path
+from pydantic import ValidationError
+from fetchkit.schemas.config import FetchKitConfig
+
+
+class ConfigError(Exception):
+    """Base exception for configuration loading errors."""
+    pass
+
+
+def load_config(path: str) -> FetchKitConfig:
+    """
+    Load a FetchKitConfig from a YAML file.
+
+    The YAML schema is fetchers-only::
+
+        start_time: "2026-01-31T00:00:00Z"
+        end_time:   "2026-02-01T00:00:00Z"
+        fetchers:
+          - type: hackernews
+            posts: { max_items: 50, order: new }
+          - type: rss
+            feeds:
+              - url: "https://example.com/rss"
+        http:  # optional
+          timeout: 15
+          max_retries: 5
+
+    Raises:
+        ConfigError: If the file is missing, malformed YAML, or fails validation.
+    """
+    config_path = Path(path).expanduser()
+    if not config_path.exists():
+        raise ConfigError(f"Config file not found: {path}")
+
+    try:
+        raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as e:
+        raise ConfigError(f"Error parsing YAML in {path}: {e}")
+    except Exception as e:
+        raise ConfigError(f"Error reading config file {path}: {e}")
+
+    if raw_config is None:
+        raw_config = {}
+
+    try:
+        return FetchKitConfig.model_validate(raw_config)
+    except ValidationError as e:
+        # Format Pydantic errors for better CLI readability
+        error_messages = []
+        for error in e.errors():
+            loc = ".".join(str(part) for part in error["loc"])
+            msg = error["msg"]
+            error_messages.append(f"  - {loc}: {msg}")
+        detail = "\n".join(error_messages)
+        raise ConfigError(f"Config validation failed for {path}:\n{detail}") from e

fetchkit/fetchers/__init__.py

@@ -0,0 +1,11 @@
+"""Fetcher package exports and builtin fetcher registration side effects."""
+
+from .registry import get_fetcher, register_fetcher
+# Import built-in fetchers to ensure they register themselves on import.
+from . import hackernews
+from . import rss
+from . import arxiv
+from . import github
+from . import lobsters
+
+__all__ = ["get_fetcher", "register_fetcher"]

fetchkit/fetchers/arxiv.py

@@ -0,0 +1,133 @@
+"""
+arXiv fetcher.
+
+Fetches papers from the arXiv export API (https://export.arxiv.org/api/query),
+which returns Atom XML — parsed with feedparser, already a fetchkit dependency.
+Multi-valued detail (all authors, categories, DOI, PDF link) is preserved in
+``Post.metadata``.
+"""
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+import feedparser
+
+from fetchkit.http import get_default_client
+from fetchkit.schemas.post import Post, Source
+from fetchkit.schemas.fetcher import ArxivFetchConfig, FetcherConfig
+from fetchkit.fetchers.base import FetcherResult
+from fetchkit.fetchers.registry import register_fetcher
+
+logger = logging.getLogger(__name__)
+
+ARXIV_API_URL = "https://export.arxiv.org/api/query"
+SOURCE_NAME = Source.ARXIV
+DEFAULT_TIMEOUT_S = 30  # arXiv can be slow
+
+
+def _build_search_query(config: ArxivFetchConfig) -> str:
+    """Combine categories and free-text query into an arXiv ``search_query``."""
+    clauses: list[str] = []
+    if config.categories:
+        cats = " OR ".join(f"cat:{c}" for c in config.categories)
+        clauses.append(f"({cats})")
+    if config.query:
+        clauses.append(f"all:{config.query}")
+    return " AND ".join(clauses) if clauses else "all:*"
+
+
+def _parse_published(entry: Any) -> Optional[datetime]:
+    """Extract the published date (UTC) from a feedparser entry."""
+    for attr in ("published_parsed", "updated_parsed"):
+        st = getattr(entry, attr, None)
+        if st:
+            return datetime(st[0], st[1], st[2], st[3], st[4], st[5], tzinfo=timezone.utc)
+    return None
+
+
+def _pdf_link(entry: Any) -> Optional[str]:
+    """Return the PDF link for an entry, if present."""
+    for link in getattr(entry, "links", []) or []:
+        if getattr(link, "type", None) == "application/pdf" or getattr(link, "title", None) == "pdf":
+            return str(getattr(link, "href", "")) or None
+    return None
+
+
+def _normalize_entry(entry: Any) -> Optional[Post]:
+    """Convert a feedparser arXiv entry into a canonical Post."""
+    created_at = _parse_published(entry)
+    abs_url = str(getattr(entry, "id", "")) or None  # arXiv abs page
+    if not abs_url:
+        return None
+
+    # arXiv id is the trailing path segment of the abs URL (e.g. 2401.12345v1).
+    arxiv_id = abs_url.rstrip("/").rsplit("/", 1)[-1]
+
+    authors = [str(a.get("name")) for a in getattr(entry, "authors", []) if a.get("name")]
+    categories = [str(t.get("term")) for t in getattr(entry, "tags", []) if t.get("term")]
+
+    metadata: dict[str, Any] = {}
+    if authors:
+        metadata["authors"] = authors
+    if categories:
+        metadata["categories"] = categories
+    if getattr(entry, "arxiv_doi", None):
+        metadata["doi"] = str(entry.arxiv_doi)
+    pdf = _pdf_link(entry)
+    if pdf:
+        metadata["pdf_url"] = pdf
+    if getattr(entry, "arxiv_primary_category", None):
+        primary = entry.arxiv_primary_category.get("term") if isinstance(entry.arxiv_primary_category, dict) else None
+        if primary:
+            metadata["primary_category"] = str(primary)
+
+    return Post(
+        id=arxiv_id,
+        source=SOURCE_NAME,
+        title=getattr(entry, "title", None),
+        text=getattr(entry, "summary", None),
+        url=pdf or abs_url,
+        author=", ".join(authors) if authors else None,
+        created_at=created_at,
+        source_url=abs_url,
+        metadata=metadata,
+    )
+
+
+def fetch_posts(config: ArxivFetchConfig) -> list[Post]:
+    """Fetch papers from arXiv matching the configured categories/query."""
+    params = {
+        "search_query": _build_search_query(config),
+        "start": 0,
+        "max_results": config.max_items,
+        "sortBy": "submittedDate",
+        "sortOrder": "descending",
+    }
+    client = get_default_client()
+    response = client.get(ARXIV_API_URL, params=params, timeout=DEFAULT_TIMEOUT_S)
+    response.raise_for_status()
+
+    feed = feedparser.parse(response.content)
+    posts: list[Post] = []
+    for entry in feed.entries:
+        post = _normalize_entry(entry)
+        if post is None:
+            continue
+        # Window filtering (datetimes are UTC-normalized by the schema).
+        if config.start_time is not None and config.end_time is not None:
+            if post.created_at is None or not (config.start_time <= post.created_at <= config.end_time):
+                continue
+        posts.append(post)
+    return posts
+
+
+@register_fetcher("arxiv")
+def fetch(config: FetcherConfig) -> FetcherResult:
+    """Fetcher protocol implementation for arXiv."""
+    if not isinstance(config, ArxivFetchConfig):
+        raise ValueError(f"Invalid config type for arxiv fetcher: {type(config)}")
+    try:
+        return FetcherResult(posts=fetch_posts(config), errors=[])
+    except Exception as e:
+        return FetcherResult(posts=[], errors=[e])

fetchkit/fetchers/base.py

@@ -0,0 +1,31 @@
+"""
+Base protocol for data fetchers.
+"""
+from typing import Protocol, runtime_checkable
+from dataclasses import dataclass, field
+from fetchkit.schemas.post import Post
+from fetchkit.schemas.fetcher import FetcherConfig
+
+
+@dataclass
+class FetcherResult:
+    """Result from a fetch operation."""
+    posts: list[Post] = field(default_factory=list)
+    errors: list[Exception] = field(default_factory=list)
+
+
+@runtime_checkable
+class Fetcher(Protocol):
+    """Protocol that all fetchers must implement."""
+
+    def __call__(self, config: FetcherConfig) -> FetcherResult:
+        """
+        Execute the fetch operation.
+
+        Args:
+            config: The configuration for this specific fetcher instance.
+
+        Returns:
+            FetcherResult containing fetched posts and any errors.
+        """
+        ...