shipwatch 0.1.0__py3-none-any.whl

shipwatch/__init__.py

@@ -0,0 +1,5 @@
+"""shipwatch: changelogs in, a cited weekly briefing out."""
+
+from shipwatch._version import __version__
+
+__all__ = ["__version__"]

shipwatch/__main__.py

@@ -0,0 +1,4 @@
+from shipwatch.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

shipwatch/_version.py

@@ -0,0 +1,3 @@
+"""Package version. Single source of truth for pyproject and the CLI."""
+
+__version__ = "0.1.0"

shipwatch/cli.py

@@ -0,0 +1,218 @@
+"""Command-line interface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from shipwatch._version import __version__
+from shipwatch.config import (
+    AppConfig,
+    bundled_sources_path,
+    load_config,
+    load_yaml,
+    parse_sources,
+    write_init_files,
+)
+from shipwatch.fetch.client import PoliteClient, check_url
+from shipwatch.models import SourceKind
+from shipwatch.output.render import coverage_line, write_outputs
+from shipwatch.pipeline.run import run_brief, run_fetch, run_pipeline
+from shipwatch.storage import Store
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="shipwatch",
+        description="Know what AI companies actually shipped this week.",
+    )
+    parser.add_argument("--version", action="version", version=f"shipwatch {__version__}")
+    parser.add_argument("--config", type=Path, help="Path to shipwatch.yaml")
+    parser.add_argument("--data-dir", type=Path, help="Override SQLite data directory")
+    parser.add_argument(
+        "--no-llm",
+        action="store_true",
+        help="Heuristic characterization, scoring, and extractive brief",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    init = sub.add_parser("init", help="Write sample config and source pack")
+    init.add_argument("--force", action="store_true")
+    init.add_argument("--dir", type=Path, default=Path("."))
+
+    fetch = sub.add_parser("fetch", help="Fetch sources into SQLite")
+    fetch.add_argument("--source", help="Fetch a single source id")
+
+    run = sub.add_parser("run", help="Fetch + score + brief")
+    _brief_flags(run)
+
+    brief = sub.add_parser("brief", help="Brief from items already in SQLite")
+    _brief_flags(brief)
+
+    sources = sub.add_parser("sources", help="Inspect the source pack")
+    src_sub = sources.add_subparsers(dest="sources_command", required=True)
+    check = src_sub.add_parser("check", help="Validate sources (add --live to hit the network)")
+    check.add_argument("--live", action="store_true")
+
+    sub.add_parser("stats", help="Show archive counts")
+    return parser
+
+
+def _brief_flags(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--since", help="Window such as 7d, 2w, or an ISO timestamp")
+    parser.add_argument("--top", type=int, dest="top_n", help="How many stories to keep")
+    parser.add_argument("--out", type=Path, help="Output directory")
+    parser.add_argument("--html", action="store_true", help="Also write a minimal index.html")
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return _dispatch(args)
+    except Exception as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+def _dispatch(args: argparse.Namespace) -> int:
+    if args.command == "init":
+        written = write_init_files(args.dir, force=args.force)
+        for path in written:
+            print(f"wrote {path}")
+        print("Edit sources.yaml, then run: shipwatch fetch && shipwatch run --no-llm")
+        return 0
+
+    cfg = _config(args)
+    if args.command == "sources":
+        return _sources_check(cfg, live=args.live)
+
+    store = Store(cfg.db_path)
+    try:
+        if args.command == "fetch":
+            stats = run_fetch(cfg, store, source_id=args.source)
+            _print_fetch(stats)
+            return 0
+        if args.command == "run":
+            _apply_brief_overrides(cfg, args)
+            briefing, items, stats = run_pipeline(
+                cfg,
+                store,
+                since=args.since,
+                top_n=args.top_n,
+                no_llm=args.no_llm,
+                html=args.html or None,
+            )
+            _print_fetch(stats)
+            _print_brief(briefing, cfg)
+            return 0
+        if args.command == "brief":
+            _apply_brief_overrides(cfg, args)
+            briefing, items, _stats = run_brief(
+                cfg,
+                store,
+                since=args.since,
+                top_n=args.top_n,
+                no_llm=args.no_llm,
+                html=args.html or None,
+            )
+            write_outputs(
+                briefing,
+                items,
+                cfg.brief.output_dir,
+                html=args.html or cfg.brief.html,
+                archive=store.list_briefings(limit=20),
+            )
+            _print_brief(briefing, cfg)
+            return 0
+        if args.command == "stats":
+            print(json.dumps(store.stats(), indent=2))
+            return 0
+    finally:
+        store.close()
+    return 1
+
+
+def _config(args: argparse.Namespace) -> AppConfig:
+    path = args.config
+    if path is None:
+        candidate = Path("shipwatch.yaml")
+        path = candidate if candidate.exists() else None
+    cfg = load_config(path)
+    if getattr(args, "data_dir", None):
+        cfg.data_dir = args.data_dir
+    if getattr(args, "no_llm", False):
+        pass
+    return cfg
+
+
+def _apply_brief_overrides(cfg: AppConfig, args: argparse.Namespace) -> None:
+    if getattr(args, "out", None):
+        cfg.brief.output_dir = args.out
+    if getattr(args, "html", False):
+        cfg.brief.html = True
+    if getattr(args, "since", None):
+        cfg.brief.since = args.since
+    if getattr(args, "top_n", None):
+        cfg.brief.top_n = args.top_n
+
+
+def _print_fetch(stats) -> None:
+    print(
+        f"fetched {stats.fetched} sources · {stats.items_considered} items ({stats.items_new} new)"
+    )
+    for outcome in stats.outcomes:
+        extra = f" HTTP {outcome.http_status}" if outcome.http_status else ""
+        detail = f" — {outcome.detail}" if outcome.detail else ""
+        print(f"  {outcome.source_id}: {outcome.status}{extra}{detail}")
+
+
+def _print_brief(briefing, cfg: AppConfig) -> None:
+    print(f"{briefing.title}")
+    print(coverage_line(briefing.coverage))
+    print(f"wrote {cfg.brief.output_dir / 'briefing.md'}")
+    print(f"wrote {cfg.brief.output_dir / 'briefing.json'}")
+    print(f"wrote {cfg.brief.output_dir / 'briefings.xml'}")
+    if cfg.brief.html:
+        print(f"wrote {cfg.brief.output_dir / 'index.html'}")
+
+
+def _sources_check(cfg: AppConfig, *, live: bool) -> int:
+    sources = cfg.sources or parse_sources(load_yaml(bundled_sources_path()))
+    errors = 0
+    for source in sources:
+        problems: list[str] = []
+        if source.kind is SourceKind.GITHUB and not source.repo:
+            problems.append("missing repo")
+        if source.kind is not SourceKind.GITHUB and not source.url:
+            problems.append("missing url")
+        if source.url and not source.url.startswith(("http://", "https://")):
+            problems.append("url must be http(s)")
+        status = "ok" if not problems else "invalid"
+        if problems:
+            errors += 1
+        extra = f" ({'; '.join(problems)})" if problems else ""
+        target = source.repo or source.url or ""
+        print(f"{source.id:24} {source.kind.value:8} {status:8} {target}{extra}")
+    print(f"{len(sources)} sources · {len(sources) - errors} valid · {errors} errors")
+    if not live:
+        return 1 if errors else 0
+
+    client = PoliteClient(cfg.fetch, store=None)
+    failed = 0
+    try:
+        for source in sources:
+            if not source.enabled:
+                print(f"{source.id:24} skipped (disabled)")
+                continue
+            result = check_url(client, source.target())
+            mark = "ok" if result["ok"] else "FAIL"
+            if not result["ok"]:
+                failed += 1
+            print(f"{source.id:24} {mark:8} {result.get('status')} {result['detail']}")
+    finally:
+        client.close()
+    print(f"live check: {len(sources) - failed} reachable · {failed} failed")
+    return 1 if failed or errors else 0

shipwatch/config.py

@@ -0,0 +1,277 @@
+"""YAML config, defaults, and the bundled source pack."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from shipwatch._version import __version__
+from shipwatch.models import SourceKind, SourceSpec
+
+DEFAULT_USER_AGENT = f"shipwatch/{__version__} (+https://github.com/andymccutcheon/shipwatch)"
+DEFAULT_FREEWHIRR_BASE = "http://127.0.0.1:8787/v1"
+DEFAULT_FREEWHIRR_MODEL = "freewhirr"
+
+
+@dataclass
+class LLMSettings:
+    base_url: str = DEFAULT_FREEWHIRR_BASE
+    model: str = DEFAULT_FREEWHIRR_MODEL
+    api_key_env: str = "SHIPWATCH_API_KEY"
+    timeout_seconds: float = 60.0
+    retries: int = 2
+    temperature: float = 0.2
+
+    def api_key(self) -> str:
+        for name in (self.api_key_env, "OPENROUTER_API_KEY", "OPENAI_API_KEY"):
+            value = os.environ.get(name, "").strip()
+            if value:
+                return value
+        return "not-used"
+
+
+@dataclass
+class ScoringSettings:
+    novelty: float = 0.25
+    impact: float = 0.30
+    breadth: float = 0.20
+    recency: float = 0.25
+    pricing_boost: float = 0.15
+    deprecation_boost: float = 0.15
+    security_boost: float = 0.10
+    recency_half_life_days: float = 7.0
+    near_dup_threshold: float = 0.62
+    cluster_threshold: float = 0.28
+    non_shipment_penalty: float = 0.40
+
+
+@dataclass
+class BriefSettings:
+    top_n: int = 12
+    since: str = "7d"
+    output_dir: Path = Path("out")
+    html: bool = False
+
+
+@dataclass
+class FetchSettings:
+    user_agent: str = DEFAULT_USER_AGENT
+    timeout_seconds: float = 30.0
+    respect_robots: bool = True
+    min_interval_seconds: float = 1.0
+    max_items_per_source: int = 40
+    github_token_env: str = "GITHUB_TOKEN"
+
+
+@dataclass
+class AppConfig:
+    config_path: Path | None = None
+    data_dir: Path = Path(".shipwatch")
+    sources_file: Path | None = None
+    sources: list[SourceSpec] = field(default_factory=list)
+    llm: LLMSettings = field(default_factory=LLMSettings)
+    scoring: ScoringSettings = field(default_factory=ScoringSettings)
+    brief: BriefSettings = field(default_factory=BriefSettings)
+    fetch: FetchSettings = field(default_factory=FetchSettings)
+
+    @property
+    def db_path(self) -> Path:
+        return self.data_dir / "shipwatch.db"
+
+    def enabled_sources(self) -> list[SourceSpec]:
+        return [s for s in self.sources if s.enabled]
+
+
+def bundled_sources_path() -> Path:
+    ref = resources.files("shipwatch").joinpath("data/sources.yaml")
+    return Path(str(ref))
+
+
+def default_config_text() -> str:
+    return f"""# shipwatch.yaml
+# Know what AI companies actually shipped this week.
+
+data_dir: .shipwatch
+sources_file: sources.yaml
+
+llm:
+  # Default: Andy's free-first OpenRouter router (https://github.com/andymccutcheon/freewhirr)
+  #   pip install freewhirr && freewhirr serve --port 8787
+  base_url: {DEFAULT_FREEWHIRR_BASE}
+  model: {DEFAULT_FREEWHIRR_MODEL}
+  api_key_env: SHIPWATCH_API_KEY
+  # OpenRouter (direct):
+  #   base_url: https://openrouter.ai/api/v1
+  #   model: meta-llama/llama-3.3-70b-instruct:free
+  #   api_key_env: OPENROUTER_API_KEY
+  # OpenAI:
+  #   base_url: https://api.openai.com/v1
+  #   model: gpt-4o-mini
+  #   api_key_env: OPENAI_API_KEY
+  retries: 2
+  temperature: 0.2
+
+scoring:
+  weights:
+    novelty: 0.25
+    impact: 0.30
+    breadth: 0.20
+    recency: 0.25
+  boosts:
+    pricing_change: 0.15
+    deprecation: 0.15
+    security: 0.10
+  penalties:
+    non_shipment: 0.40
+  recency_half_life_days: 7
+  near_dup_threshold: 0.62
+  cluster_threshold: 0.28
+
+brief:
+  top_n: 12
+  since: 7d
+  output_dir: out
+  html: false
+
+fetch:
+  user_agent: "{DEFAULT_USER_AGENT}"
+  timeout_seconds: 30
+  respect_robots: true
+  min_interval_seconds: 1.0
+  max_items_per_source: 40
+"""
+
+
+def load_yaml(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    return data if isinstance(data, dict) else {}
+
+
+def parse_sources(payload: Any) -> list[SourceSpec]:
+    rows = payload.get("sources", []) if isinstance(payload, dict) else payload
+    if not isinstance(rows, list):
+        raise ValueError("sources file must be a list or a mapping with a 'sources' key")
+    sources: list[SourceSpec] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            raise ValueError("each source must be a mapping")
+        sources.append(
+            SourceSpec(
+                id=str(row["id"]),
+                name=str(row.get("name") or row["id"]),
+                kind=SourceKind(str(row["kind"])),
+                url=row.get("url"),
+                repo=row.get("repo"),
+                selector=row.get("selector"),
+                enabled=bool(row.get("enabled", True)),
+                tags=list(row.get("tags") or []),
+                notes=row.get("notes"),
+            )
+        )
+    return sources
+
+
+def _weights(raw: dict[str, Any]) -> ScoringSettings:
+    weights = raw.get("weights") or {}
+    boosts = raw.get("boosts") or {}
+    penalties = raw.get("penalties") or {}
+    return ScoringSettings(
+        novelty=float(weights.get("novelty", 0.25)),
+        impact=float(weights.get("impact", 0.30)),
+        breadth=float(weights.get("breadth", 0.20)),
+        recency=float(weights.get("recency", 0.25)),
+        pricing_boost=float(boosts.get("pricing_change", 0.15)),
+        deprecation_boost=float(boosts.get("deprecation", 0.15)),
+        security_boost=float(boosts.get("security", 0.10)),
+        recency_half_life_days=float(raw.get("recency_half_life_days", 7)),
+        near_dup_threshold=float(raw.get("near_dup_threshold", 0.62)),
+        cluster_threshold=float(raw.get("cluster_threshold", 0.28)),
+        non_shipment_penalty=float(penalties.get("non_shipment", 0.40)),
+    )
+
+
+def load_config(path: Path | None) -> AppConfig:
+    cfg = AppConfig(config_path=path)
+    data: dict[str, Any] = {}
+    if path and path.exists():
+        data = load_yaml(path)
+        root = path.parent
+    else:
+        root = Path.cwd()
+
+    cfg.data_dir = Path(
+        data.get("data_dir") or os.environ.get("SHIPWATCH_DATA_DIR") or cfg.data_dir
+    )
+    if not cfg.data_dir.is_absolute():
+        cfg.data_dir = (root / cfg.data_dir).resolve() if path else Path.cwd() / cfg.data_dir
+
+    sources_file = data.get("sources_file")
+    inline = data.get("sources")
+    if sources_file:
+        src_path = Path(sources_file)
+        if not src_path.is_absolute():
+            src_path = root / src_path
+        cfg.sources_file = src_path
+        cfg.sources = parse_sources(load_yaml(src_path))
+    elif inline:
+        cfg.sources = parse_sources(inline)
+    else:
+        cfg.sources = parse_sources(load_yaml(bundled_sources_path()))
+
+    llm = data.get("llm") or {}
+    cfg.llm = LLMSettings(
+        base_url=os.environ.get("SHIPWATCH_BASE_URL")
+        or llm.get("base_url")
+        or DEFAULT_FREEWHIRR_BASE,
+        model=os.environ.get("SHIPWATCH_MODEL") or llm.get("model") or DEFAULT_FREEWHIRR_MODEL,
+        api_key_env=llm.get("api_key_env") or "SHIPWATCH_API_KEY",
+        timeout_seconds=float(llm.get("timeout_seconds", 60)),
+        retries=int(llm.get("retries", 2)),
+        temperature=float(llm.get("temperature", 0.2)),
+    )
+    cfg.scoring = _weights(data.get("scoring") or {})
+    brief = data.get("brief") or {}
+    cfg.brief = BriefSettings(
+        top_n=int(brief.get("top_n", 12)),
+        since=str(brief.get("since", "7d")),
+        output_dir=Path(brief.get("output_dir") or "out"),
+        html=bool(brief.get("html", False)),
+    )
+    if not cfg.brief.output_dir.is_absolute():
+        cfg.brief.output_dir = (
+            (root / cfg.brief.output_dir).resolve() if path else Path.cwd() / cfg.brief.output_dir
+        )
+    fetch = data.get("fetch") or {}
+    cfg.fetch = FetchSettings(
+        user_agent=fetch.get("user_agent") or DEFAULT_USER_AGENT,
+        timeout_seconds=float(fetch.get("timeout_seconds", 30)),
+        respect_robots=bool(fetch.get("respect_robots", True)),
+        min_interval_seconds=float(fetch.get("min_interval_seconds", 1.0)),
+        max_items_per_source=int(fetch.get("max_items_per_source", 40)),
+        github_token_env=fetch.get("github_token_env") or "GITHUB_TOKEN",
+    )
+    return cfg
+
+
+def write_init_files(directory: Path, *, force: bool = False) -> list[Path]:
+    directory.mkdir(parents=True, exist_ok=True)
+    written: list[Path] = []
+    config_path = directory / "shipwatch.yaml"
+    sources_path = directory / "sources.yaml"
+    if config_path.exists() and not force:
+        raise FileExistsError(f"{config_path} already exists (pass --force to overwrite)")
+    config_path.write_text(default_config_text(), encoding="utf-8")
+    written.append(config_path)
+    pack = bundled_sources_path().read_text(encoding="utf-8")
+    if sources_path.exists() and not force:
+        raise FileExistsError(f"{sources_path} already exists (pass --force to overwrite)")
+    sources_path.write_text(pack, encoding="utf-8")
+    written.append(sources_path)
+    return written

shipwatch/data/sources.yaml

@@ -0,0 +1,172 @@
+# Sample source pack (~26 well-known AI and developer-tool changelogs).
+# URLs were fetched successfully on 2026-10-11 with the shipwatch user agent.
+# Prefer RSS/Atom and GitHub Releases; HTML pages are a fallback and may be
+# thin when a site is client-rendered.
+#
+# Edit freely. Disable a row with enabled: false.
+
+sources:
+  - id: openai-news
+    name: OpenAI News
+    kind: rss
+    url: https://openai.com/news/rss.xml
+    tags: [openai, models]
+
+  - id: openai-api
+    name: OpenAI API Changelog
+    kind: html
+    url: https://developers.openai.com/api/docs/changelog
+    tags: [openai, api]
+
+  - id: anthropic-news
+    name: Anthropic News
+    kind: html
+    url: https://www.anthropic.com/news
+    tags: [anthropic, models]
+
+  - id: anthropic-release-notes
+    name: Anthropic Release Notes
+    kind: html
+    url: https://docs.anthropic.com/en/release-notes/overview
+    tags: [anthropic, api]
+    notes: Redirects to platform.claude.com as of 2026-10-11.
+
+  - id: google-ai-blog
+    name: Google AI Blog
+    kind: rss
+    url: https://blog.google/technology/ai/rss/
+    tags: [google, gemini]
+
+  - id: gemini-api
+    name: Gemini API Changelog
+    kind: html
+    url: https://ai.google.dev/gemini-api/docs/changelog
+    tags: [google, api]
+
+  - id: gemini-docs-sitemap
+    name: Gemini docs (new pages)
+    kind: sitemap
+    url: https://ai.google.dev/sitemap.xml
+    tags: [google, docs]
+    notes: Optional new-page detector. High volume; disable if noisy.
+
+  - id: mistral-news
+    name: Mistral News
+    kind: rss
+    url: https://mistral.ai/news/rss
+    tags: [mistral, models]
+
+  - id: cursor
+    name: Cursor Changelog
+    kind: html
+    url: https://cursor.com/changelog
+    tags: [cursor, editor]
+    notes: Marketing changelog; may be JS-rendered.
+
+  - id: vercel
+    name: Vercel
+    kind: rss
+    url: https://vercel.com/atom
+    tags: [vercel, platform]
+
+  - id: huggingface
+    name: Hugging Face Blog
+    kind: rss
+    url: https://huggingface.co/blog/feed.xml
+    tags: [huggingface, models]
+
+  - id: langchain
+    name: LangChain
+    kind: github
+    repo: langchain-ai/langchain
+    tags: [langchain, framework]
+
+  - id: openrouter
+    name: OpenRouter API Changelog
+    kind: html
+    url: https://openrouter.ai/docs/changelog
+    tags: [openrouter, api]
+
+  - id: llamaindex
+    name: LlamaIndex
+    kind: github
+    repo: run-llama/llama_index
+    tags: [llamaindex, framework]
+
+  - id: groq
+    name: Groq Changelog
+    kind: html
+    url: https://console.groq.com/docs/changelog
+    tags: [groq, inference]
+
+  - id: cohere
+    name: Cohere Changelog
+    kind: html
+    url: https://docs.cohere.com/changelog
+    tags: [cohere, models]
+
+  - id: pinecone
+    name: Pinecone Release Notes
+    kind: html
+    url: https://docs.pinecone.io/release-notes/2026
+    tags: [pinecone, retrieval]
+
+  - id: replicate
+    name: Replicate Changelog
+    kind: html
+    url: https://replicate.com/changelog
+    tags: [replicate, inference]
+
+  - id: cloudflare
+    name: Cloudflare Blog
+    kind: rss
+    url: https://blog.cloudflare.com/rss/
+    tags: [cloudflare, platform]
+
+  - id: nextjs
+    name: Next.js
+    kind: github
+    repo: vercel/next.js
+    tags: [vercel, nextjs]
+
+  - id: fastapi
+    name: FastAPI
+    kind: github
+    repo: fastapi/fastapi
+    tags: [python, api]
+
+  - id: pydantic
+    name: Pydantic
+    kind: github
+    repo: pydantic/pydantic
+    tags: [python, validation]
+
+  - id: supabase
+    name: Supabase Changelog
+    kind: html
+    url: https://supabase.com/changelog
+    tags: [supabase, platform]
+
+  - id: github-changelog
+    name: GitHub Changelog
+    kind: rss
+    url: https://github.blog/changelog/feed/
+    tags: [github, platform]
+
+  - id: xai
+    name: xAI Release Notes
+    kind: html
+    url: https://docs.x.ai/docs/release-notes
+    tags: [xai, models]
+
+  - id: deepseek
+    name: DeepSeek API Updates
+    kind: html
+    url: https://api-docs.deepseek.com/updates
+    tags: [deepseek, models]
+
+  - id: together
+    name: Together AI Changelog
+    kind: html
+    url: https://docs.together.ai/docs/changelog
+    tags: [together, inference]