newswatcher 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- newswatcher-0.1.0/.agents/plugins/marketplace.json +12 -0
- newswatcher-0.1.0/.claude-plugin/marketplace.json +13 -0
- newswatcher-0.1.0/.gitattributes +11 -0
- newswatcher-0.1.0/.github/workflows/check.yml +24 -0
- newswatcher-0.1.0/.github/workflows/publish.yml +61 -0
- newswatcher-0.1.0/.github/workflows/reusable-check.yml +82 -0
- newswatcher-0.1.0/.gitignore +35 -0
- newswatcher-0.1.0/LICENSE +21 -0
- newswatcher-0.1.0/Makefile +13 -0
- newswatcher-0.1.0/PKG-INFO +367 -0
- newswatcher-0.1.0/README.ko.md +320 -0
- newswatcher-0.1.0/README.md +326 -0
- newswatcher-0.1.0/docs/korean-news-rss.md +285 -0
- newswatcher-0.1.0/docs/world-news-rss.md +164 -0
- newswatcher-0.1.0/plugins/newswatcher/.claude-plugin/plugin.json +11 -0
- newswatcher-0.1.0/plugins/newswatcher/.codex-plugin/plugin.json +20 -0
- newswatcher-0.1.0/plugins/newswatcher/skills/poll/SKILL.md +112 -0
- newswatcher-0.1.0/pyproject.toml +104 -0
- newswatcher-0.1.0/src/newswatcher/__init__.py +13 -0
- newswatcher-0.1.0/src/newswatcher/__main__.py +10 -0
- newswatcher-0.1.0/src/newswatcher/_atomic.py +36 -0
- newswatcher-0.1.0/src/newswatcher/_llm.py +82 -0
- newswatcher-0.1.0/src/newswatcher/_select.py +36 -0
- newswatcher-0.1.0/src/newswatcher/_toml.py +54 -0
- newswatcher-0.1.0/src/newswatcher/body.py +47 -0
- newswatcher-0.1.0/src/newswatcher/cli.py +551 -0
- newswatcher-0.1.0/src/newswatcher/config.py +217 -0
- newswatcher-0.1.0/src/newswatcher/crawl.py +104 -0
- newswatcher-0.1.0/src/newswatcher/credentials.py +286 -0
- newswatcher-0.1.0/src/newswatcher/digest.py +257 -0
- newswatcher-0.1.0/src/newswatcher/errors.py +68 -0
- newswatcher-0.1.0/src/newswatcher/feed.py +142 -0
- newswatcher-0.1.0/src/newswatcher/heal.py +195 -0
- newswatcher-0.1.0/src/newswatcher/http.py +130 -0
- newswatcher-0.1.0/src/newswatcher/lock.py +78 -0
- newswatcher-0.1.0/src/newswatcher/match.py +68 -0
- newswatcher-0.1.0/src/newswatcher/poll.py +124 -0
- newswatcher-0.1.0/src/newswatcher/py.typed +0 -0
- newswatcher-0.1.0/src/newswatcher/robots.py +88 -0
- newswatcher-0.1.0/src/newswatcher/schedule.py +330 -0
- newswatcher-0.1.0/src/newswatcher/sources.py +197 -0
- newswatcher-0.1.0/src/newswatcher/state.py +148 -0
- newswatcher-0.1.0/src/newswatcher/store.py +173 -0
- newswatcher-0.1.0/src/newswatcher/stories.py +110 -0
- newswatcher-0.1.0/src/newswatcher/summarize.py +61 -0
- newswatcher-0.1.0/src/newswatcher/topics.py +105 -0
- newswatcher-0.1.0/tests/__init__.py +0 -0
- newswatcher-0.1.0/tests/conftest.py +42 -0
- newswatcher-0.1.0/tests/test_atomic.py +49 -0
- newswatcher-0.1.0/tests/test_body.py +41 -0
- newswatcher-0.1.0/tests/test_cli.py +718 -0
- newswatcher-0.1.0/tests/test_config.py +28 -0
- newswatcher-0.1.0/tests/test_crawl.py +93 -0
- newswatcher-0.1.0/tests/test_credentials.py +332 -0
- newswatcher-0.1.0/tests/test_digest.py +392 -0
- newswatcher-0.1.0/tests/test_feed.py +46 -0
- newswatcher-0.1.0/tests/test_heal.py +134 -0
- newswatcher-0.1.0/tests/test_http.py +289 -0
- newswatcher-0.1.0/tests/test_llm.py +139 -0
- newswatcher-0.1.0/tests/test_lock.py +63 -0
- newswatcher-0.1.0/tests/test_match.py +60 -0
- newswatcher-0.1.0/tests/test_packaging.py +13 -0
- newswatcher-0.1.0/tests/test_poll.py +136 -0
- newswatcher-0.1.0/tests/test_robots.py +58 -0
- newswatcher-0.1.0/tests/test_schedule.py +358 -0
- newswatcher-0.1.0/tests/test_sources.py +44 -0
- newswatcher-0.1.0/tests/test_state.py +142 -0
- newswatcher-0.1.0/tests/test_store.py +158 -0
- newswatcher-0.1.0/tests/test_stories.py +136 -0
- newswatcher-0.1.0/tests/test_summarize.py +68 -0
- newswatcher-0.1.0/tests/test_toml.py +43 -0
- newswatcher-0.1.0/tests/test_topics.py +36 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "newswatcher",
|
|
3
|
+
"interface": { "displayName": "newswatcher" },
|
|
4
|
+
"plugins": [
|
|
5
|
+
{
|
|
6
|
+
"name": "newswatcher",
|
|
7
|
+
"source": { "source": "local", "path": "./plugins/newswatcher" },
|
|
8
|
+
"policy": { "installation": "AVAILABLE", "authentication": "ON_INSTALL" },
|
|
9
|
+
"category": "Productivity"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "newswatcher",
|
|
3
|
+
"owner": {
|
|
4
|
+
"name": "seokhoonj"
|
|
5
|
+
},
|
|
6
|
+
"plugins": [
|
|
7
|
+
{
|
|
8
|
+
"name": "newswatcher",
|
|
9
|
+
"source": "./plugins/newswatcher",
|
|
10
|
+
"description": "Run a newswatcher poll from a Claude Code conversation: collect new articles from your feeds, summarize them with an LLM, and deliver the topic digest by email or chat. Calls the newswatcher package's CLI."
|
|
11
|
+
}
|
|
12
|
+
]
|
|
13
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Normalize line endings: text files are stored as LF in the repository no
|
|
2
|
+
# matter which machine commits. This repo's .git is shared across a Linux box
|
|
3
|
+
# and a Windows box (via Dropbox), so without this the Windows CRLF working
|
|
4
|
+
# tree shows up as whole-file phantom diffs on Linux. Binary files are left
|
|
5
|
+
# untouched by git's own detection.
|
|
6
|
+
#
|
|
7
|
+
# `eol=lf` makes the *working tree* LF too, and it is not tidiness here: Dropbox
|
|
8
|
+
# syncs the checked-out files, not the index, so a CRLF checkout on Windows is
|
|
9
|
+
# what the Linux box then opens. `text=auto` alone left the two boxes disagreeing
|
|
10
|
+
# about every line of every file either had written.
|
|
11
|
+
* text=auto eol=lf
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: check
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
push:
|
|
6
|
+
branches: [main]
|
|
7
|
+
pull_request:
|
|
8
|
+
|
|
9
|
+
# A newer push to the same branch or PR makes the older run's result obsolete,
|
|
10
|
+
# so cancel it rather than let both run to completion.
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
# The check only reads the code to lint, type, and test it -- it never writes to the
|
|
16
|
+
# repo -- so it runs with the least privilege that still allows a checkout.
|
|
17
|
+
permissions:
|
|
18
|
+
contents: read
|
|
19
|
+
|
|
20
|
+
jobs:
|
|
21
|
+
check:
|
|
22
|
+
# The full check lives in reusable-check.yml so one definition serves the push / PR
|
|
23
|
+
# check and any release gate that later reuses it.
|
|
24
|
+
uses: ./.github/workflows/reusable-check.yml
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
# Publish to PyPI on a GitHub Release, via PyPI Trusted Publishing (OIDC): no API
|
|
4
|
+
# token or password is stored anywhere -- GitHub mints a short-lived identity token
|
|
5
|
+
# that PyPI verifies against the publisher registered for this repo (owner seokhoonj
|
|
6
|
+
# / repo newswatcher / this workflow / environment "pypi").
|
|
7
|
+
#
|
|
8
|
+
# Three jobs. `gate` runs the same reusable check that check.yml runs, on the
|
|
9
|
+
# release commit, so a red build (a lint/type error, a broken test, a lost py.typed
|
|
10
|
+
# or console script) cannot reach an upload. `build` builds the sdist and wheel and
|
|
11
|
+
# hands them to `publish` as an artifact. `publish` -- the only job holding the OIDC
|
|
12
|
+
# token -- downloads that artifact and uploads it, and checks out no source, so the
|
|
13
|
+
# privileged step runs the least code.
|
|
14
|
+
|
|
15
|
+
on:
|
|
16
|
+
release:
|
|
17
|
+
types: [published]
|
|
18
|
+
|
|
19
|
+
# Workflow-level floor: any job without its own `permissions:` block still gets a
|
|
20
|
+
# read-only token, so a job added later cannot silently inherit a write-capable
|
|
21
|
+
# default. The build and publish jobs narrow it further below.
|
|
22
|
+
permissions:
|
|
23
|
+
contents: read
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
gate:
|
|
27
|
+
uses: ./.github/workflows/reusable-check.yml
|
|
28
|
+
|
|
29
|
+
build:
|
|
30
|
+
needs: gate
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
timeout-minutes: 15
|
|
33
|
+
permissions:
|
|
34
|
+
contents: read
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
37
|
+
with:
|
|
38
|
+
persist-credentials: false
|
|
39
|
+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
40
|
+
- name: Build sdist and wheel, then check the metadata
|
|
41
|
+
run: |
|
|
42
|
+
uv build
|
|
43
|
+
uvx twine check dist/*
|
|
44
|
+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
45
|
+
with:
|
|
46
|
+
name: dist
|
|
47
|
+
path: dist/
|
|
48
|
+
|
|
49
|
+
publish:
|
|
50
|
+
needs: build
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
timeout-minutes: 15
|
|
53
|
+
environment: pypi
|
|
54
|
+
permissions:
|
|
55
|
+
id-token: write # OIDC; this is what replaces a stored token
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
58
|
+
with:
|
|
59
|
+
name: dist
|
|
60
|
+
path: dist/
|
|
61
|
+
- uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
name: reusable check
|
|
2
|
+
|
|
3
|
+
# The full check -- tests, lint, types, and the packaging assertions -- factored into
|
|
4
|
+
# one reusable job so a push / PR check and any release gate that later reuses it run
|
|
5
|
+
# the *identical* thing rather than two copies that drift.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
workflow_call:
|
|
9
|
+
|
|
10
|
+
# The floor lives on the shared body so every caller runs the check read-only. A
|
|
11
|
+
# reusable workflow can only downscope from its caller, so this never widens a token.
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
check:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
timeout-minutes: 15
|
|
19
|
+
strategy:
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
# The floor and the current release. requires-python says >=3.11, so 3.11 is the
|
|
23
|
+
# version that claim has to hold on; the upper end tracks whatever CPython ships.
|
|
24
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
25
|
+
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
28
|
+
with:
|
|
29
|
+
persist-credentials: false
|
|
30
|
+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
31
|
+
|
|
32
|
+
- name: Install
|
|
33
|
+
# A venv per matrix version, not `uv pip --system`: the runner's system Python
|
|
34
|
+
# is externally managed and is one fixed version whatever the matrix says. This
|
|
35
|
+
# is also what the README tells a person to run.
|
|
36
|
+
run: |
|
|
37
|
+
uv venv --python ${{ matrix.python-version }}
|
|
38
|
+
uv pip install -e . --group dev
|
|
39
|
+
.venv/bin/python -c "import sys; print('testing on', sys.version)"
|
|
40
|
+
|
|
41
|
+
- name: Test
|
|
42
|
+
run: .venv/bin/pytest -q
|
|
43
|
+
|
|
44
|
+
- name: Lint
|
|
45
|
+
run: .venv/bin/ruff check src tests
|
|
46
|
+
|
|
47
|
+
- name: Types
|
|
48
|
+
run: .venv/bin/mypy
|
|
49
|
+
|
|
50
|
+
- name: Confirm the package imports with only its declared dependencies
|
|
51
|
+
# newswatcher pulls in third-party runtime deps (feedparser, bs4, requests, ...).
|
|
52
|
+
# Installed into a bare environment with nothing but those declared deps, it must
|
|
53
|
+
# import and its CLI must run -- so an import the pyproject forgot to declare
|
|
54
|
+
# fails here rather than in a user's fresh install.
|
|
55
|
+
run: |
|
|
56
|
+
uv venv /tmp/bare --python ${{ matrix.python-version }}
|
|
57
|
+
uv pip install --python /tmp/bare/bin/python .
|
|
58
|
+
/tmp/bare/bin/python -c "import newswatcher; print('newswatcher', newswatcher.__version__, 'imports')"
|
|
59
|
+
/tmp/bare/bin/newswatcher --help > /dev/null
|
|
60
|
+
|
|
61
|
+
- name: Confirm the console script installed
|
|
62
|
+
# pyproject declares a `newswatcher` entry point; a wheel that dropped it would
|
|
63
|
+
# leave the skill and the docs pointing at a command that is not there.
|
|
64
|
+
run: |
|
|
65
|
+
uv venv /tmp/cli --python ${{ matrix.python-version }}
|
|
66
|
+
uv pip install --python /tmp/cli/bin/python .
|
|
67
|
+
/tmp/cli/bin/newswatcher --version
|
|
68
|
+
|
|
69
|
+
- name: Confirm py.typed ships in the built package
|
|
70
|
+
# The hints are invisible to a user's type checker unless py.typed rides along
|
|
71
|
+
# (PEP 561), and the source cannot answer whether the wheel kept it: the file can
|
|
72
|
+
# sit in git while the build omits it. Install the built package and look for it.
|
|
73
|
+
run: |
|
|
74
|
+
uv venv /tmp/typed --python ${{ matrix.python-version }}
|
|
75
|
+
uv pip install --python /tmp/typed/bin/python .
|
|
76
|
+
/tmp/typed/bin/python -c "
|
|
77
|
+
import importlib.util, pathlib
|
|
78
|
+
spec = importlib.util.find_spec('newswatcher')
|
|
79
|
+
root = pathlib.Path(spec.origin).parent
|
|
80
|
+
assert (root / 'py.typed').is_file(), 'py.typed missing from the installed package'
|
|
81
|
+
print('py.typed ships alongside newswatcher')
|
|
82
|
+
"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
.env
|
|
11
|
+
|
|
12
|
+
# uv
|
|
13
|
+
uv.lock
|
|
14
|
+
|
|
15
|
+
# Tooling caches
|
|
16
|
+
.pytest_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
.mypy_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
htmlcov/
|
|
21
|
+
|
|
22
|
+
# Local scratch (never tracked)
|
|
23
|
+
dev/
|
|
24
|
+
refs/
|
|
25
|
+
|
|
26
|
+
# AI coding agents
|
|
27
|
+
CLAUDE.md
|
|
28
|
+
.claude/
|
|
29
|
+
AGENTS.md
|
|
30
|
+
AGENT.md
|
|
31
|
+
.codex/
|
|
32
|
+
GEMINI.md
|
|
33
|
+
.gemini/
|
|
34
|
+
|
|
35
|
+
docs/superpowers/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 seokhoonj
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: newswatcher
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Collect topic-matched articles from RSS and robots-permitted crawl sources, summarize them, and mail a digest.
|
|
5
|
+
Project-URL: Homepage, https://github.com/seokhoonj/newswatcher
|
|
6
|
+
Project-URL: Repository, https://github.com/seokhoonj/newswatcher
|
|
7
|
+
Project-URL: Issues, https://github.com/seokhoonj/newswatcher/issues
|
|
8
|
+
Author-email: Seokhoon Joo <seokhoonj@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: crawl,digest,email,news,rss,topics,watch
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Communications :: Email
|
|
22
|
+
Classifier: Topic :: Internet
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Requires-Dist: beautifulsoup4>=4.12
|
|
26
|
+
Requires-Dist: credbox<0.5,>=0.4.0
|
|
27
|
+
Requires-Dist: feedparser>=6.0
|
|
28
|
+
Requires-Dist: lxml>=5.0
|
|
29
|
+
Requires-Dist: requests>=2.31
|
|
30
|
+
Requires-Dist: soupsieve>=2.4
|
|
31
|
+
Requires-Dist: thinchat<0.4,>=0.3.1
|
|
32
|
+
Requires-Dist: trafilatura>=1.8
|
|
33
|
+
Provides-Extra: all
|
|
34
|
+
Requires-Dist: mailmail>=0.4.2; extra == 'all'
|
|
35
|
+
Requires-Dist: pushpush>=0.3.1; extra == 'all'
|
|
36
|
+
Provides-Extra: chat
|
|
37
|
+
Requires-Dist: pushpush>=0.3.1; extra == 'chat'
|
|
38
|
+
Provides-Extra: email
|
|
39
|
+
Requires-Dist: mailmail>=0.4.2; extra == 'email'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# newswatcher
|
|
43
|
+
|
|
44
|
+
[](https://github.com/seokhoonj/newswatcher/actions/workflows/check.yml)
|
|
45
|
+
[](https://pypi.org/project/newswatcher/)
|
|
46
|
+
[](https://pypi.org/project/newswatcher/)
|
|
47
|
+
[](https://github.com/seokhoonj/newswatcher/blob/main/LICENSE)
|
|
48
|
+
|
|
49
|
+
**English** | [한국어](README.ko.md)
|
|
50
|
+
|
|
51
|
+
newswatcher watches RSS feeds and robots-permitted listing pages, matches new
|
|
52
|
+
articles against topics you define, summarizes the matches with an LLM, and
|
|
53
|
+
sends one topic-grouped digest by email, chat, or both. Delivery is optional — the
|
|
54
|
+
core collects, summarizes, and archives on its own; email and chat are opt-in add-ons
|
|
55
|
+
(`newswatcher[email]` / `newswatcher[chat]`). Several outlets covering the same story
|
|
56
|
+
collapse into a single entry. The topics are yours to define, so
|
|
57
|
+
the same tool tracks a stock ticker, a technology, a policy beat, or any subject
|
|
58
|
+
a feed covers.
|
|
59
|
+
|
|
60
|
+
## 1. Install
|
|
61
|
+
|
|
62
|
+
newswatcher requires Python 3.11 or newer. The core — collect, summarize, archive — installs on
|
|
63
|
+
its own; delivery is opt-in, so add the channel you want:
|
|
64
|
+
|
|
65
|
+
```sh
|
|
66
|
+
pip install newswatcher # core: collect, summarize, archive (read with `articles`)
|
|
67
|
+
pip install "newswatcher[email]" # + email digests, via mailmail
|
|
68
|
+
pip install "newswatcher[chat]" # + chat digests, via pushpush
|
|
69
|
+
pip install "newswatcher[all]" # + both
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The Quickstart below needs only the core `newswatcher` — email and chat come later (Delivery).
|
|
73
|
+
|
|
74
|
+
## 2. Quickstart
|
|
75
|
+
|
|
76
|
+
These are terminal commands (a shell — Terminal, PowerShell, or Command Prompt — not the
|
|
77
|
+
Python prompt). First get a free Gemini API key from
|
|
78
|
+
[Google AI Studio](https://aistudio.google.com/apikey) (sign in with Google, click **Create
|
|
79
|
+
API key**, copy it), then store it — you are prompted, and the key is not echoed:
|
|
80
|
+
|
|
81
|
+
```sh
|
|
82
|
+
newswatcher set-key gemini
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Define a topic, register an RSS source, run one pass, and read the summaries — no email setup
|
|
86
|
+
needed to get a first result:
|
|
87
|
+
|
|
88
|
+
```sh
|
|
89
|
+
newswatcher add-topic markets --include stocks Fed "interest rate" earnings --exclude sports
|
|
90
|
+
newswatcher add-source korea-herald "https://www.koreaherald.com/rss/newsAll" --kind rss --topic markets
|
|
91
|
+
newswatcher poll --no-mail
|
|
92
|
+
newswatcher articles
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
If `articles` shows nothing, the poll just matched no new articles yet — that is normal, not an
|
|
96
|
+
error; broaden the `--include` keywords or run `poll` again later.
|
|
97
|
+
|
|
98
|
+
A topic matches on the feed's own language, so pair the keywords with the feed: English
|
|
99
|
+
keywords for an English feed, Korean keywords for a Korean feed. `newswatcher topics` and
|
|
100
|
+
`newswatcher sources` show what you registered.
|
|
101
|
+
|
|
102
|
+
To have the digest **emailed** or sent to **chat** instead of read with `articles`, install the
|
|
103
|
+
delivery extra and configure the channel once — see [Delivery](#4-delivery).
|
|
104
|
+
|
|
105
|
+
## 3. Commands
|
|
106
|
+
|
|
107
|
+
Run `newswatcher --help` or `newswatcher <command> --help` for every option;
|
|
108
|
+
`newswatcher --version` prints the version.
|
|
109
|
+
|
|
110
|
+
| Command | What it does |
|
|
111
|
+
|---------|--------------|
|
|
112
|
+
| `add-topic <name> [--include WORD...] [--exclude WORD...]` | Define a topic filter: a name, `--include` keywords (an article matches when it has any one of them), and optional `--exclude` keywords (any one rejects it). Empty includes match every article. |
|
|
113
|
+
| `topics` | List the defined topics with their include / exclude keywords. |
|
|
114
|
+
| `add-source <name> <url> [--kind rss\|crawl] [--topic NAME]... [--keep-all]` | Register a source — an RSS feed (`--kind rss`) or a robots-permitted crawl page (`--kind crawl`) — and the `--topic`s to test it against. A crawl source also needs selectors: `--item --title --link` (required), `--date --body-selector` (optional). `--keep-all` keeps every article from the source without keyword filtering (for a trade feed that is wholly on-topic). |
|
|
115
|
+
| `sources` | List the registered sources with their kind, URL, and topics. |
|
|
116
|
+
| `set-key <provider>` | Store an LLM provider's API key with thinchat, prompted without echo (in thinchat's own credential store). The provider is `gemini`, `openai`, or `claude`; the key is also read from the matching `*_API_KEY` environment variable, which takes precedence. |
|
|
117
|
+
| `setup [--provider P]` | Fill in the missing secrets for every channel in one guided pass — the LLM key with thinchat, each email password with mailmail, each chat token with pushpush — prompting without echo and printing where each landed. Skips what is already set, and points you at the tool to configure a channel that has no account or route yet. |
|
|
118
|
+
| `doctor [--provider P]` | Show where each secret and config file lives and whether it is set, without printing any secret. Exits non-zero when a configured channel is missing its secret or its store is unreadable, so a scheduled run can gate on a complete setup. |
|
|
119
|
+
| `recent <url> [--limit N]` | Fetch and print a feed's latest items (title + link) without storing or summarizing — a quick check of a URL before you register it. `--limit N` caps how many. |
|
|
120
|
+
| `poll` | Run one pass: fetch every source, keep the new articles that match a topic, summarize them, archive them, and send the digest. `--to` / `--push` set destinations; `--no-mail` collects without sending; `--no-store` skips archiving; `--no-heal` skips selector repair; `--provider` / `--model` choose the LLM. |
|
|
121
|
+
| `watch [--every N]` | Run `poll` repeatedly in the foreground, every `--every` minutes (default 30), until you stop it. Takes all of `poll`'s options. |
|
|
122
|
+
| `articles [--topic NAME] [--since DATE] [--until DATE]` | List archived articles (title, our summary, link), optionally narrowed to a topic and a half-open `[since, until)` date range. |
|
|
123
|
+
| `heal [--dry-run] [--provider P] [--model M]` | Check crawl sources whose selectors stopped matching and repair them with an LLM, validated against the live page. `--dry-run` reports the proposed fix without writing it. |
|
|
124
|
+
| `schedule install\|status\|remove [--every N]` | Install, show, or remove the recurring poll in the OS scheduler (cron on Linux/macOS, schtasks on Windows). `--every N` sets the interval. |
|
|
125
|
+
|
|
126
|
+
## 4. Delivery
|
|
127
|
+
|
|
128
|
+
The digest is sent by email, to a chat channel, or both — set one or both destinations.
|
|
129
|
+
Each channel is an opt-in extra (`newswatcher[email]` / `newswatcher[chat]`); its companion
|
|
130
|
+
package keeps its own credentials, so newswatcher never stores your email password or bot token.
|
|
131
|
+
Without either extra, newswatcher still collects, summarizes, and archives — read the archive with
|
|
132
|
+
`newswatcher articles`.
|
|
133
|
+
|
|
134
|
+
- Email goes through the mailmail package (`newswatcher[email]`). Set up an account (or an
|
|
135
|
+
address-book alias) once with mailmail's own CLI (`mailmail --help`); then `--to ADDRESS`, or the
|
|
136
|
+
`NEWSWATCHER_DIGEST_TO` setting, names that alias or a plain address.
|
|
137
|
+
- Chat goes through the pushpush package (`newswatcher[chat]`). Configure a route (a bot plus its
|
|
138
|
+
destination — Telegram, Slack, or Discord) once with pushpush's own CLI (`pushpush --help`); then
|
|
139
|
+
`--push ROUTE`, or the `NEWSWATCHER_DIGEST_PUSH` setting, names that route. The digest is sent as
|
|
140
|
+
one markdown message.
|
|
141
|
+
|
|
142
|
+
So newswatcher holds no secret of its own: the LLM key lives with thinchat, the email password
|
|
143
|
+
with mailmail, the chat token with pushpush — each in its own store, exactly as when the tool is
|
|
144
|
+
used on its own. Configure them all in one guided pass with `newswatcher setup`, and see the full
|
|
145
|
+
map — what is set and where it lives — with `newswatcher doctor`.
|
|
146
|
+
|
|
147
|
+
## 5. News feeds
|
|
148
|
+
|
|
149
|
+
Any valid RSS/Atom feed works as a source. A representative set of verified
|
|
150
|
+
Korean feeds is below; the full list, grouped by section and marked with which
|
|
151
|
+
were live at verification, is in [docs/korean-news-rss.md](docs/korean-news-rss.md).
|
|
152
|
+
A site with no feed can still be followed with a `--kind crawl` source.
|
|
153
|
+
|
|
154
|
+
| Outlet | Beat | Feed URL |
|
|
155
|
+
|--------|------|----------|
|
|
156
|
+
| 연합뉴스 (Yonhap) | wire | `https://www.yna.co.kr/rss/news.xml` |
|
|
157
|
+
| 한국경제 (Hankyung) | economy | `https://www.hankyung.com/feed/all-news` |
|
|
158
|
+
| 조선비즈 (ChosunBiz) | economy | `https://biz.chosun.com/arc/outboundfeeds/rss/?outputType=xml` |
|
|
159
|
+
| 매일경제 (Maeil) | economy | `https://www.mk.co.kr/rss/30000001/` |
|
|
160
|
+
| 이데일리 (Edaily) | economy | `http://rss.edaily.co.kr/edaily_news.xml` |
|
|
161
|
+
| 머니투데이 (MoneyToday) | economy | `http://rss.mt.co.kr/mt_news.xml` |
|
|
162
|
+
| 전자신문 (ETNews) | tech | `https://rss.etnews.com/Section901.xml` |
|
|
163
|
+
| 지디넷코리아 (ZDNet Korea) | tech | `https://feeds.feedburner.com/zdkorea` |
|
|
164
|
+
| The Korea Herald | English | `https://www.koreaherald.com/rss/newsAll` |
|
|
165
|
+
| The Korea Times | English | `https://feed.koreatimes.co.kr/k/allnews.xml` |
|
|
166
|
+
|
|
167
|
+
A representative set of verified international (English) feeds is below; the full
|
|
168
|
+
list, grouped by category and marked with which outlets paywall the article body,
|
|
169
|
+
is in [docs/world-news-rss.md](docs/world-news-rss.md). Pair these with English
|
|
170
|
+
topic keywords.
|
|
171
|
+
|
|
172
|
+
| Outlet | Beat | Feed URL |
|
|
173
|
+
|--------|------|----------|
|
|
174
|
+
| BBC News | wire | `https://feeds.bbci.co.uk/news/world/rss.xml` |
|
|
175
|
+
| The Guardian | wire | `https://www.theguardian.com/world/rss` |
|
|
176
|
+
| Al Jazeera | wire | `https://www.aljazeera.com/xml/rss/all.xml` |
|
|
177
|
+
| The New York Times | world | `https://rss.nytimes.com/services/xml/rss/nyt/World.xml` |
|
|
178
|
+
| CNBC | economy | `https://www.cnbc.com/id/100003114/device/rss/rss.html` |
|
|
179
|
+
| MarketWatch | economy | `http://feeds.marketwatch.com/marketwatch/topstories/` |
|
|
180
|
+
| TechCrunch | tech | `https://techcrunch.com/feed/` |
|
|
181
|
+
| The Verge | tech | `https://www.theverge.com/rss/index.xml` |
|
|
182
|
+
| Nature | science | `https://www.nature.com/nature.rss` |
|
|
183
|
+
|
|
184
|
+
## 6. Configuration files
|
|
185
|
+
|
|
186
|
+
newswatcher stores hand-edited configuration under
|
|
187
|
+
`$XDG_CONFIG_HOME/newswatcher`, or `~/.config/newswatcher` when
|
|
188
|
+
`XDG_CONFIG_HOME` is unset. The CLI writes the same files, so CLI and manual
|
|
189
|
+
configuration can be mixed.
|
|
190
|
+
|
|
191
|
+
`topics.toml` contains topic filters. An article matches when its title or feed
|
|
192
|
+
summary contains any include keyword and no exclude keyword. An empty
|
|
193
|
+
`includes` list matches every article.
|
|
194
|
+
|
|
195
|
+
```toml
|
|
196
|
+
[[topic]]
|
|
197
|
+
name = "markets"
|
|
198
|
+
includes = ["stocks", "Fed", "interest rate", "earnings"]
|
|
199
|
+
excludes = ["sports"]
|
|
200
|
+
|
|
201
|
+
[[topic]]
|
|
202
|
+
name = "semiconductors"
|
|
203
|
+
includes = ["chip", "foundry", "HBM", "TSMC", "Nvidia"]
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
`sources.toml` contains RSS or crawl sources. `topics` selects the topic filters
|
|
207
|
+
applied to a source. Set `keep_all = true` for a source whose every article
|
|
208
|
+
should be retained without keyword filtering.
|
|
209
|
+
|
|
210
|
+
```toml
|
|
211
|
+
[[source]]
|
|
212
|
+
name = "korea-herald"
|
|
213
|
+
kind = "rss"
|
|
214
|
+
url = "https://www.koreaherald.com/rss/newsAll"
|
|
215
|
+
topics = ["markets", "semiconductors"]
|
|
216
|
+
|
|
217
|
+
[[source]]
|
|
218
|
+
name = "exchange-notices"
|
|
219
|
+
kind = "crawl"
|
|
220
|
+
url = "https://example.com/markets/notices"
|
|
221
|
+
topics = ["markets"]
|
|
222
|
+
item = "article.news-item"
|
|
223
|
+
title = "h2"
|
|
224
|
+
link = "a@href"
|
|
225
|
+
date = "time"
|
|
226
|
+
body_selector = "main article"
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
The `item`, `title`, and `link` selectors are required for crawl sources;
|
|
230
|
+
`date` and `body_selector` are optional. The link selector uses the
|
|
231
|
+
`css@attribute` form when the URL is stored in an attribute.
|
|
232
|
+
|
|
233
|
+
Non-secret settings can also be placed in `config.toml`. Environment variables
|
|
234
|
+
take precedence over corresponding settings there. For example,
|
|
235
|
+
`NEWSWATCHER_DIGEST_TO` maps to `digest_to`, and `NEWSWATCHER_DIGEST_PUSH` to `digest_push`.
|
|
236
|
+
`NEWSWATCHER_DEDUP_THRESHOLD` (`dedup_threshold`, 0.0–1.0, default 0.5) sets how alike two
|
|
237
|
+
headlines must be to collapse as one story — raise it to merge less, lower it to merge
|
|
238
|
+
more. The article archive and run state use
|
|
239
|
+
the XDG data and state directories; `NEWSWATCHER_DATA_DIR` and
|
|
240
|
+
`NEWSWATCHER_STATE_DIR` can override them. The archive deletes nothing by default; to
|
|
241
|
+
prune old records, set `NEWSWATCHER_ARCHIVE_KEEP_DAYS` (`archive_keep_days`, a positive
|
|
242
|
+
integer) and each poll removes archived articles older than that after the digest is
|
|
243
|
+
sent. Leaving it unset keeps everything (this deletion is irreversible, so enable it
|
|
244
|
+
deliberately).
|
|
245
|
+
|
|
246
|
+
## 7. Provider keys and model
|
|
247
|
+
|
|
248
|
+
A **provider** is the LLM service that writes the summaries. newswatcher supports four; pass the
|
|
249
|
+
name in the left column to `--provider` or `set-key`:
|
|
250
|
+
|
|
251
|
+
| Provider | Key (environment variable) | Where to get a key |
|
|
252
|
+
|----------|----------------------------|--------------------|
|
|
253
|
+
| `gemini` (default) | `GEMINI_API_KEY` | [Google AI Studio](https://aistudio.google.com/apikey) — free tier |
|
|
254
|
+
| `openai` | `OPENAI_API_KEY` | [platform.openai.com](https://platform.openai.com/api-keys) |
|
|
255
|
+
| `claude` | `CLAUDE_API_KEY` | [console.anthropic.com](https://console.anthropic.com/settings/keys) |
|
|
256
|
+
| `ollama` | — (runs locally) | no key needed |
|
|
257
|
+
|
|
258
|
+
The provider key is a secret, and it lives with thinchat — the library newswatcher
|
|
259
|
+
summarizes through — not with newswatcher. Store it once with `setup` (which configures
|
|
260
|
+
email and chat in the same pass) or with `set-key` for the key alone; both prompt without
|
|
261
|
+
echoing and write to thinchat's own credential store:
|
|
262
|
+
|
|
263
|
+
```sh
|
|
264
|
+
newswatcher setup # the LLM key, plus email and chat, in one pass
|
|
265
|
+
newswatcher set-key gemini # just the LLM key
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Each key is also read from its environment variable (the table above), which takes precedence,
|
|
269
|
+
so a one-off run can supply a key without storing anything.
|
|
270
|
+
|
|
271
|
+
newswatcher summarizes with Gemini's free tier by default. Choose another provider,
|
|
272
|
+
and optionally a specific model, with `--provider` / `--model`, or persistently
|
|
273
|
+
with the `NEWSWATCHER_LLM_PROVIDER` / `NEWSWATCHER_LLM_MODEL` settings (`llm_provider`
|
|
274
|
+
and `llm_model` in `config.toml`):
|
|
275
|
+
|
|
276
|
+
```sh
|
|
277
|
+
newswatcher poll --provider claude --model claude-sonnet-5
|
|
278
|
+
export NEWSWATCHER_LLM_PROVIDER=openai
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
## 8. Responsible collection
|
|
282
|
+
|
|
283
|
+
Every feed, listing-page, and article request is checked against the site's
|
|
284
|
+
robots policy before it is sent, and newswatcher identifies itself with its user
|
|
285
|
+
agent. A disallowed URL is not fetched. The durable archive and outbound digest
|
|
286
|
+
contain the LLM-written summary, source link, and metadata only. Raw article
|
|
287
|
+
bodies are transient summary input and are neither archived nor sent.
|
|
288
|
+
|
|
289
|
+
## 9. Scheduling
|
|
290
|
+
|
|
291
|
+
Install a recurring poll every 30 minutes:
|
|
292
|
+
|
|
293
|
+
```sh
|
|
294
|
+
newswatcher schedule install
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Choose another interval with minutes, `Nm`, or `Nh`, and inspect or remove the
|
|
298
|
+
job as needed:
|
|
299
|
+
|
|
300
|
+
```sh
|
|
301
|
+
newswatcher schedule install --every 2h
|
|
302
|
+
newswatcher schedule status
|
|
303
|
+
newswatcher schedule remove
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
Scheduling uses `crontab` on Linux and macOS and `schtasks` on Windows. On Windows
|
|
307
|
+
any interval under a day works (`--every 45`, `--every 5h`); on Linux and macOS cron
|
|
308
|
+
only fires intervals that divide evenly (15/20/30 min, 1/2/4/8/12 h, or daily) and
|
|
309
|
+
rejects the rest rather than mis-scheduling them. The scheduled process uses the same
|
|
310
|
+
configuration as an interactive poll, so make sure the LLM key is reachable (from
|
|
311
|
+
thinchat's store or its environment variable) along with any settings not stored
|
|
312
|
+
in `config.toml`. `newswatcher doctor` confirms it before you schedule.
|
|
313
|
+
|
|
314
|
+
On Windows the task is registered under the installing user and runs in their
|
|
315
|
+
interactive session, so it does not fire while nobody is signed in — a locked screen is
|
|
316
|
+
fine, a machine sitting at the sign-in screen is not. It also inherits the Task Scheduler
|
|
317
|
+
default of not starting on battery power. Check it with
|
|
318
|
+
`schtasks /Query /TN newswatcher-poll`. On Linux and macOS the cron job has neither
|
|
319
|
+
restriction.
|
|
320
|
+
|
|
321
|
+
A poll takes a single-instance lock, so a scheduled poll and a manual one never run
|
|
322
|
+
at once — whichever starts second logs that a poll is already running and exits. The
|
|
323
|
+
lock uses `flock` on Linux and macOS and `msvcrt` on Windows.
|
|
324
|
+
|
|
325
|
+
## 10. Use it from an AI coding agent
|
|
326
|
+
|
|
327
|
+
This repo ships a `poll` skill: ask in plain words ("run my newswatcher poll", "check the
|
|
328
|
+
news") and it runs one poll and relays what it found.
|
|
329
|
+
|
|
330
|
+
### Claude Code
|
|
331
|
+
|
|
332
|
+
In the Claude Code chat, add the marketplace and install:
|
|
333
|
+
|
|
334
|
+
```
|
|
335
|
+
/plugin marketplace add seokhoonj/newswatcher
|
|
336
|
+
/plugin install newswatcher@newswatcher
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
Then invoke it with `/newswatcher:poll`, or just ask in plain language. The skill calls the
|
|
340
|
+
`newswatcher` command, so install the package too (`pip install newswatcher`). See
|
|
341
|
+
`plugins/newswatcher/skills/poll/SKILL.md`.
|
|
342
|
+
|
|
343
|
+
### Codex
|
|
344
|
+
|
|
345
|
+
In your terminal, add the marketplace and install:
|
|
346
|
+
|
|
347
|
+
```
|
|
348
|
+
codex plugin marketplace add seokhoonj/newswatcher
|
|
349
|
+
codex plugin add newswatcher@newswatcher
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
The `poll` skill responds automatically to matching requests.
|
|
353
|
+
|
|
354
|
+
### By hand (symlink)
|
|
355
|
+
|
|
356
|
+
Symlink the skill into your skills directory and call it as `/poll`:
|
|
357
|
+
|
|
358
|
+
```sh
|
|
359
|
+
ln -s "$PWD/plugins/newswatcher/skills/poll" ~/.claude/skills/poll # Claude Code -> /poll
|
|
360
|
+
ln -s "$PWD/plugins/newswatcher/skills/poll" ~/.codex/skills/poll # Codex -> $newswatcher:poll
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
Claude Code picks it up immediately; Codex needs a restart to load it.
|
|
364
|
+
|
|
365
|
+
## 11. License
|
|
366
|
+
|
|
367
|
+
[MIT](LICENSE)
|