unseenet-bots 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unseenet_bots-0.1.0/LICENSE +177 -0
- unseenet_bots-0.1.0/PKG-INFO +117 -0
- unseenet_bots-0.1.0/README.md +101 -0
- unseenet_bots-0.1.0/pyproject.toml +37 -0
- unseenet_bots-0.1.0/setup.cfg +4 -0
- unseenet_bots-0.1.0/src/unseenet_bots/__init__.py +4 -0
- unseenet_bots-0.1.0/src/unseenet_bots/runner.py +208 -0
- unseenet_bots-0.1.0/src/unseenet_bots/sources/__init__.py +237 -0
- unseenet_bots-0.1.0/src/unseenet_bots/sources/feeds.py +208 -0
- unseenet_bots-0.1.0/src/unseenet_bots/sources/wikipedia.py +97 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/PKG-INFO +117 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/SOURCES.txt +16 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/dependency_links.txt +1 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/entry_points.txt +2 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/requires.txt +5 -0
- unseenet_bots-0.1.0/src/unseenet_bots.egg-info/top_level.txt +1 -0
- unseenet_bots-0.1.0/tests/test_bots.py +82 -0
- unseenet_bots-0.1.0/tests/test_feeds.py +80 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: unseenet-bots
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bots that fill Unseenet channels with current content, built on the Unseenet core
|
|
5
|
+
Author: Andreas Lorensen
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/unseenet/unseenet-bots
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: unseenet<0.5,>=0.4.1
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# unseenet-bots
|
|
18
|
+
|
|
19
|
+
Bots that keep [Unseenet](https://unseenet.org) channels filled with
|
|
20
|
+
current content, and an example of how to build a bot on the Unseenet
|
|
21
|
+
core library.
|
|
22
|
+
|
|
23
|
+
**Status:** early. Six sources work, listed below.
|
|
24
|
+
|
|
25
|
+
## Idea
|
|
26
|
+
|
|
27
|
+
- One bot is one key, a *poster* in a channel (never the owner), with
|
|
28
|
+
`bot: true` in its profile (UN-RFC 0030 section 6).
|
|
29
|
+
- One source per bot: fetch, turn into a post with a credit line
|
|
30
|
+
(author, source, licence, link), post on a schedule.
|
|
31
|
+
- Built only on the public core API of
|
|
32
|
+
[unseenet/unseenet](https://github.com/unseenet/unseenet), so it
|
|
33
|
+
doubles as an example for anybody writing their own bot.
|
|
34
|
+
- Readers can comment if the channel has `comments: open`
|
|
35
|
+
(UN-RFC 0030).
|
|
36
|
+
|
|
37
|
+
## Running it
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
pip install unseenet-bots
|
|
41
|
+
cp bots.example.toml bots.toml # fill in groups and key files
|
|
42
|
+
unseenet-bots -c bots.toml new-key ~/.config/unseenet-bots/news.key
|
|
43
|
+
# -> make the printed key a poster in the channel (Make poster)
|
|
44
|
+
unseenet ripen --key-file ~/.config/unseenet-bots/news.key --to 0 \
|
|
45
|
+
--node NODE.onion --publish # once: 7-11 minutes, see below
|
|
46
|
+
unseenet-bots -c bots.toml run --dry-run # see what would be posted
|
|
47
|
+
unseenet-bots -c bots.toml run # post what is new
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**A new key ripens first** (UN-RFC 0025): nodes take posts only from keys
|
|
51
|
+
that have done a few minutes of work, which is what keeps throwaway keys
|
|
52
|
+
from spamming. Do it once at setup with `unseenet ripen` as above; its
|
|
53
|
+
chain goes in `<key file>.state.json`, beside the key, where the runner
|
|
54
|
+
finds it (it holds nothing secret). A first `run` without it would ripen
|
|
55
|
+
then, and take as long. Ripening on to level 3 removes the extra postage
|
|
56
|
+
a young key pays: `unseenet ripen --key-file KEY --to 3 --node NODE --publish`
|
|
57
|
+
in the background, about 2.5-4 days on one core at low priority.
|
|
58
|
+
|
|
59
|
+
Each `run` fetches every source once and posts what has not been posted
|
|
60
|
+
before (remembered in a state file, and checked against the channel's
|
|
61
|
+
newest threads). Schedule it, e.g. every hour, with the systemd units in
|
|
62
|
+
`contrib/`. It starts its own Tor, or uses a running one if
|
|
63
|
+
`UNSEENET_TOR_SOCKS` names its SOCKS port. Sources are fetched directly
|
|
64
|
+
from the web, not over Tor.
|
|
65
|
+
|
|
66
|
+
| Source | Posts | Licence |
|
|
67
|
+
|---|---|---|
|
|
68
|
+
| `wikipedia-news` | one post per story in Wikipedia's *In the news* | CC BY-SA 4.0 |
|
|
69
|
+
| `wikipedia-onthisday` | one post a day with the day's anniversaries | CC BY-SA 4.0 |
|
|
70
|
+
| `globalvoices` | whole stories from Global Voices, credited at the top | CC BY 3.0 |
|
|
71
|
+
| `theconversation` | the opening paragraphs of articles from The Conversation, and a link to the rest; use `max_per_day`, as their terms do not allow republishing every article | CC BY-ND 4.0 |
|
|
72
|
+
| `eff` | whole posts from EFF's Deeplinks blog | CC BY 4.0 |
|
|
73
|
+
| `nasa-iotd` | NASA's Image of the Day: the caption and a link to the picture | not copyrighted |
|
|
74
|
+
|
|
75
|
+
Images are left out of every post, because they often have licences of
|
|
76
|
+
their own; each post links to the original. Only items from the last two
|
|
77
|
+
days are posted (`max_age_hours`), so a new bot does not fill its channel
|
|
78
|
+
with old news.
|
|
79
|
+
|
|
80
|
+
Open the channel for comments (*Edit group* → *Readers may comment on
|
|
81
|
+
posts*, UN-RFC 0030) so readers can talk about each post.
|
|
82
|
+
|
|
83
|
+
## Writing a source
|
|
84
|
+
|
|
85
|
+
A source is a class with a `name` and `fetch(today) -> list[Item]` in
|
|
86
|
+
`src/unseenet_bots/sources/`. An `Item` has a stable `id` (so nothing is
|
|
87
|
+
posted twice), a `subject` and a markdown `body` that ends with the
|
|
88
|
+
credit line its licence asks for. See `wikipedia.py`, and *Writing a bot*
|
|
89
|
+
in the Unseenet docs for the core API underneath.
|
|
90
|
+
|
|
91
|
+
## Content and licences
|
|
92
|
+
|
|
93
|
+
Only sources whose licences allow reposting are used, and each post
|
|
94
|
+
credits its source as the licence asks. News agencies whose terms
|
|
95
|
+
forbid redistribution are not used. The first sources are listed in
|
|
96
|
+
issue #1.
|
|
97
|
+
|
|
98
|
+
Whoever runs a bot is responsible for its content and for following
|
|
99
|
+
the source's licence.
|
|
100
|
+
|
|
101
|
+
## Versions and releases
|
|
102
|
+
|
|
103
|
+
Versions follow [semantic versioning](https://semver.org/) on the same
|
|
104
|
+
principles as [unseenet](https://github.com/unseenet/unseenet/blob/main/docs/development.md#versioning):
|
|
105
|
+
tags `vX.Y.Z` on `main`, the changes in [CHANGELOG.md](CHANGELOG.md), and
|
|
106
|
+
releases on [PyPI](https://pypi.org/p/unseenet-bots). Each release names the
|
|
107
|
+
minor version of unseenet it works with (`unseenet>=0.4.1,<0.5`).
|
|
108
|
+
|
|
109
|
+
To release: move the entries under `## [Unreleased]` in `CHANGELOG.md` to a
|
|
110
|
+
new `## [X.Y.Z] - YYYY-MM-DD` section, set `__version__` in
|
|
111
|
+
`src/unseenet_bots/__init__.py`, commit, then
|
|
112
|
+
`git tag -a vX.Y.Z -m "unseenet-bots X.Y.Z" && git push origin vX.Y.Z`.
|
|
113
|
+
|
|
114
|
+
## Licence
|
|
115
|
+
|
|
116
|
+
The code is licensed under the Apache License 2.0. Content posted by
|
|
117
|
+
the bots keeps the licence of its source.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# unseenet-bots
|
|
2
|
+
|
|
3
|
+
Bots that keep [Unseenet](https://unseenet.org) channels filled with
|
|
4
|
+
current content, and an example of how to build a bot on the Unseenet
|
|
5
|
+
core library.
|
|
6
|
+
|
|
7
|
+
**Status:** early. Six sources work, listed below.
|
|
8
|
+
|
|
9
|
+
## Idea
|
|
10
|
+
|
|
11
|
+
- One bot is one key, a *poster* in a channel (never the owner), with
|
|
12
|
+
`bot: true` in its profile (UN-RFC 0030 section 6).
|
|
13
|
+
- One source per bot: fetch, turn into a post with a credit line
|
|
14
|
+
(author, source, licence, link), post on a schedule.
|
|
15
|
+
- Built only on the public core API of
|
|
16
|
+
[unseenet/unseenet](https://github.com/unseenet/unseenet), so it
|
|
17
|
+
doubles as an example for anybody writing their own bot.
|
|
18
|
+
- Readers can comment if the channel has `comments: open`
|
|
19
|
+
(UN-RFC 0030).
|
|
20
|
+
|
|
21
|
+
## Running it
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
pip install unseenet-bots
|
|
25
|
+
cp bots.example.toml bots.toml # fill in groups and key files
|
|
26
|
+
unseenet-bots -c bots.toml new-key ~/.config/unseenet-bots/news.key
|
|
27
|
+
# -> make the printed key a poster in the channel (Make poster)
|
|
28
|
+
unseenet ripen --key-file ~/.config/unseenet-bots/news.key --to 0 \
|
|
29
|
+
--node NODE.onion --publish # once: 7-11 minutes, see below
|
|
30
|
+
unseenet-bots -c bots.toml run --dry-run # see what would be posted
|
|
31
|
+
unseenet-bots -c bots.toml run # post what is new
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
**A new key ripens first** (UN-RFC 0025): nodes take posts only from keys
|
|
35
|
+
that have done a few minutes of work, which is what keeps throwaway keys
|
|
36
|
+
from spamming. Do it once at setup with `unseenet ripen` as above; its
|
|
37
|
+
chain goes in `<key file>.state.json`, beside the key, where the runner
|
|
38
|
+
finds it (it holds nothing secret). A first `run` without it would ripen
|
|
39
|
+
then, and take as long. Ripening on to level 3 removes the extra postage
|
|
40
|
+
a young key pays: `unseenet ripen --key-file KEY --to 3 --node NODE --publish`
|
|
41
|
+
in the background, about 2.5-4 days on one core at low priority.
|
|
42
|
+
|
|
43
|
+
Each `run` fetches every source once and posts what has not been posted
|
|
44
|
+
before (remembered in a state file, and checked against the channel's
|
|
45
|
+
newest threads). Schedule it, e.g. every hour, with the systemd units in
|
|
46
|
+
`contrib/`. It starts its own Tor, or uses a running one if
|
|
47
|
+
`UNSEENET_TOR_SOCKS` names its SOCKS port. Sources are fetched directly
|
|
48
|
+
from the web, not over Tor.
|
|
49
|
+
|
|
50
|
+
| Source | Posts | Licence |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| `wikipedia-news` | one post per story in Wikipedia's *In the news* | CC BY-SA 4.0 |
|
|
53
|
+
| `wikipedia-onthisday` | one post a day with the day's anniversaries | CC BY-SA 4.0 |
|
|
54
|
+
| `globalvoices` | whole stories from Global Voices, credited at the top | CC BY 3.0 |
|
|
55
|
+
| `theconversation` | the opening paragraphs of articles from The Conversation, and a link to the rest; use `max_per_day`, as their terms do not allow republishing every article | CC BY-ND 4.0 |
|
|
56
|
+
| `eff` | whole posts from EFF's Deeplinks blog | CC BY 4.0 |
|
|
57
|
+
| `nasa-iotd` | NASA's Image of the Day: the caption and a link to the picture | not copyrighted |
|
|
58
|
+
|
|
59
|
+
Images are left out of every post, because they often have licences of
|
|
60
|
+
their own; each post links to the original. Only items from the last two
|
|
61
|
+
days are posted (`max_age_hours`), so a new bot does not fill its channel
|
|
62
|
+
with old news.
|
|
63
|
+
|
|
64
|
+
Open the channel for comments (*Edit group* → *Readers may comment on
|
|
65
|
+
posts*, UN-RFC 0030) so readers can talk about each post.
|
|
66
|
+
|
|
67
|
+
## Writing a source
|
|
68
|
+
|
|
69
|
+
A source is a class with a `name` and `fetch(today) -> list[Item]` in
|
|
70
|
+
`src/unseenet_bots/sources/`. An `Item` has a stable `id` (so nothing is
|
|
71
|
+
posted twice), a `subject` and a markdown `body` that ends with the
|
|
72
|
+
credit line its licence asks for. See `wikipedia.py`, and *Writing a bot*
|
|
73
|
+
in the Unseenet docs for the core API underneath.
|
|
74
|
+
|
|
75
|
+
## Content and licences
|
|
76
|
+
|
|
77
|
+
Only sources whose licences allow reposting are used, and each post
|
|
78
|
+
credits its source as the licence asks. News agencies whose terms
|
|
79
|
+
forbid redistribution are not used. The first sources are listed in
|
|
80
|
+
issue #1.
|
|
81
|
+
|
|
82
|
+
Whoever runs a bot is responsible for its content and for following
|
|
83
|
+
the source's licence.
|
|
84
|
+
|
|
85
|
+
## Versions and releases
|
|
86
|
+
|
|
87
|
+
Versions follow [semantic versioning](https://semver.org/) on the same
|
|
88
|
+
principles as [unseenet](https://github.com/unseenet/unseenet/blob/main/docs/development.md#versioning):
|
|
89
|
+
tags `vX.Y.Z` on `main`, the changes in [CHANGELOG.md](CHANGELOG.md), and
|
|
90
|
+
releases on [PyPI](https://pypi.org/p/unseenet-bots). Each release names the
|
|
91
|
+
minor version of unseenet it works with (`unseenet>=0.4.1,<0.5`).
|
|
92
|
+
|
|
93
|
+
To release: move the entries under `## [Unreleased]` in `CHANGELOG.md` to a
|
|
94
|
+
new `## [X.Y.Z] - YYYY-MM-DD` section, set `__version__` in
|
|
95
|
+
`src/unseenet_bots/__init__.py`, commit, then
|
|
96
|
+
`git tag -a vX.Y.Z -m "unseenet-bots X.Y.Z" && git push origin vX.Y.Z`.
|
|
97
|
+
|
|
98
|
+
## Licence
|
|
99
|
+
|
|
100
|
+
The code is licensed under the Apache License 2.0. Content posted by
|
|
101
|
+
the bots keeps the licence of its source.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "unseenet-bots"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Bots that fill Unseenet channels with current content, built on the Unseenet core"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Andreas Lorensen" }]
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
dependencies = [
|
|
15
|
+
# the headless posting API, unseenet.core.poster (unseenet/unseenet#166)
|
|
16
|
+
"unseenet>=0.4.1,<0.5", # one minor version of the core at a time
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.23"]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
unseenet-bots = "unseenet_bots.runner:main"
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/unseenet/unseenet-bots"
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.dynamic]
|
|
29
|
+
version = { attr = "unseenet_bots.__version__" }
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
pythonpath = ["src"]
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
asyncio_mode = "auto"
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Run bots from a config file. Each run fetches each bot's source once and
|
|
2
|
+
posts what is new; schedule it with a systemd timer or cron.
|
|
3
|
+
|
|
4
|
+
unseenet-bots -c bots.toml new-key news.key # make a key; make it a poster
|
|
5
|
+
unseenet-bots -c bots.toml run --dry-run # show what would be posted
|
|
6
|
+
unseenet-bots -c bots.toml run # post
|
|
7
|
+
unseenet-bots -c bots.toml run news # only the bot named news
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import tomllib
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import datetime, timedelta, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from unseenet.core import crypto
|
|
23
|
+
from unseenet.core.markup import MARKDOWN_TYPE
|
|
24
|
+
from unseenet.core.poster import Poster, PosterError, load_key
|
|
25
|
+
|
|
26
|
+
from .sources import Item, get_source
|
|
27
|
+
|
|
28
|
+
KEEP = 1000 # remembered item IDs per bot
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Bot:
|
|
33
|
+
name: str
|
|
34
|
+
source: str
|
|
35
|
+
group: str
|
|
36
|
+
from_name: str
|
|
37
|
+
options: dict = field(default_factory=dict)
|
|
38
|
+
key_file: str | None = None
|
|
39
|
+
vault: str | None = None
|
|
40
|
+
identity: str | None = None
|
|
41
|
+
seed_env: str = "UNSEENET_SEED"
|
|
42
|
+
max_per_run: int = 5
|
|
43
|
+
max_per_day: int | None = None # e.g. for sources that may not be republished whole
|
|
44
|
+
max_age_hours: float = 48 # never post what the source published longer ago
|
|
45
|
+
nodes: list[str] | None = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def load_config(path: Path) -> tuple[dict, list[Bot]]:
|
|
49
|
+
data = tomllib.loads(path.read_text())
|
|
50
|
+
defaults = data.get("defaults", {})
|
|
51
|
+
bots = []
|
|
52
|
+
for name, b in data.get("bots", {}).items():
|
|
53
|
+
if name.startswith("_"):
|
|
54
|
+
raise SystemExit(f"bot {name}: names starting with _ are reserved")
|
|
55
|
+
try:
|
|
56
|
+
bots.append(Bot(
|
|
57
|
+
name=name, source=b["source"], group=b["group"],
|
|
58
|
+
from_name=b.get("from_name", f"{name} (bot)"), options=b.get("options", {}),
|
|
59
|
+
key_file=_path(b.get("key_file")), vault=_path(b.get("vault")),
|
|
60
|
+
identity=b.get("identity"), seed_env=b.get("seed_env", "UNSEENET_SEED"),
|
|
61
|
+
max_per_run=int(b.get("max_per_run", 5)),
|
|
62
|
+
max_per_day=int(b["max_per_day"]) if "max_per_day" in b else None,
|
|
63
|
+
max_age_hours=float(b.get("max_age_hours", 48)),
|
|
64
|
+
nodes=b.get("nodes", defaults.get("nodes"))))
|
|
65
|
+
except KeyError as exc:
|
|
66
|
+
raise SystemExit(f"bot {name}: missing {exc}") from exc
|
|
67
|
+
return defaults, bots
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _path(value: str | None) -> str | None:
|
|
71
|
+
return os.path.expanduser(value) if value else None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class State:
|
|
75
|
+
"""Which items each bot has posted: {bot: {item id: Message-ID}}."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, path: Path):
|
|
78
|
+
self.path = path
|
|
79
|
+
try:
|
|
80
|
+
self.data = json.loads(path.read_text())
|
|
81
|
+
except (OSError, ValueError):
|
|
82
|
+
self.data = {}
|
|
83
|
+
|
|
84
|
+
def posted(self, bot: str) -> dict:
|
|
85
|
+
return self.data.setdefault(bot, {})
|
|
86
|
+
|
|
87
|
+
def mark(self, bot: str, item_id: str, message_id: str, now: datetime | None = None) -> None:
|
|
88
|
+
posted = self.posted(bot)
|
|
89
|
+
posted[item_id] = message_id
|
|
90
|
+
for old in list(posted)[:-KEEP]:
|
|
91
|
+
del posted[old]
|
|
92
|
+
times = self.data.setdefault("_times", {}).setdefault(bot, [])
|
|
93
|
+
times.append((now or datetime.now(timezone.utc)).isoformat())
|
|
94
|
+
del times[:-KEEP]
|
|
95
|
+
|
|
96
|
+
def posted_since(self, bot: str, since: datetime) -> int:
|
|
97
|
+
return sum(1 for t in self.data.get("_times", {}).get(bot, [])
|
|
98
|
+
if datetime.fromisoformat(t) >= since)
|
|
99
|
+
|
|
100
|
+
def save(self) -> None:
|
|
101
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
102
|
+
tmp = self.path.with_suffix(".tmp")
|
|
103
|
+
tmp.write_text(json.dumps(self.data, indent=1))
|
|
104
|
+
tmp.replace(self.path)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def new_items(items: list[Item], seen: dict, recent_subjects: set[str], limit: int,
|
|
108
|
+
now: datetime | None = None, max_age_hours: float | None = None) -> list[Item]:
|
|
109
|
+
"""What to post now: not posted before, not too old, at most ``limit``."""
|
|
110
|
+
oldest = (now or datetime.now(timezone.utc)) - timedelta(hours=max_age_hours) \
|
|
111
|
+
if max_age_hours else None
|
|
112
|
+
out = [i for i in items if i.id not in seen and i.subject not in recent_subjects
|
|
113
|
+
and (oldest is None or i.published is None or i.published >= oldest)]
|
|
114
|
+
return out[:max(0, limit)]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def key_state(bot: Bot) -> Path | None:
|
|
118
|
+
"""Where the bot key's ripening chain is kept (UN-RFC 0025): next to its key file,
|
|
119
|
+
where ``unseenet ripen --key-file`` puts it at setup, so a key ripened there is the
|
|
120
|
+
one that posts. Holds nothing secret. Without a key file, the core's default."""
|
|
121
|
+
return Path(_path(bot.key_file) + ".state.json") if bot.key_file else None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _limit(bot: Bot, state: State, now: datetime) -> int:
|
|
125
|
+
if bot.max_per_day is None:
|
|
126
|
+
return bot.max_per_run
|
|
127
|
+
left = bot.max_per_day - state.posted_since(bot.name, now - timedelta(days=1))
|
|
128
|
+
return min(bot.max_per_run, left)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
async def run_bot(bot: Bot, state: State, dry_run: bool = False, today=None,
|
|
132
|
+
poster_factory=Poster, out=print, now: datetime | None = None) -> int:
|
|
133
|
+
"""Post what is new for one bot. Returns the number of posts made."""
|
|
134
|
+
now = now or datetime.now(timezone.utc)
|
|
135
|
+
today = today or now.date()
|
|
136
|
+
limit = _limit(bot, state, now)
|
|
137
|
+
if limit <= 0:
|
|
138
|
+
out(f"{bot.name}: {bot.max_per_day} posts in the last day already; waiting")
|
|
139
|
+
return 0
|
|
140
|
+
source = get_source(bot.source, **bot.options)
|
|
141
|
+
items = source.fetch(today)
|
|
142
|
+
if dry_run:
|
|
143
|
+
for item in new_items(items, state.posted(bot.name), set(), limit, now, bot.max_age_hours):
|
|
144
|
+
out(f"--- {bot.name} -> {bot.group}\nSubject: {item.subject}\n\n{item.body}")
|
|
145
|
+
return 0
|
|
146
|
+
key = load_key(key_file=bot.key_file, vault=bot.vault, identity=bot.identity,
|
|
147
|
+
seed_env=bot.seed_env)
|
|
148
|
+
posted = 0
|
|
149
|
+
async with poster_factory(key, bot.nodes, from_name=bot.from_name, state=key_state(bot),
|
|
150
|
+
log=lambda m: out(f" {bot.name}: {m}")) as p:
|
|
151
|
+
recent = {subject for _, subject in await p.recent_threads(bot.group)}
|
|
152
|
+
for item in new_items(items, state.posted(bot.name), recent, limit, now, bot.max_age_hours):
|
|
153
|
+
# the sources write Markdown (Item.body); say so, or clients show it raw
|
|
154
|
+
result = await p.post(bot.group, item.subject, item.body,
|
|
155
|
+
content_type=MARKDOWN_TYPE)
|
|
156
|
+
if result.ok:
|
|
157
|
+
state.mark(bot.name, item.id, result.message_id, now)
|
|
158
|
+
state.save() # after every post: a crash posts nothing twice
|
|
159
|
+
posted += 1
|
|
160
|
+
out(f"{bot.name}: posted {item.subject!r} to {len(result.accepted)} node(s)")
|
|
161
|
+
else:
|
|
162
|
+
out(f"{bot.name}: NOT posted {item.subject!r}: {result.refused or result.unreachable}")
|
|
163
|
+
return posted
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def main(argv: list[str] | None = None) -> None:
|
|
167
|
+
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
168
|
+
p.add_argument("-c", "--config", type=Path, default=Path("bots.toml"))
|
|
169
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
170
|
+
r = sub.add_parser("run", help="fetch and post what is new")
|
|
171
|
+
r.add_argument("bots", nargs="*", help="only these bots (default: all)")
|
|
172
|
+
r.add_argument("--dry-run", action="store_true", help="show the posts, post nothing")
|
|
173
|
+
sub.add_parser("list", help="list the configured bots")
|
|
174
|
+
k = sub.add_parser("new-key", help="make a key for a bot and show its public key")
|
|
175
|
+
k.add_argument("file", type=Path)
|
|
176
|
+
args = p.parse_args(argv)
|
|
177
|
+
|
|
178
|
+
if args.cmd == "new-key":
|
|
179
|
+
if args.file.exists():
|
|
180
|
+
sys.exit(f"{args.file} exists; not overwriting a key")
|
|
181
|
+
ident = crypto.Identity.generate()
|
|
182
|
+
args.file.touch(mode=0o600)
|
|
183
|
+
args.file.write_text(ident.seed.hex() + "\n")
|
|
184
|
+
print(f"Wrote {args.file}. Make this key a poster in the bot's channel:\n{ident.key_text}")
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
defaults, bots = load_config(args.config)
|
|
188
|
+
if args.cmd == "list":
|
|
189
|
+
for b in bots:
|
|
190
|
+
print(f"{b.name:16} {b.source:22} {b.group}")
|
|
191
|
+
return
|
|
192
|
+
chosen = [b for b in bots if not args.bots or b.name in args.bots]
|
|
193
|
+
if not chosen:
|
|
194
|
+
sys.exit("no such bot")
|
|
195
|
+
state = State(Path(_path(defaults.get("state_file", "~/.local/state/unseenet-bots/state.json"))))
|
|
196
|
+
failed = False
|
|
197
|
+
for bot in chosen:
|
|
198
|
+
try:
|
|
199
|
+
asyncio.run(run_bot(bot, state, dry_run=args.dry_run))
|
|
200
|
+
except (PosterError, OSError, ValueError) as exc:
|
|
201
|
+
print(f"{bot.name}: failed: {exc}", file=sys.stderr)
|
|
202
|
+
failed = True
|
|
203
|
+
if failed:
|
|
204
|
+
sys.exit(1)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
if __name__ == "__main__":
|
|
208
|
+
main()
|