alcedo-lit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. alcedo_lit-0.1.0/LICENSE +21 -0
  2. alcedo_lit-0.1.0/PKG-INFO +225 -0
  3. alcedo_lit-0.1.0/README.md +192 -0
  4. alcedo_lit-0.1.0/THIRD_PARTY_NOTICES.md +36 -0
  5. alcedo_lit-0.1.0/pyproject.toml +66 -0
  6. alcedo_lit-0.1.0/setup.cfg +4 -0
  7. alcedo_lit-0.1.0/src/alcedo_lit/__init__.py +3 -0
  8. alcedo_lit-0.1.0/src/alcedo_lit/__main__.py +6 -0
  9. alcedo_lit-0.1.0/src/alcedo_lit/check.py +141 -0
  10. alcedo_lit-0.1.0/src/alcedo_lit/cli.py +112 -0
  11. alcedo_lit-0.1.0/src/alcedo_lit/config.py +37 -0
  12. alcedo_lit-0.1.0/src/alcedo_lit/doi.py +146 -0
  13. alcedo_lit-0.1.0/src/alcedo_lit/net.py +14 -0
  14. alcedo_lit-0.1.0/src/alcedo_lit/pipeline.py +447 -0
  15. alcedo_lit-0.1.0/src/alcedo_lit/resolve.py +174 -0
  16. alcedo_lit-0.1.0/src/alcedo_lit/results.py +92 -0
  17. alcedo_lit-0.1.0/src/alcedo_lit/setup_wizard.py +108 -0
  18. alcedo_lit-0.1.0/src/alcedo_lit/sources/__init__.py +3 -0
  19. alcedo_lit-0.1.0/src/alcedo_lit/sources/openalex.py +56 -0
  20. alcedo_lit-0.1.0/src/alcedo_lit/sources/scholar.py +125 -0
  21. alcedo_lit-0.1.0/src/alcedo_lit/sources/searchapi.py +47 -0
  22. alcedo_lit-0.1.0/src/alcedo_lit/sources/semantic_scholar.py +56 -0
  23. alcedo_lit-0.1.0/src/alcedo_lit/sources/serpapi.py +63 -0
  24. alcedo_lit-0.1.0/src/alcedo_lit/sources/serper.py +56 -0
  25. alcedo_lit-0.1.0/src/alcedo_lit/verify.py +54 -0
  26. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/PKG-INFO +225 -0
  27. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/SOURCES.txt +42 -0
  28. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/dependency_links.txt +1 -0
  29. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/entry_points.txt +2 -0
  30. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/requires.txt +11 -0
  31. alcedo_lit-0.1.0/src/alcedo_lit.egg-info/top_level.txt +1 -0
  32. alcedo_lit-0.1.0/tests/test_chain.py +502 -0
  33. alcedo_lit-0.1.0/tests/test_check.py +134 -0
  34. alcedo_lit-0.1.0/tests/test_combined.py +74 -0
  35. alcedo_lit-0.1.0/tests/test_doi.py +129 -0
  36. alcedo_lit-0.1.0/tests/test_new_sources.py +211 -0
  37. alcedo_lit-0.1.0/tests/test_openalex.py +74 -0
  38. alcedo_lit-0.1.0/tests/test_pipeline.py +257 -0
  39. alcedo_lit-0.1.0/tests/test_resolve.py +174 -0
  40. alcedo_lit-0.1.0/tests/test_results.py +39 -0
  41. alcedo_lit-0.1.0/tests/test_scholar_paging.py +48 -0
  42. alcedo_lit-0.1.0/tests/test_searchapi.py +74 -0
  43. alcedo_lit-0.1.0/tests/test_serpapi.py +84 -0
  44. alcedo_lit-0.1.0/tests/test_verify.py +104 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Alcedo contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,225 @@
1
+ Metadata-Version: 2.4
2
+ Name: alcedo-lit
3
+ Version: 0.1.0
4
+ Summary: Collect DOIs from a list of literature search queries, grouped by query.
5
+ Author: Alcedo contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Prostochell-544/Alcedo
8
+ Project-URL: Issues, https://github.com/Prostochell-544/Alcedo/issues
9
+ Project-URL: Changelog, https://github.com/Prostochell-544/Alcedo/blob/main/CHANGELOG.md
10
+ Keywords: literature search,doi,bibliometrics,systematic review,openalex,semantic scholar,google scholar
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ License-File: THIRD_PARTY_NOTICES.md
23
+ Requires-Dist: requests>=2.31
24
+ Requires-Dist: tqdm>=4.66
25
+ Provides-Extra: scholar
26
+ Requires-Dist: scholarly>=1.7.11; extra == "scholar"
27
+ Requires-Dist: bibtexparser<2; extra == "scholar"
28
+ Requires-Dist: httpx<0.28; extra == "scholar"
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8; extra == "dev"
31
+ Requires-Dist: ruff>=0.6; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ <p align="center">
35
+ <img src="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-logo.png" alt="Alcedo logo: a kingfisher in flight" width="220">
36
+ <br>
37
+ <picture>
38
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-wordmark-dark.svg">
39
+ <img src="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-wordmark-light.svg" alt="Alcedo: open metadata extractor" width="320">
40
+ </picture>
41
+ </p>
42
+
43
+ # Alcedo
44
+
45
+ *Alcedo* is the Latin name of the kingfisher, which dives and picks out exactly what it wants.
46
+
47
+ Alcedo turns a list of literature search queries into DOIs grouped by query. Put your
48
+ boolean queries in a file and run one command. The results are a text file with one
49
+ block per query. There is no export to reference managers yet.
50
+
51
+ **Status: alpha.** Options may change. See the [roadmap](https://github.com/Prostochell-544/Alcedo/blob/main/docs/ROADMAP.md) and the [changelog](https://github.com/Prostochell-544/Alcedo/blob/main/CHANGELOG.md).
52
+ Documentation in other languages: [Russian](https://github.com/Prostochell-544/Alcedo/blob/main/docs/ru/query-rules.md), [Ukrainian](https://github.com/Prostochell-544/Alcedo/blob/main/docs/uk/query-rules.md).
53
+
54
+ ## What it does
55
+
56
+ - Takes up to 5 result pages (50 results) per query, or all of them if there are fewer.
57
+ - Searches Google Scholar through a search API with your own key. Without a working API it uses Semantic Scholar and
58
+ OpenAlex, plus direct Google Scholar search if you turned that on. See [Which source to use](#which-source-to-use).
59
+ - A source that fails is left out, and the block says so.
60
+ - Scholar gives links rather than DOIs, so Alcedo looks for the DOI in the URL, Europe PMC and
61
+ publisher meta tags, then by title in Crossref and OpenAlex.
62
+ - Saves after every query. `--resume` continues after an interruption and collects failed queries again.
63
+ - Writes a second file listing each DOI once, with the queries that found it. Papers found by the most queries come first.
64
+ - With `--verify-dois`, checks every DOI in the doi.org registry and marks the ones that do not exist.
65
+
66
+ ## Install
67
+
68
+ Requires Python 3.10 or newer.
69
+
70
+ ```bash
71
+ pip install alcedo-lit
72
+ ```
73
+
74
+ For direct Google Scholar search, which most people will not need, install the extra as well:
75
+ `pip install "alcedo-lit[scholar]"`.
76
+
77
+ Then choose how to search. This explains the options and saves your key, which is not shown as you type:
78
+
79
+ ```bash
80
+ alcedo setup
81
+ ```
82
+
83
+ ## Quick start
84
+
85
+ The repository includes a sample: [queries.txt](https://github.com/Prostochell-544/Alcedo/blob/main/queries.txt) has two queries about the kingfisher, and
86
+ [alcedo_results.txt](https://github.com/Prostochell-544/Alcedo/blob/main/alcedo_results.txt) and [alcedo_results_unique.csv](https://github.com/Prostochell-544/Alcedo/blob/main/alcedo_results_unique.csv) are their
87
+ results. There is another set of queries in [examples/queries.txt](https://github.com/Prostochell-544/Alcedo/blob/main/examples/queries.txt).
88
+
89
+ For your own search, create a `queries.txt` in any folder, with `;` after each query:
90
+
91
+ ```text
92
+ biomimetic nacre ("brick-and-mortar" OR "layered structure") composite fracture toughness;
93
+ gecko-inspired dry adhesive (micropillar OR "hierarchical microstructure") adhesion;
94
+ ```
95
+
96
+ From that folder, check the queries for common mistakes and then run them:
97
+
98
+ ```bash
99
+ alcedo check
100
+ alcedo
101
+ ```
102
+
103
+ The results go to `alcedo_results.txt` in the current folder:
104
+
105
+ ```text
106
+ === Query: gecko-inspired dry adhesive (micropillar OR "hierarchical microstructure") adhesion ===
107
+ Source: OpenAlex
108
+ https://doi.org/10.1039/c3sm52921e
109
+ https://doi.org/10.1098/rsif.2018.0681
110
+ ```
111
+
112
+ The `Source:` line appears when the results did not all come from a search API. Records with no DOI are
113
+ listed as `DOI not found: <title>`, and a query that failed has a `Query error` line. If DOI
114
+ lookups failed along the way, the final message says how many, because those records may still have a DOI.
115
+
116
+ Alcedo will not replace an existing results file: use `--resume` to continue it or `--overwrite` to start again.
117
+
118
+ `alcedo_results_unique.csv` lists each DOI once, with the number of queries that found it and the queries themselves.
119
+ It is built from the results file and uses no extra searches. Cells are separated by `;`, which Excel in European
120
+ locales expects.
121
+
122
+ To check DOIs against the doi.org registry, add `--verify-dois`. This also works on results you already have:
123
+ `alcedo --resume --verify-dois`. A DOI the registry does not know becomes `DOI not registered: <doi>`, and one that
124
+ could not be checked stays in the results and is counted in the final message. The check uses no OpenAlex budget.
125
+
126
+ To check the wording of a query quickly, look at its first 10 results:
127
+
128
+ ```bash
129
+ alcedo --pages 1 -i test.txt -o test_results.txt --overwrite
130
+ ```
131
+
132
+ Read [how to write queries](https://github.com/Prostochell-544/Alcedo/blob/main/docs/query-syntax.md) before a large run. Some mistakes fail silently: a lowercase
133
+ `or`, for example, is treated as AND. `alcedo check` finds the common ones.
134
+
135
+ ## Which source to use
136
+
137
+ With a working API key, a query uses only the API. Otherwise it uses the free sources together, in this order:
138
+ Semantic Scholar, OpenAlex, and direct search if you turned it on. Each gives up to 50 results and repeats are
139
+ removed, so a block can hold more than 50 records. A source that fails is left out, and the `Source:` line says so.
140
+ If every source fails, the block has a `Query error` line and `--resume` collects it again.
141
+
142
+ The free options do not give the same results as an API. The figures below come from small tests with relevance
143
+ judged by an AI assistant, not by a domain expert. Treat them as a rough guide. Results vary a lot by query.
144
+
145
+ | Source | What you need | Cost at the time of writing | How it did |
146
+ |---|---|---|---|
147
+ | Google Scholar via [SearchApi](https://www.searchapi.io) | Your key, no phone number | 100 free requests to try it, then from $40 a month | Best first results: 13 of the first 20 were exactly on topic. Same first results as a browser in our check |
148
+ | Google Scholar via [Serper](https://serper.dev) | Your key, no card | 2,500 free queries, which is about 500 queries of 50 results, then one-off packs from $50 for 50,000, valid for 6 months | Same first 10 results as SearchApi on both test queries, and 13 of the first 20 exactly on topic. Later results differed |
149
+ | Google Scholar via [SerpApi](https://serpapi.com) | Your key and a phone number | 250 free searches a month, paid plans from $25 a month | Not measured; Alcedo has not yet been run against the live service |
150
+ | [Semantic Scholar](https://www.semanticscholar.org/product/api) | Nothing; a free key gives you your own limit | Free | 9 of the first 10 exactly on topic on one query, no results on a stricter one. Finds few but precise papers, most cited first. Not measured inside Alcedo |
151
+ | OpenAlex | Nothing; a [free key](https://openalex.org/settings/api) raises the daily budget | Free | 5 of the first 20 exactly on topic. Its share of on-topic results ranged from 34% against 73% for the API to about equal. It also found papers Scholar missed |
152
+ | Google Scholar directly | `pip install "alcedo-lit[scholar]"` and your agreement | Free | Changes a lot between runs and networks, see below |
153
+
154
+ A query with 50 results uses 3 requests on SearchApi and 5 on Serper. You create the key and accept the provider's terms, and the
155
+ provider sees your queries. If you have neither a key nor a budget for one, the free catalogues still work, and direct
156
+ search can add to them on a network Scholar accepts.
157
+
158
+ In a run where Scholar answered in full, the free sources together found 38 of the 45 exactly on-topic papers known
159
+ from all our runs, against 29 for SearchApi. Their first results were weaker, with 9 of the first 20 exactly on topic
160
+ against 13, and they took 15 minutes for 2 queries against 3.5, mostly on direct search and DOI lookups. A key gives you
161
+ the best first results quickly, but going without one does not have to mean missing papers.
162
+
163
+ Publications that use results from Semantic Scholar should cite the Semantic Scholar Open Data Platform, as its
164
+ [API license](https://www.semanticscholar.org/product/api/license) asks.
165
+
166
+ ### Direct search
167
+
168
+ It is free, and Scholar may simply refuse it. Scholar often refuses VPNs and shared networks: in our test, one VPN
169
+ provider was refused even in a browser and another was accepted. If it refuses you, try without the VPN, and check
170
+ the same search in a browser.
171
+
172
+ **An answer from Scholar does not mean the list is complete.** In some of our tests, the script got 6 to 27 results
173
+ for queries that give 50 through an API, mostly recent papers, with key works missing, and repeating one query gave
174
+ different lists. In a later run it got all 50, nearly all of them also in the API's list. Parsing does not explain the
175
+ difference, we do not know the cause, and you cannot tell in advance which you will get. So Alcedo puts direct results after the free catalogues,
176
+ where they can only add to the list, and marks those blocks in the results file. Do not rely on direct search for a
177
+ literature review. If you add a key later, `alcedo --redo-free` collects those blocks again.
178
+
179
+ Alcedo asks once before using direct search, and remembers your answer. `alcedo setup` changes it, and
180
+ `--direct-scholar` or `--no-direct-scholar` overrides it for one run.
181
+
182
+ ### Keys and everyday commands
183
+
184
+ `alcedo setup` stores keys as plain text in your user settings (`%APPDATA%\alcedo\config.json` on Windows,
185
+ `~/.config/alcedo/config.json` elsewhere). Run it again and type `-` to remove a key. The `SEARCHAPI_KEY`,
186
+ `SERPER_API_KEY`, `SERPAPI_KEY`, `OPENALEX_API_KEY` and `SEMANTIC_SCHOLAR_API_KEY` environment variables take priority
187
+ over that file, and command line options take priority over both.
188
+
189
+ ```bash
190
+ alcedo --resume # continue after an interruption
191
+ alcedo --redo-free # collect blocks from the free sources again, for example after adding a key
192
+ alcedo --source openalex # OpenAlex only
193
+ alcedo --no-semantic-scholar # leave Semantic Scholar out of the free sources
194
+ alcedo --help # all options
195
+ ```
196
+
197
+ ## Responsible use
198
+
199
+ - Direct Google Scholar search is off until you agree to it. Scholar's `robots.txt` disallows automated search, and
200
+ you are responsible for following Google's terms. Alcedo waits between queries and stops asking when Scholar
201
+ refuses. The `scholarly` library it relies on sends a random browser User-Agent and may try to open a browser for
202
+ a captcha, and it has no setting to turn that off.
203
+ - Apart from that route, requests identify themselves as `alcedo/<version>`. With `--mailto`, your
204
+ address goes to Crossref, OpenAlex, Semantic Scholar and Europe PMC, never to publisher sites.
205
+ - Alcedo collects metadata. It does not download papers and will not integrate shadow libraries.
206
+
207
+ ## Development
208
+
209
+ ```bash
210
+ pip install -e ".[dev]"
211
+ python -m pytest
212
+ ruff check .
213
+ ```
214
+
215
+ The tests use no network. Git tracks the sample in the project root, so do not run your own queries there:
216
+ `queries.txt`, `alcedo_results.txt` and `alcedo_results_unique.csv` would show up as changes. Use `private/`, which
217
+ Git ignores, so your topic stays out of the repository.
218
+
219
+ ## License
220
+
221
+ The code is under the [MIT](https://github.com/Prostochell-544/Alcedo/blob/main/LICENSE) license. The logo is not: see the [third-party notices](https://github.com/Prostochell-544/Alcedo/blob/main/THIRD_PARTY_NOTICES.md).
222
+
223
+ ## Special thanks
224
+
225
+ To [kesha_xdd](https://t.me/kesha_xdd), who drew the Alcedo logo.
@@ -0,0 +1,192 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-logo.png" alt="Alcedo logo: a kingfisher in flight" width="220">
3
+ <br>
4
+ <picture>
5
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-wordmark-dark.svg">
6
+ <img src="https://raw.githubusercontent.com/Prostochell-544/Alcedo/main/docs/assets/alcedo-wordmark-light.svg" alt="Alcedo: open metadata extractor" width="320">
7
+ </picture>
8
+ </p>
9
+
10
+ # Alcedo
11
+
12
+ *Alcedo* is the Latin name of the kingfisher, which dives and picks out exactly what it wants.
13
+
14
+ Alcedo turns a list of literature search queries into DOIs grouped by query. Put your
15
+ boolean queries in a file and run one command. The results are a text file with one
16
+ block per query. There is no export to reference managers yet.
17
+
18
+ **Status: alpha.** Options may change. See the [roadmap](https://github.com/Prostochell-544/Alcedo/blob/main/docs/ROADMAP.md) and the [changelog](https://github.com/Prostochell-544/Alcedo/blob/main/CHANGELOG.md).
19
+ Documentation in other languages: [Russian](https://github.com/Prostochell-544/Alcedo/blob/main/docs/ru/query-rules.md), [Ukrainian](https://github.com/Prostochell-544/Alcedo/blob/main/docs/uk/query-rules.md).
20
+
21
+ ## What it does
22
+
23
+ - Takes up to 5 result pages (50 results) per query, or all of them if there are fewer.
24
+ - Searches Google Scholar through a search API with your own key. Without a working API it uses Semantic Scholar and
25
+ OpenAlex, plus direct Google Scholar search if you turned that on. See [Which source to use](#which-source-to-use).
26
+ - A source that fails is left out, and the block says so.
27
+ - Scholar gives links rather than DOIs, so Alcedo looks for the DOI in the URL, Europe PMC and
28
+ publisher meta tags, then by title in Crossref and OpenAlex.
29
+ - Saves after every query. `--resume` continues after an interruption and collects failed queries again.
30
+ - Writes a second file listing each DOI once, with the queries that found it. Papers found by the most queries come first.
31
+ - With `--verify-dois`, checks every DOI in the doi.org registry and marks the ones that do not exist.
32
+
33
+ ## Install
34
+
35
+ Requires Python 3.10 or newer.
36
+
37
+ ```bash
38
+ pip install alcedo-lit
39
+ ```
40
+
41
+ For direct Google Scholar search, which most people will not need, install the extra as well:
42
+ `pip install "alcedo-lit[scholar]"`.
43
+
44
+ Then choose how to search. This explains the options and saves your key, which is not shown as you type:
45
+
46
+ ```bash
47
+ alcedo setup
48
+ ```
49
+
50
+ ## Quick start
51
+
52
+ The repository includes a sample: [queries.txt](https://github.com/Prostochell-544/Alcedo/blob/main/queries.txt) has two queries about the kingfisher, and
53
+ [alcedo_results.txt](https://github.com/Prostochell-544/Alcedo/blob/main/alcedo_results.txt) and [alcedo_results_unique.csv](https://github.com/Prostochell-544/Alcedo/blob/main/alcedo_results_unique.csv) are their
54
+ results. There is another set of queries in [examples/queries.txt](https://github.com/Prostochell-544/Alcedo/blob/main/examples/queries.txt).
55
+
56
+ For your own search, create a `queries.txt` in any folder, with `;` after each query:
57
+
58
+ ```text
59
+ biomimetic nacre ("brick-and-mortar" OR "layered structure") composite fracture toughness;
60
+ gecko-inspired dry adhesive (micropillar OR "hierarchical microstructure") adhesion;
61
+ ```
62
+
63
+ From that folder, check the queries for common mistakes and then run them:
64
+
65
+ ```bash
66
+ alcedo check
67
+ alcedo
68
+ ```
69
+
70
+ The results go to `alcedo_results.txt` in the current folder:
71
+
72
+ ```text
73
+ === Query: gecko-inspired dry adhesive (micropillar OR "hierarchical microstructure") adhesion ===
74
+ Source: OpenAlex
75
+ https://doi.org/10.1039/c3sm52921e
76
+ https://doi.org/10.1098/rsif.2018.0681
77
+ ```
78
+
79
+ The `Source:` line appears when the results did not all come from a search API. Records with no DOI are
80
+ listed as `DOI not found: <title>`, and a query that failed has a `Query error` line. If DOI
81
+ lookups failed along the way, the final message says how many, because those records may still have a DOI.
82
+
83
+ Alcedo will not replace an existing results file: use `--resume` to continue it or `--overwrite` to start again.
84
+
85
+ `alcedo_results_unique.csv` lists each DOI once, with the number of queries that found it and the queries themselves.
86
+ It is built from the results file and uses no extra searches. Cells are separated by `;`, which Excel in European
87
+ locales expects.
88
+
89
+ To check DOIs against the doi.org registry, add `--verify-dois`. This also works on results you already have:
90
+ `alcedo --resume --verify-dois`. A DOI the registry does not know becomes `DOI not registered: <doi>`, and one that
91
+ could not be checked stays in the results and is counted in the final message. The check uses no OpenAlex budget.
92
+
93
+ To check the wording of a query quickly, look at its first 10 results:
94
+
95
+ ```bash
96
+ alcedo --pages 1 -i test.txt -o test_results.txt --overwrite
97
+ ```
98
+
99
+ Read [how to write queries](https://github.com/Prostochell-544/Alcedo/blob/main/docs/query-syntax.md) before a large run. Some mistakes fail silently: a lowercase
100
+ `or`, for example, is treated as AND. `alcedo check` finds the common ones.
101
+
102
+ ## Which source to use
103
+
104
+ With a working API key, a query uses only the API. Otherwise it uses the free sources together, in this order:
105
+ Semantic Scholar, OpenAlex, and direct search if you turned it on. Each gives up to 50 results and repeats are
106
+ removed, so a block can hold more than 50 records. A source that fails is left out, and the `Source:` line says so.
107
+ If every source fails, the block has a `Query error` line and `--resume` collects it again.
108
+
109
+ The free options do not give the same results as an API. The figures below come from small tests with relevance
110
+ judged by an AI assistant, not by a domain expert. Treat them as a rough guide. Results vary a lot by query.
111
+
112
+ | Source | What you need | Cost at the time of writing | How it did |
113
+ |---|---|---|---|
114
+ | Google Scholar via [SearchApi](https://www.searchapi.io) | Your key, no phone number | 100 free requests to try it, then from $40 a month | Best first results: 13 of the first 20 were exactly on topic. Same first results as a browser in our check |
115
+ | Google Scholar via [Serper](https://serper.dev) | Your key, no card | 2,500 free queries, which is about 500 queries of 50 results, then one-off packs from $50 for 50,000, valid for 6 months | Same first 10 results as SearchApi on both test queries, and 13 of the first 20 exactly on topic. Later results differed |
116
+ | Google Scholar via [SerpApi](https://serpapi.com) | Your key and a phone number | 250 free searches a month, paid plans from $25 a month | Not measured; Alcedo has not yet been run against the live service |
117
+ | [Semantic Scholar](https://www.semanticscholar.org/product/api) | Nothing; a free key gives you your own limit | Free | 9 of the first 10 exactly on topic on one query, no results on a stricter one. Finds few but precise papers, most cited first. Not measured inside Alcedo |
118
+ | OpenAlex | Nothing; a [free key](https://openalex.org/settings/api) raises the daily budget | Free | 5 of the first 20 exactly on topic. Its share of on-topic results ranged from 34% against 73% for the API to about equal. It also found papers Scholar missed |
119
+ | Google Scholar directly | `pip install "alcedo-lit[scholar]"` and your agreement | Free | Changes a lot between runs and networks, see below |
120
+
121
+ A query with 50 results uses 3 requests on SearchApi and 5 on Serper. You create the key and accept the provider's terms, and the
122
+ provider sees your queries. If you have neither a key nor a budget for one, the free catalogues still work, and direct
123
+ search can add to them on a network Scholar accepts.
124
+
125
+ In a run where Scholar answered in full, the free sources together found 38 of the 45 exactly on-topic papers known
126
+ from all our runs, against 29 for SearchApi. Their first results were weaker, with 9 of the first 20 exactly on topic
127
+ against 13, and they took 15 minutes for 2 queries against 3.5, mostly on direct search and DOI lookups. A key gives you
128
+ the best first results quickly, but going without one does not have to mean missing papers.
129
+
130
+ Publications that use results from Semantic Scholar should cite the Semantic Scholar Open Data Platform, as its
131
+ [API license](https://www.semanticscholar.org/product/api/license) asks.
132
+
133
+ ### Direct search
134
+
135
+ It is free, and Scholar may simply refuse it. Scholar often refuses VPNs and shared networks: in our test, one VPN
136
+ provider was refused even in a browser and another was accepted. If it refuses you, try without the VPN, and check
137
+ the same search in a browser.
138
+
139
+ **An answer from Scholar does not mean the list is complete.** In some of our tests, the script got 6 to 27 results
140
+ for queries that give 50 through an API, mostly recent papers, with key works missing, and repeating one query gave
141
+ different lists. In a later run it got all 50, nearly all of them also in the API's list. Parsing does not explain the
142
+ difference, we do not know the cause, and you cannot tell in advance which you will get. So Alcedo puts direct results after the free catalogues,
143
+ where they can only add to the list, and marks those blocks in the results file. Do not rely on direct search for a
144
+ literature review. If you add a key later, `alcedo --redo-free` collects those blocks again.
145
+
146
+ Alcedo asks once before using direct search, and remembers your answer. `alcedo setup` changes it, and
147
+ `--direct-scholar` or `--no-direct-scholar` overrides it for one run.
148
+
149
+ ### Keys and everyday commands
150
+
151
+ `alcedo setup` stores keys as plain text in your user settings (`%APPDATA%\alcedo\config.json` on Windows,
152
+ `~/.config/alcedo/config.json` elsewhere). Run it again and type `-` to remove a key. The `SEARCHAPI_KEY`,
153
+ `SERPER_API_KEY`, `SERPAPI_KEY`, `OPENALEX_API_KEY` and `SEMANTIC_SCHOLAR_API_KEY` environment variables take priority
154
+ over that file, and command line options take priority over both.
155
+
156
+ ```bash
157
+ alcedo --resume # continue after an interruption
158
+ alcedo --redo-free # collect blocks from the free sources again, for example after adding a key
159
+ alcedo --source openalex # OpenAlex only
160
+ alcedo --no-semantic-scholar # leave Semantic Scholar out of the free sources
161
+ alcedo --help # all options
162
+ ```
163
+
164
+ ## Responsible use
165
+
166
+ - Direct Google Scholar search is off until you agree to it. Scholar's `robots.txt` disallows automated search, and
167
+ you are responsible for following Google's terms. Alcedo waits between queries and stops asking when Scholar
168
+ refuses. The `scholarly` library it relies on sends a random browser User-Agent and may try to open a browser for
169
+ a captcha, and it has no setting to turn that off.
170
+ - Apart from that route, requests identify themselves as `alcedo/<version>`. With `--mailto`, your
171
+ address goes to Crossref, OpenAlex, Semantic Scholar and Europe PMC, never to publisher sites.
172
+ - Alcedo collects metadata. It does not download papers and will not integrate shadow libraries.
173
+
174
+ ## Development
175
+
176
+ ```bash
177
+ pip install -e ".[dev]"
178
+ python -m pytest
179
+ ruff check .
180
+ ```
181
+
182
+ The tests use no network. Git tracks the sample in the project root, so do not run your own queries there:
183
+ `queries.txt`, `alcedo_results.txt` and `alcedo_results_unique.csv` would show up as changes. Use `private/`, which
184
+ Git ignores, so your topic stays out of the repository.
185
+
186
+ ## License
187
+
188
+ The code is under the [MIT](https://github.com/Prostochell-544/Alcedo/blob/main/LICENSE) license. The logo is not: see the [third-party notices](https://github.com/Prostochell-544/Alcedo/blob/main/THIRD_PARTY_NOTICES.md).
189
+
190
+ ## Special thanks
191
+
192
+ To [kesha_xdd](https://t.me/kesha_xdd), who drew the Alcedo logo.
@@ -0,0 +1,36 @@
1
+ # Third-party notices
2
+
3
+ Alcedo contains no code copied from other projects so far.
4
+
5
+ If you copy code from another project, add an entry here with the project name, its
6
+ URL, the files concerned and the full text of its license and copyright notice. Keep
7
+ any notices from the original source. Apache-2.0 projects may also have a NOTICE file
8
+ that has to be preserved.
9
+
10
+ Projects whose ideas informed Alcedo's design are listed in `docs/ROADMAP.md`.
11
+
12
+ ## Logo
13
+
14
+ `docs/assets/alcedo-logo.png` was drawn by [kesha_xdd](https://t.me/kesha_xdd). All rights stay with the artist, who
15
+ allowed its use for this project. It is not covered by the MIT license of the code, so forks and other projects need
16
+ the artist's own permission to use it.
17
+
18
+ ## Wordmark
19
+
20
+ `docs/assets/alcedo-wordmark-light.svg` and `alcedo-wordmark-dark.svg` show text set in Orbitron and converted to
21
+ outlines. The repository does not contain the font itself.
22
+
23
+ Orbitron: Copyright 2018 The Orbitron Project Authors (https://github.com/theleagueof/orbitron), with Reserved Font
24
+ Name "Orbitron". Licensed under the SIL Open Font License, Version 1.1: https://openfontlicense.org
25
+
26
+ ## Runtime dependencies
27
+
28
+ These are installed separately and are not distributed with Alcedo.
29
+
30
+ | Package | License | Needed for |
31
+ |---|---|---|
32
+ | requests | Apache-2.0 | always |
33
+ | tqdm | MPL-2.0 AND MIT | always |
34
+ | scholarly | Unlicense | optional `scholar` extra |
35
+ | bibtexparser | LGPLv3 or BSD (version 1.x) | optional `scholar` extra |
36
+ | httpx | BSD-3-Clause | optional `scholar` extra |
@@ -0,0 +1,66 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "alcedo-lit"
7
+ dynamic = ["version"]
8
+ description = "Collect DOIs from a list of literature search queries, grouped by query."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE", "THIRD_PARTY_NOTICES.md"]
13
+ authors = [{ name = "Alcedo contributors" }]
14
+ keywords = ["literature search", "doi", "bibliometrics", "systematic review", "openalex", "semantic scholar",
15
+ "google scholar"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Environment :: Console",
19
+ "Intended Audience :: Science/Research",
20
+ "Operating System :: OS Independent",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.10",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Topic :: Scientific/Engineering",
25
+ ]
26
+ dependencies = [
27
+ "requests>=2.31",
28
+ "tqdm>=4.66",
29
+ ]
30
+
31
+ [project.urls]
32
+ Homepage = "https://github.com/Prostochell-544/Alcedo"
33
+ Issues = "https://github.com/Prostochell-544/Alcedo/issues"
34
+ Changelog = "https://github.com/Prostochell-544/Alcedo/blob/main/CHANGELOG.md"
35
+
36
+ [project.optional-dependencies]
37
+ # Direct Google Scholar access. scholarly 1.7.11 imports a module that
38
+ # bibtexparser 2 removed, and passes an argument to httpx.Client that httpx 0.28 removed.
39
+ scholar = [
40
+ "scholarly>=1.7.11",
41
+ "bibtexparser<2",
42
+ "httpx<0.28",
43
+ ]
44
+ dev = [
45
+ "pytest>=8",
46
+ "ruff>=0.6",
47
+ ]
48
+
49
+ [project.scripts]
50
+ alcedo = "alcedo_lit.cli:main"
51
+
52
+ [tool.setuptools.dynamic]
53
+ version = { attr = "alcedo_lit.__version__" }
54
+
55
+ [tool.setuptools.packages.find]
56
+ where = ["src"]
57
+
58
+ [tool.pytest.ini_options]
59
+ testpaths = ["tests"]
60
+
61
+ [tool.ruff]
62
+ line-length = 120
63
+ src = ["src", "tests"]
64
+
65
+ [tool.ruff.lint]
66
+ select = ["E", "F", "I", "UP", "B"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """Alcedo: collect DOIs from a list of literature search queries, grouped by query."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())