scopusflow 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scopusflow-0.2.0 → scopusflow-0.3.0}/CHANGELOG.md +25 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/CITATION.cff +2 -2
- {scopusflow-0.2.0 → scopusflow-0.3.0}/CONTRIBUTING.md +5 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/PKG-INFO +6 -6
- {scopusflow-0.2.0 → scopusflow-0.3.0}/README.md +5 -5
- scopusflow-0.3.0/_snippets/plot-setup.md +45 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/about.md +2 -2
- scopusflow-0.3.0/docs/assets/app-compare.png +0 -0
- scopusflow-0.3.0/docs/assets/app-window.png +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/analysing-a-literature.md +37 -44
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/building-a-reference-set.md +32 -32
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/comparing-topics.md +14 -4
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/designing-queries.md +2 -2
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/getting-started.md +17 -21
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/keywords-and-references.md +50 -25
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/plans-and-quota.md +1 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/the-app.md +16 -8
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/guides/tracking-literature-change.md +50 -45
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/index.md +4 -2
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/licence.md +2 -7
- scopusflow-0.3.0/docs/reference/analyse.md +100 -0
- scopusflow-0.3.0/docs/reference/records.md +26 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/stylesheets/brand.css +63 -10
- {scopusflow-0.2.0 → scopusflow-0.3.0}/examples/quickstart.py +20 -28
- {scopusflow-0.2.0 → scopusflow-0.3.0}/mkdocs.yml +1 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/pyproject.toml +9 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/__init__.py +3 -1
- scopusflow-0.3.0/src/scopusflow/_data/example_records.csv +139 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/app.py +88 -32
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/compare.py +2 -2
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/count.py +5 -5
- scopusflow-0.3.0/src/scopusflow/data.py +71 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/diff.py +6 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/export.py +1 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/intersections.py +10 -10
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/plan.py +5 -5
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/plots.py +2 -2
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/records.py +1 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/trend.py +1 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_abstract.py +9 -3
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_app.py +56 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_compare.py +2 -1
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_core.py +1 -1
- scopusflow-0.3.0/tests/test_data.py +54 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_trend.py +1 -1
- scopusflow-0.2.0/docs/reference/analyse.md +0 -22
- scopusflow-0.2.0/docs/reference/records.md +0 -14
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.gitattributes +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/CODE_OF_CONDUCT.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/SECURITY.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/dependabot.yml +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/workflows/docs.yml +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/workflows/publish.yml +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.github/workflows/tests.yaml +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/.gitignore +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/LICENSE +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/assets/favicon.png +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/assets/favicon.svg +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/assets/logo.png +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/changelog.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/reference/dois.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/reference/plan-and-query.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/docs/reference/retrieve.md +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/overrides/partials/header.html +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/abstract.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/app_helpers.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/corpus.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/exceptions.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/fetch.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/py.typed +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/src/scopusflow/query.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_corpus.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_export.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_fetch.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_intersections.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_plots.py +0 -0
- {scopusflow-0.2.0 → scopusflow-0.3.0}/tests/test_records.py +0 -0
|
@@ -5,7 +5,31 @@ All notable changes to this project are documented here.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
## [
|
|
8
|
+
## [0.3.0] - 2026-07-23
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `example_records()` returns a bundled worked-example harvest of 138 real
|
|
13
|
+
journal articles on graphene supercapacitors, 2015 to 2024, over the standard
|
|
14
|
+
`RECORD_COLUMNS` schema. The records come from OpenAlex (CC0) rather than
|
|
15
|
+
Scopus, whose terms do not permit redistributing retrieved records, and they
|
|
16
|
+
are the same corpus the R twin ships.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- Every guide and reference example now runs on that bundled harvest instead of
|
|
21
|
+
a fabricated frame, with the live API call shown alongside. The app's demo
|
|
22
|
+
mode replays it too, so a first visit shows real articles and a real
|
|
23
|
+
publication curve; a demo cell for a year outside the corpus returns nothing
|
|
24
|
+
and says so in the log rather than padding itself out. The app's year slider
|
|
25
|
+
opens on the corpus span, and the demo comparison, whose counts are still
|
|
26
|
+
simulated, now says so beneath the figure.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
|
|
30
|
+
- `extract_dois` no longer emits the literal `<NA>` for a record with no DOI,
|
|
31
|
+
which a nullable string column produces where a float NaN would have been
|
|
32
|
+
skipped.
|
|
9
33
|
|
|
10
34
|
## [0.2.0] - 2026-07-15
|
|
11
35
|
|
|
@@ -13,8 +13,8 @@ authors:
|
|
|
13
13
|
- family-names: Bernabeu
|
|
14
14
|
given-names: Pablo
|
|
15
15
|
orcid: "https://orcid.org/0000-0003-1083-2460"
|
|
16
|
-
version: 0.
|
|
17
|
-
date-released: "2026-07-
|
|
16
|
+
version: 0.3.0
|
|
17
|
+
date-released: "2026-07-23"
|
|
18
18
|
license: MIT
|
|
19
19
|
doi: "10.5281/zenodo.21252666"
|
|
20
20
|
repository-code: "https://github.com/pablobernabeu/scopusflow-py"
|
|
@@ -49,6 +49,11 @@ A release runs through these steps:
|
|
|
49
49
|
builds the distribution and uploads it to PyPI through trusted publishing, so
|
|
50
50
|
no token is stored in the repository.
|
|
51
51
|
|
|
52
|
+
If the app's layout changed in the release, recapture `docs/assets/app-window.png`
|
|
53
|
+
and `docs/assets/app-compare.png` from `scopusflow-gui` in demo mode, which needs
|
|
54
|
+
no key. Every other figure on the docs site is rendered at build time and cannot
|
|
55
|
+
go stale, but these two are static and will drift silently.
|
|
56
|
+
|
|
52
57
|
Trusted publishing needs a one-time setup on PyPI. In the project's Publishing
|
|
53
58
|
settings, add this repository as a trusted publisher with workflow file
|
|
54
59
|
`publish.yml` and environment `pypi`. The workflow handles every PyPI release
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scopusflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A reproducible workflow layer over pybliometrics for Scopus searches
|
|
5
5
|
Project-URL: Homepage, https://github.com/pablobernabeu/scopusflow-py
|
|
6
6
|
Project-URL: Documentation, https://pablobernabeu.github.io/scopusflow-py/
|
|
@@ -82,7 +82,7 @@ The example below builds a query, plans a harvest partitioned by year, retrieves
|
|
|
82
82
|
import scopusflow as sf
|
|
83
83
|
|
|
84
84
|
q = sf.scopus_query("graphene", "supercapacitor", field="TITLE-ABS-KEY")
|
|
85
|
-
plan = sf.SearchPlan(q, years=range(
|
|
85
|
+
plan = sf.SearchPlan(q, years=range(2015, 2025), partition="year")
|
|
86
86
|
|
|
87
87
|
records = sf.fetch_plan(plan, cache_dir="harvest", resume=True)
|
|
88
88
|
|
|
@@ -101,7 +101,7 @@ The publication trend can be tallied from a harvest you already hold, or fetched
|
|
|
101
101
|
|
|
102
102
|
```python
|
|
103
103
|
trend = sf.year_counts(records)
|
|
104
|
-
trend = sf.scopus_trend(q, years=range(
|
|
104
|
+
trend = sf.scopus_trend(q, years=range(2015, 2025))
|
|
105
105
|
```
|
|
106
106
|
|
|
107
107
|
A topic comparison shows how sub-topics grow within the reference literature over time. The abstract and export helpers carry the work onward into reading and into a reference manager.
|
|
@@ -116,11 +116,11 @@ with open("scopus-records.bib", "w", encoding="utf-8") as fh:
|
|
|
116
116
|
fh.write(sf.to_bibtex(records))
|
|
117
117
|
```
|
|
118
118
|
|
|
119
|
-
The pure-logic helpers, from query building to DOI tracking, need no API key and are exercised by the offline tests. Everything that contacts the API needs a key, and the plots need the optional `plot` extra.
|
|
119
|
+
The pure-logic helpers, from query building to DOI tracking, need no API key and are exercised by the offline tests. Everything that contacts the API needs a key, and the plots need the optional `plot` extra. To try the analysis half before configuring anything, `sf.example_records()` returns a bundled harvest of 138 real articles in the same schema. It is not a Scopus harvest, since retrieved records may not be redistributed; the [documentation](https://pablobernabeu.github.io/scopusflow-py/guides/getting-started/#the-bundled-harvest) explains where it comes from.
|
|
120
120
|
|
|
121
121
|
## A code-free app
|
|
122
122
|
|
|
123
|
-
A local [NiceGUI](https://nicegui.io) app drives the whole workflow without writing code, and mirrors every choice back as a runnable Python script, so it works as an on-ramp to the package rather than a replacement. It runs on your own machine, so your API key never leaves it, and a demo mode lets you try the flow with
|
|
123
|
+
A local [NiceGUI](https://nicegui.io) app drives the whole workflow without writing code, and mirrors every choice back as a runnable Python script, so it works as an on-ramp to the package rather than a replacement. It runs on your own machine, so your API key never leaves it, and a demo mode lets you try the flow with no key at all, replaying the bundled set of real published articles in place of a harvest.
|
|
124
124
|
|
|
125
125
|
```bash
|
|
126
126
|
pip install "scopusflow[app]"
|
|
@@ -133,7 +133,7 @@ The retrieval runs in the background with a live progress terminal. Results appe
|
|
|
133
133
|
|
|
134
134
|
If scopusflow contributes to published work, please cite it. GitHub builds a ready-made citation from [`CITATION.cff`](CITATION.cff) through the *Cite this repository* button.
|
|
135
135
|
|
|
136
|
-
> Bernabeu, P. (2026). scopusflow: A reproducible workflow layer over pybliometrics for Scopus searches. Python package version 0.
|
|
136
|
+
> Bernabeu, P. (2026). scopusflow: A reproducible workflow layer over pybliometrics for Scopus searches. Python package version 0.3.0. https://doi.org/10.5281/zenodo.21252666
|
|
137
137
|
|
|
138
138
|
The [About page](https://pablobernabeu.github.io/scopusflow-py/about/) carries the same citation with a BibTeX entry.
|
|
139
139
|
|
|
@@ -44,7 +44,7 @@ The example below builds a query, plans a harvest partitioned by year, retrieves
|
|
|
44
44
|
import scopusflow as sf
|
|
45
45
|
|
|
46
46
|
q = sf.scopus_query("graphene", "supercapacitor", field="TITLE-ABS-KEY")
|
|
47
|
-
plan = sf.SearchPlan(q, years=range(
|
|
47
|
+
plan = sf.SearchPlan(q, years=range(2015, 2025), partition="year")
|
|
48
48
|
|
|
49
49
|
records = sf.fetch_plan(plan, cache_dir="harvest", resume=True)
|
|
50
50
|
|
|
@@ -63,7 +63,7 @@ The publication trend can be tallied from a harvest you already hold, or fetched
|
|
|
63
63
|
|
|
64
64
|
```python
|
|
65
65
|
trend = sf.year_counts(records)
|
|
66
|
-
trend = sf.scopus_trend(q, years=range(
|
|
66
|
+
trend = sf.scopus_trend(q, years=range(2015, 2025))
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
A topic comparison shows how sub-topics grow within the reference literature over time. The abstract and export helpers carry the work onward into reading and into a reference manager.
|
|
@@ -78,11 +78,11 @@ with open("scopus-records.bib", "w", encoding="utf-8") as fh:
|
|
|
78
78
|
fh.write(sf.to_bibtex(records))
|
|
79
79
|
```
|
|
80
80
|
|
|
81
|
-
The pure-logic helpers, from query building to DOI tracking, need no API key and are exercised by the offline tests. Everything that contacts the API needs a key, and the plots need the optional `plot` extra.
|
|
81
|
+
The pure-logic helpers, from query building to DOI tracking, need no API key and are exercised by the offline tests. Everything that contacts the API needs a key, and the plots need the optional `plot` extra. To try the analysis half before configuring anything, `sf.example_records()` returns a bundled harvest of 138 real articles in the same schema. It is not a Scopus harvest, since retrieved records may not be redistributed; the [documentation](https://pablobernabeu.github.io/scopusflow-py/guides/getting-started/#the-bundled-harvest) explains where it comes from.
|
|
82
82
|
|
|
83
83
|
## A code-free app
|
|
84
84
|
|
|
85
|
-
A local [NiceGUI](https://nicegui.io) app drives the whole workflow without writing code, and mirrors every choice back as a runnable Python script, so it works as an on-ramp to the package rather than a replacement. It runs on your own machine, so your API key never leaves it, and a demo mode lets you try the flow with
|
|
85
|
+
A local [NiceGUI](https://nicegui.io) app drives the whole workflow without writing code, and mirrors every choice back as a runnable Python script, so it works as an on-ramp to the package rather than a replacement. It runs on your own machine, so your API key never leaves it, and a demo mode lets you try the flow with no key at all, replaying the bundled set of real published articles in place of a harvest.
|
|
86
86
|
|
|
87
87
|
```bash
|
|
88
88
|
pip install "scopusflow[app]"
|
|
@@ -95,7 +95,7 @@ The retrieval runs in the background with a live progress terminal. Results appe
|
|
|
95
95
|
|
|
96
96
|
If scopusflow contributes to published work, please cite it. GitHub builds a ready-made citation from [`CITATION.cff`](CITATION.cff) through the *Cite this repository* button.
|
|
97
97
|
|
|
98
|
-
> Bernabeu, P. (2026). scopusflow: A reproducible workflow layer over pybliometrics for Scopus searches. Python package version 0.
|
|
98
|
+
> Bernabeu, P. (2026). scopusflow: A reproducible workflow layer over pybliometrics for Scopus searches. Python package version 0.3.0. https://doi.org/10.5281/zenodo.21252666
|
|
99
99
|
|
|
100
100
|
The [About page](https://pablobernabeu.github.io/scopusflow-py/about/) carries the same citation with a BibTeX entry.
|
|
101
101
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
```python exec="1" session="reference-analyse"
|
|
2
|
+
import html as _html
|
|
3
|
+
import matplotlib
|
|
4
|
+
matplotlib.use("Agg")
|
|
5
|
+
import matplotlib.pyplot as plt
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import scopusflow as sf
|
|
8
|
+
from io import StringIO
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _clean_table(html_table):
|
|
12
|
+
# Drop pandas' own class and header alignment so the theme's table
|
|
13
|
+
# styling (padding, borders, zebra rows) applies instead.
|
|
14
|
+
return (html_table.replace(' class="dataframe"', "")
|
|
15
|
+
.replace(' style="text-align: right;"', ""))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def out(x):
|
|
19
|
+
if isinstance(x, pd.DataFrame):
|
|
20
|
+
print(_clean_table(x.to_html(index=False, border=0)))
|
|
21
|
+
elif isinstance(x, pd.Series):
|
|
22
|
+
label = x.index.name or "value"
|
|
23
|
+
df = x.rename("count").reset_index()
|
|
24
|
+
df.columns = [label, "count"]
|
|
25
|
+
print(_clean_table(df.to_html(index=False, border=0)))
|
|
26
|
+
else:
|
|
27
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def show():
|
|
31
|
+
plt.gcf().set_size_inches(7.2, 3.8)
|
|
32
|
+
plt.tight_layout()
|
|
33
|
+
plt.gcf().canvas.draw() # let the label de-collision settle before saving
|
|
34
|
+
buffer = StringIO()
|
|
35
|
+
# Transparent so the page background shows through in both colour schemes;
|
|
36
|
+
# matplotlib otherwise paints an opaque white figure and axes patch.
|
|
37
|
+
plt.savefig(buffer, format="svg", transparent=True)
|
|
38
|
+
plt.close()
|
|
39
|
+
print(buffer.getvalue())
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# The bundled worked-example harvest, over the RECORD_COLUMNS schema, so the
|
|
43
|
+
# examples below run at build time without a key and without contacting the API.
|
|
44
|
+
records = sf.example_records()
|
|
45
|
+
```
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
If scopusflow contributes to published work, please cite it.
|
|
6
6
|
|
|
7
7
|
> Bernabeu, P. (2026). scopusflow: A reproducible workflow layer over
|
|
8
|
-
> pybliometrics for Scopus searches. Python package version 0.
|
|
8
|
+
> pybliometrics for Scopus searches. Python package version 0.3.0.
|
|
9
9
|
> https://doi.org/10.5281/zenodo.21252666
|
|
10
10
|
|
|
11
11
|
```
|
|
@@ -13,7 +13,7 @@ If scopusflow contributes to published work, please cite it.
|
|
|
13
13
|
title = {scopusflow: A reproducible workflow layer over pybliometrics for {Scopus} searches},
|
|
14
14
|
author = {Pablo Bernabeu},
|
|
15
15
|
year = {2026},
|
|
16
|
-
note = {Python package version 0.
|
|
16
|
+
note = {Python package version 0.3.0},
|
|
17
17
|
doi = {10.5281/zenodo.21252666},
|
|
18
18
|
url = {https://doi.org/10.5281/zenodo.21252666},
|
|
19
19
|
}
|
|
Binary file
|
|
Binary file
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
# Analysing a literature
|
|
2
2
|
|
|
3
|
-
Once records are in hand, scopusflow turns them into the figures a bibliometric study usually needs, all from the one stable schema. The examples here run on a
|
|
3
|
+
Once records are in hand, scopusflow turns them into the figures a bibliometric study usually needs, all from the one stable schema. The examples here run on the harvest bundled with the package, which holds 138 real articles on graphene supercapacitors published between 2015 and 2024. It stands in for a Scopus retrieval because retrieved records cannot be redistributed, and it lets every figure on this page be drawn without a key. The live call that would produce `records` in practice is shown alongside, and [Get started](getting-started.md#the-bundled-harvest) gives the fuller account of where the bundled set comes from.
|
|
4
4
|
|
|
5
5
|
```python exec="1" session="analysing-a-literature"
|
|
6
6
|
import html as _html
|
|
7
|
-
import random
|
|
8
7
|
import matplotlib
|
|
9
8
|
matplotlib.use("Agg")
|
|
10
9
|
import matplotlib.pyplot as plt
|
|
@@ -29,38 +28,35 @@ def out(x):
|
|
|
29
28
|
df.columns = [label, "count"]
|
|
30
29
|
print(_clean_table(df.to_html(index=False, border=0)))
|
|
31
30
|
else:
|
|
32
|
-
print("<pre>" + _html.escape(str(x)) + "</pre>")
|
|
31
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
33
32
|
|
|
34
33
|
|
|
35
34
|
def show():
|
|
36
35
|
plt.tight_layout()
|
|
37
36
|
plt.gcf().canvas.draw() # let the label de-collision settle before saving
|
|
38
37
|
buffer = StringIO()
|
|
39
|
-
|
|
38
|
+
# Transparent so the page background shows through in both colour schemes;
|
|
39
|
+
# matplotlib otherwise paints an opaque white figure and axes patch.
|
|
40
|
+
plt.savefig(buffer, format="svg", transparent=True)
|
|
40
41
|
plt.close()
|
|
41
42
|
print(buffer.getvalue())
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Where the records come from
|
|
46
|
+
|
|
47
|
+
A live harvest is two calls, a query and a plan run through [`fetch_plan`][scopusflow.fetch.fetch_plan], and both need a configured key.
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
q = sf.scopus_query("graphene", "supercapacitor", field="TITLE-ABS-KEY")
|
|
51
|
+
plan = sf.SearchPlan(q, years=range(2015, 2025), partition="year")
|
|
52
|
+
records = sf.fetch_plan(plan, cache_dir="graphene-harvest")
|
|
53
|
+
```
|
|
42
54
|
|
|
55
|
+
[`example_records`][scopusflow.data.example_records] returns the bundled equivalent, already normalised into the same schema, which is what the rest of this page works on.
|
|
43
56
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
# and its plot show a clear ordering rather than a near-uniform split.
|
|
48
|
-
picker = random.Random(8)
|
|
49
|
-
rows = []
|
|
50
|
-
for yi, year in enumerate(range(2016, 2023)):
|
|
51
|
-
for j in range(4 + yi * 2):
|
|
52
|
-
rows.append({
|
|
53
|
-
"entry_number": len(rows) + 1,
|
|
54
|
-
"scopus_id": f"{year}{j:03d}",
|
|
55
|
-
"doi": f"10.1000/demo.{year}.{j:03d}",
|
|
56
|
-
"title": f"Record {j + 1} from {year}",
|
|
57
|
-
"authors": authors[j % len(authors)] + ";" + authors[(j + 1) % len(authors)],
|
|
58
|
-
"year": year, "date": f"{year}-01-01",
|
|
59
|
-
"publication": picker.choices(sources, weights=(9, 7, 5, 3, 2))[0],
|
|
60
|
-
"citations": (j * 7 + year) % 120,
|
|
61
|
-
"query": "graphene supercapacitor",
|
|
62
|
-
})
|
|
63
|
-
records = pd.DataFrame(rows, columns=sf.RECORD_COLUMNS)
|
|
57
|
+
```python exec="1" source="material-block" session="analysing-a-literature"
|
|
58
|
+
records = sf.example_records()
|
|
59
|
+
out(records[["title", "year", "publication", "citations"]].head(3))
|
|
64
60
|
```
|
|
65
61
|
|
|
66
62
|
## What is in a record set
|
|
@@ -68,13 +64,15 @@ records = pd.DataFrame(rows, columns=sf.RECORD_COLUMNS)
|
|
|
68
64
|
[`top`][scopusflow.records.top] tallies the most frequent sources or authors. Author strings that hold several names are split, so each contributor is counted once per record.
|
|
69
65
|
|
|
70
66
|
```python exec="1" source="material-block" session="analysing-a-literature"
|
|
71
|
-
out(sf.top(records, by="source"))
|
|
67
|
+
out(sf.top(records, by="source", n=5))
|
|
72
68
|
out(sf.top(records, by="author", n=6))
|
|
73
69
|
```
|
|
74
70
|
|
|
71
|
+
The tally has a long tail, as a topic literature usually does. ACS Applied Materials & Interfaces heads it with eight records, while 77 of the 90 journals represented appear only once or twice. The two records whose source title is missing are dropped from the tally rather than counted as an empty name.
|
|
72
|
+
|
|
75
73
|
## How a literature grows
|
|
76
74
|
|
|
77
|
-
[`year_counts`][scopusflow.trend.year_counts] is the offline tally of records per year from a set you already hold.
|
|
75
|
+
[`year_counts`][scopusflow.trend.year_counts] is the offline tally of records per year from a set you already hold. Because the bundled harvest is a complete pull rather than a sample, these counts are the publications the query matched in each year.
|
|
78
76
|
|
|
79
77
|
```python exec="1" source="material-block" session="analysing-a-literature"
|
|
80
78
|
trend = sf.year_counts(records)
|
|
@@ -84,7 +82,7 @@ out(trend)
|
|
|
84
82
|
[`scopus_trend`][scopusflow.trend.scopus_trend] instead asks the API for the count in each year without downloading the records, which is far cheaper when all you want is the shape of the growth. It needs a key, so it is shown rather than run.
|
|
85
83
|
|
|
86
84
|
```python
|
|
87
|
-
trend = sf.scopus_trend(q, years=range(
|
|
85
|
+
trend = sf.scopus_trend(q, years=range(2015, 2025))
|
|
88
86
|
```
|
|
89
87
|
|
|
90
88
|
## Turn it into figures
|
|
@@ -149,26 +147,21 @@ abstracts = sf.scopus_abstract(dois[:10], by="doi")
|
|
|
149
147
|
abstracts[["doi", "title", "year"]]
|
|
150
148
|
```
|
|
151
149
|
|
|
152
|
-
The result is one row per identifier, over the stable `ABSTRACT_COLUMNS` schema.
|
|
150
|
+
The result is one row per identifier, over the stable `ABSTRACT_COLUMNS` schema. The bundled harvest carries the bibliographic fields but no abstract text, so the frame below takes its two most-cited records and leaves the one column the corpus cannot supply as a placeholder. Everything else in those rows is the real record.
|
|
153
151
|
|
|
154
152
|
```python exec="1" source="material-block" session="analysing-a-literature"
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
"capacitance retention benchmarked against commercial baselines.",
|
|
167
|
-
"publication": "Science", "date": "2020-11-20", "year": 2020,
|
|
168
|
-
"citations": 88},
|
|
169
|
-
], columns=sf.abstract.ABSTRACT_COLUMNS)
|
|
153
|
+
most_cited = records.nlargest(2, "citations")
|
|
154
|
+
abstracts = pd.DataFrame({
|
|
155
|
+
"scopus_id": pd.NA,
|
|
156
|
+
"doi": most_cited["doi"],
|
|
157
|
+
"title": most_cited["title"],
|
|
158
|
+
"abstract": "(the bundled harvest carries no abstract text)",
|
|
159
|
+
"publication": most_cited["publication"],
|
|
160
|
+
"date": most_cited["date"],
|
|
161
|
+
"year": most_cited["year"],
|
|
162
|
+
"citations": most_cited["citations"],
|
|
163
|
+
}, columns=sf.abstract.ABSTRACT_COLUMNS)
|
|
170
164
|
out(abstracts[["title", "publication", "year", "citations"]])
|
|
171
|
-
out(abstracts.loc[0, "abstract"][:40] + "...")
|
|
172
165
|
```
|
|
173
166
|
|
|
174
167
|
A failed identifier does not stop the batch, but yields an all-NA row that still records the id, together with a warning.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Building a reference set
|
|
2
2
|
|
|
3
|
-
A retrieval becomes useful once it leaves the package, as a reading list in a reference manager or as input to a writing project. This guide covers that export end of the workflow, taking a set of records from the API through a clean DOI list and on into the two interchange formats that reference managers read.
|
|
3
|
+
A retrieval becomes useful once it leaves the package, as a reading list in a reference manager or as input to a writing project. This guide covers that export end of the workflow, taking a set of records from the API through a clean DOI list and on into the two interchange formats that reference managers read. The worked output runs on the harvest bundled with the package, 138 real articles on graphene supercapacitors, since records retrieved from Scopus may not be redistributed and no key is assumed here. The live call that would replace it is shown first.
|
|
4
4
|
|
|
5
5
|
```python exec="1" session="building-a-reference-set"
|
|
6
6
|
import scopusflow as sf
|
|
@@ -24,26 +24,7 @@ def out(x):
|
|
|
24
24
|
df.columns = [label, "count"]
|
|
25
25
|
print(_clean_table(df.to_html(index=False, border=0)))
|
|
26
26
|
else:
|
|
27
|
-
print("<pre>" + _html.escape(str(x)) + "</pre>")
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
sources = ["Nature", "Science", "Carbon", "Nano Letters", "Advanced Materials"]
|
|
31
|
-
authors = ["Lee J.", "Park S.", "Kim H.", "Garcia M.", "Zhang F.", "Abbott B."]
|
|
32
|
-
rows = []
|
|
33
|
-
for j, year in enumerate(range(2018, 2023)):
|
|
34
|
-
for k in range(4):
|
|
35
|
-
rows.append({
|
|
36
|
-
"entry_number": len(rows) + 1,
|
|
37
|
-
"scopus_id": f"{year}{k:03d}",
|
|
38
|
-
"doi": f"10.1000/demo.{year}.{k:03d}",
|
|
39
|
-
"title": f"A study of graphene supercapacitors, part {len(rows) + 1}",
|
|
40
|
-
"authors": authors[k % len(authors)] + ";" + authors[(k + 1) % len(authors)],
|
|
41
|
-
"year": year, "date": f"{year}-01-01",
|
|
42
|
-
"publication": sources[(year + k) % len(sources)],
|
|
43
|
-
"citations": (k * 7 + year) % 120,
|
|
44
|
-
"query": "graphene supercapacitor",
|
|
45
|
-
})
|
|
46
|
-
records = pd.DataFrame(rows, columns=sf.RECORD_COLUMNS)
|
|
27
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
47
28
|
```
|
|
48
29
|
|
|
49
30
|
## Fetch the record set
|
|
@@ -54,15 +35,16 @@ Everything here starts from a frame of records. You build one by running a [`Sea
|
|
|
54
35
|
import scopusflow as sf
|
|
55
36
|
|
|
56
37
|
q = sf.scopus_query("graphene", "supercapacitor", field="TITLE-ABS-KEY")
|
|
57
|
-
plan = sf.SearchPlan(q, years=range(
|
|
38
|
+
plan = sf.SearchPlan(q, years=range(2015, 2025), partition="year")
|
|
58
39
|
|
|
59
40
|
records = sf.fetch_plan(plan, cache_dir="graphene-harvest")
|
|
60
41
|
records.shape
|
|
61
42
|
```
|
|
62
43
|
|
|
63
|
-
|
|
44
|
+
[`example_records`][scopusflow.data.example_records] returns the bundled stand-in for that harvest, which is what the output below runs on. The result is an ordinary pandas DataFrame underneath, so it drops straight into any analysis you already have. The columns are always the same whatever the query was, which is what lets the DOI and export helpers that follow rely on them.
|
|
64
45
|
|
|
65
46
|
```python exec="1" source="material-block" session="building-a-reference-set"
|
|
47
|
+
records = sf.example_records()
|
|
66
48
|
out(records[["title", "year", "doi"]].head())
|
|
67
49
|
```
|
|
68
50
|
|
|
@@ -75,13 +57,26 @@ dois = sf.extract_dois(records)
|
|
|
75
57
|
out(dois[:5])
|
|
76
58
|
```
|
|
77
59
|
|
|
78
|
-
The function works offline because it only reads the frame you already hold. It returns a plain Python list rather than writing a file, so you stay in control of where anything lands.
|
|
60
|
+
The function works offline because it only reads the frame you already hold. It returns a plain Python list rather than writing a file, so you stay in control of where anything lands. The list runs to 127 entries against 138 records, because the eleven records that carry no DOI are dropped rather than passed on as blanks.
|
|
61
|
+
|
|
62
|
+
De-duplication earns its keep once two retrievals are put together. Concatenating a pull of 2015 to 2020 with a later one of 2019 onwards repeats every record in the two overlapping years, and a reference manager fed the raw column would import each of those twice. To show what the cleaning has to see through, the second copy of one DOI is rewritten here into the resolver form an aggregator often hands back.
|
|
79
63
|
|
|
80
64
|
```python exec="1" source="material-block" session="building-a-reference-set"
|
|
81
|
-
|
|
82
|
-
|
|
65
|
+
import pandas as pd
|
|
66
|
+
|
|
67
|
+
combined = pd.concat(
|
|
68
|
+
[records[records["year"] <= 2020], records[records["year"] >= 2019]],
|
|
69
|
+
ignore_index=True,
|
|
70
|
+
)
|
|
71
|
+
repeat = combined.index[combined.duplicated("entry_number") & combined["doi"].notna()][0]
|
|
72
|
+
combined.loc[repeat, "doi"] = "https://doi.org/" + combined.loc[repeat, "doi"].upper()
|
|
73
|
+
|
|
74
|
+
all_dois = sf.extract_dois(combined, dedupe=False)
|
|
75
|
+
out((len(combined), len(all_dois), len(sf.extract_dois(combined))))
|
|
83
76
|
```
|
|
84
77
|
|
|
78
|
+
The 170 concatenated records yield 155 raw DOIs, the difference being the records with none, and 127 once duplicates are removed. The rewritten entry collapses into its plain twin, since the comparison ignores both the resolver prefix and the letter case. Subtracting one count from the other is the quickest way to see how much overlap two retrievals produced. Pass `dedupe=False` when you want the repeats kept, for instance to count how often a DOI recurs across cells.
|
|
79
|
+
|
|
85
80
|
Writing the list out is then a one-liner with the standard library, which keeps the package free of any implicit filesystem writes.
|
|
86
81
|
|
|
87
82
|
```python
|
|
@@ -92,20 +87,25 @@ Path("reference-set.txt").write_text("\n".join(dois), encoding="utf-8")
|
|
|
92
87
|
|
|
93
88
|
## Render to BibTeX and RIS
|
|
94
89
|
|
|
95
|
-
A DOI list is enough for an import-by-identifier, but a full record carries more. [`to_bibtex`][scopusflow.export.to_bibtex] and [`to_ris`][scopusflow.export.to_ris] render the set in the two formats that reference managers read, so a search moves straight into Zotero, EndNote, Mendeley or a LaTeX bibliography. Each record becomes one entry
|
|
90
|
+
A DOI list is enough for an import-by-identifier, but a full record carries more. [`to_bibtex`][scopusflow.export.to_bibtex] and [`to_ris`][scopusflow.export.to_ris] render the set in the two formats that reference managers read, so a search moves straight into Zotero, EndNote, Mendeley or a LaTeX bibliography. Each record becomes one entry, and both functions are pure and offline, returning a string rather than touching disk. The first three records make the point, the second of them being one of the eleven with no DOI, which simply loses its `DO` line rather than carrying an empty one.
|
|
96
91
|
|
|
97
92
|
```python exec="1" source="material-block" session="building-a-reference-set"
|
|
98
|
-
|
|
99
|
-
out(ris[:320])
|
|
93
|
+
out(sf.to_ris(records.head(3)))
|
|
100
94
|
```
|
|
101
95
|
|
|
102
|
-
|
|
96
|
+
An author string holding several names is split into one `AU` line each. That is not visible above, because the bundled records name only the first author of each paper, whereas a live Scopus harvest returns the full semicolon-joined list and every name in it gets its own line.
|
|
97
|
+
|
|
98
|
+
BibTeX works the same way, one `@article` entry per record. The citation keys are built from the leading word of the first author's name and the year, and made unique within the export so that biber does not reject duplicates. Scopus returns authors surname first, so that leading word is normally the surname; the bundled records give the given name first, which is why the keys below read the way they do. The three records here are chosen to show the two details worth checking, a journal name carrying an ampersand, which BibTeX has to escape, and two papers by the same first author in the same year, whose keys would otherwise collide.
|
|
103
99
|
|
|
104
100
|
```python exec="1" source="material-block" session="building-a-reference-set"
|
|
105
|
-
|
|
106
|
-
|
|
101
|
+
ampersand = records[records["publication"].fillna("").str.contains("&")].head(1)
|
|
102
|
+
same_author = records[(records["authors"] == "Hao Yang") & (records["year"] == 2017)]
|
|
103
|
+
|
|
104
|
+
out(sf.to_bibtex(pd.concat([ampersand, same_author])))
|
|
107
105
|
```
|
|
108
106
|
|
|
107
|
+
The `note` field that ordinarily carries the Scopus identifier is absent throughout, since the bundled records have none. A live harvest fills it, and drops it for any record where Scopus itself returned no identifier.
|
|
108
|
+
|
|
109
109
|
## Save the export
|
|
110
110
|
|
|
111
111
|
Because both functions hand back a string, writing the whole set is again a plain file write, in whichever format your downstream tool expects. A `.bib` file feeds a LaTeX bibliography directly, while a `.ris` file imports into most reference managers.
|
|
@@ -27,7 +27,7 @@ def out(x):
|
|
|
27
27
|
df.columns = [label, "count"]
|
|
28
28
|
print(_clean_table(df.to_html(index=False, border=0)))
|
|
29
29
|
else:
|
|
30
|
-
print("<pre>" + _html.escape(str(x)) + "</pre>")
|
|
30
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def show():
|
|
@@ -35,7 +35,9 @@ def show():
|
|
|
35
35
|
plt.tight_layout()
|
|
36
36
|
plt.gcf().canvas.draw() # let the label de-collision settle before saving
|
|
37
37
|
buffer = StringIO()
|
|
38
|
-
|
|
38
|
+
# Transparent so the page background shows through in both colour schemes;
|
|
39
|
+
# matplotlib otherwise paints an opaque white figure and axes patch.
|
|
40
|
+
plt.savefig(buffer, format="svg", transparent=True)
|
|
39
41
|
plt.close()
|
|
40
42
|
print(buffer.getvalue())
|
|
41
43
|
```
|
|
@@ -179,7 +181,10 @@ The return value is an ordinary matplotlib `Axes`, so a different style, a saved
|
|
|
179
181
|
When there are only a few topics the lines are labelled directly, so no legend is needed. Beyond a handful the labels would collide, so [`plot_comparison`][scopusflow.plots.plot_comparison] falls back to a legend instead. By default matplotlib chooses where the legend sits, but `legend_inside=True` places it inside the axes in whichever corner has the most free space (here the top-left, which these rising lines leave empty), which saves the width an outside legend would otherwise take.
|
|
180
182
|
|
|
181
183
|
```python exec="1" source="material-block" html="1" session="comparing-topics"
|
|
182
|
-
many =
|
|
184
|
+
many = ["lithium-ion", "sodium-ion", "solid-state electrolytes",
|
|
185
|
+
"silicon anodes", "lithium-sulfur", "redox flow",
|
|
186
|
+
"supercapacitors", "thermoelectrics", "photocatalysis",
|
|
187
|
+
"fuel cells"]
|
|
183
188
|
ref_n = np.linspace(500, 2000, len(years)).round().astype(int)
|
|
184
189
|
|
|
185
190
|
rows = []
|
|
@@ -196,7 +201,7 @@ for i, topic in enumerate(many):
|
|
|
196
201
|
pct = end * (0.5 + 0.5 * j / (len(years) - 1))
|
|
197
202
|
rows.append({
|
|
198
203
|
"query": topic, "query_type": "comparison",
|
|
199
|
-
"abridged_query":
|
|
204
|
+
"abridged_query": topic, "year": year,
|
|
200
205
|
"n": int(pct * n / 100), "reference_n": int(n),
|
|
201
206
|
"comparison_percentage": pct, "average_comparison_percentage": end,
|
|
202
207
|
})
|
|
@@ -206,6 +211,11 @@ ax = sf.plot_comparison(cmp_many, legend_inside=True)
|
|
|
206
211
|
show()
|
|
207
212
|
```
|
|
208
213
|
|
|
214
|
+
The legend has taken the top-left corner, which the rising lines leave free, and
|
|
215
|
+
it has split into two columns because there are more than eight topics. Names of
|
|
216
|
+
this length are what the corner has to accommodate in practice, so the fit is
|
|
217
|
+
worth checking against real topics rather than short placeholders.
|
|
218
|
+
|
|
209
219
|
## Reading the result as a table
|
|
210
220
|
|
|
211
221
|
Sometimes the numbers matter more than the picture. Because the output is a pandas frame, the usual tools apply. Here are the comparison topics ranked by their average share.
|
|
@@ -24,9 +24,9 @@ def out(x):
|
|
|
24
24
|
df.columns = [label, "count"]
|
|
25
25
|
print(_clean_table(df.to_html(index=False, border=0)))
|
|
26
26
|
elif isinstance(x, list) and all(isinstance(i, str) for i in x):
|
|
27
|
-
print("<pre>" + _html.escape(chr(10).join(x)) + "</pre>")
|
|
27
|
+
print("<pre><code>" + _html.escape(chr(10).join(x)) + "</code></pre>")
|
|
28
28
|
else:
|
|
29
|
-
print("<pre>" + _html.escape(str(x)) + "</pre>")
|
|
29
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
```python exec="1" source="material-block" session="designing-queries"
|
|
@@ -23,7 +23,7 @@ def out(x):
|
|
|
23
23
|
df.columns = [label, "count"]
|
|
24
24
|
print(_clean_table(df.to_html(index=False, border=0)))
|
|
25
25
|
else:
|
|
26
|
-
print("<pre>" + _html.escape(str(x)) + "</pre>")
|
|
26
|
+
print("<pre><code>" + _html.escape(str(x)) + "</code></pre>")
|
|
27
27
|
```
|
|
28
28
|
|
|
29
29
|
```python exec="1" source="material-block" session="getting-started"
|
|
@@ -42,7 +42,7 @@ out(q)
|
|
|
42
42
|
A [`SearchPlan`][scopusflow.plan.SearchPlan] wraps that query in a description you can print, version-control and partition. Partitioning by year keeps each cell small enough to stay under the API's offset ceiling, and it makes the harvest resumable cell by cell.
|
|
43
43
|
|
|
44
44
|
```python exec="1" source="material-block" session="getting-started"
|
|
45
|
-
plan = sf.SearchPlan(q, years=range(
|
|
45
|
+
plan = sf.SearchPlan(q, years=range(2015, 2025), partition="year")
|
|
46
46
|
out([(c.cell, c.year) for c in plan.cells()])
|
|
47
47
|
```
|
|
48
48
|
|
|
@@ -54,34 +54,27 @@ A count tells you what a harvest will cost before you pay for it. Both calls bel
|
|
|
54
54
|
|
|
55
55
|
```python
|
|
56
56
|
# Both need a configured Scopus key.
|
|
57
|
-
sf.scopus_count(q, years=range(
|
|
57
|
+
sf.scopus_count(q, years=range(2015, 2025))
|
|
58
58
|
records = sf.fetch_plan(plan, cache_dir="harvest", resume=True)
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
-
##
|
|
61
|
+
## The bundled harvest
|
|
62
62
|
|
|
63
|
-
Whether records come from the API or from data you already hold, they share one schema, the columns named in [`RECORD_COLUMNS`][scopusflow.records.RECORD_COLUMNS].
|
|
63
|
+
Whether records come from the API or from data you already hold, they share one schema, the columns named in [`RECORD_COLUMNS`][scopusflow.records.RECORD_COLUMNS]. So that the rest of this guide, and every other page on this site, runs without a key, the package ships a worked example in exactly that shape. [`example_records`][scopusflow.data.example_records] returns it.
|
|
64
64
|
|
|
65
65
|
```python exec="1" source="material-block" session="getting-started"
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
records = pd.DataFrame([
|
|
69
|
-
{"entry_number": 1, "scopus_id": "1", "doi": "10.1000/aaa",
|
|
70
|
-
"title": "A reproducible workflow", "authors": "Smith J.;Doe A.",
|
|
71
|
-
"year": 2020, "date": "2020-05-01", "publication": "J. Bibliometrics",
|
|
72
|
-
"citations": 12, "query": q},
|
|
73
|
-
{"entry_number": 2, "scopus_id": "2", "doi": "10.1000/bbb",
|
|
74
|
-
"title": "Quota-aware querying", "authors": "Doe A.",
|
|
75
|
-
"year": 2021, "date": "2021-01-10", "publication": "Scientometrics Today",
|
|
76
|
-
"citations": 3, "query": q},
|
|
77
|
-
], columns=sf.RECORD_COLUMNS)
|
|
78
|
-
out(records[["title", "year", "publication", "citations"]])
|
|
66
|
+
records = sf.example_records()
|
|
67
|
+
out(records[["title", "year", "publication", "citations"]].head())
|
|
79
68
|
```
|
|
80
69
|
|
|
81
|
-
[`
|
|
70
|
+
Those are real journal articles on graphene supercapacitors, 138 of them published between 2015 and 2024, carrying their real titles, DOIs, journals, first authors and citation counts. They are deliberately not a Scopus harvest, and no package can ship one, because Elsevier's API terms do not permit redistributing the records a key retrieves. These come instead from OpenAlex, whose metadata is released under CC0 and may therefore travel with the package, reshaped into the schema [`fetch_plan`][scopusflow.fetch.fetch_plan] returns. Run the query above against Scopus and you get a frame of the same shape, with the same columns and the same handling, though not an identical set of records.
|
|
71
|
+
|
|
72
|
+
Two consequences are worth knowing before the examples below. The harvest is complete rather than sampled, so the number of rows in a year is the number of publications the query matched that year, and a trend drawn from it is a real publication curve. And `scopus_id` is empty throughout, these records not having come from Scopus, so anything that would key on the Scopus identifier falls back to the DOI. Eleven records carry no DOI and two no source title, left exactly as they arrive, because a real harvest has gaps of the same kind.
|
|
73
|
+
|
|
74
|
+
[`top`][scopusflow.records.top] tallies the most frequent sources or authors from the frame, counting each contributor once per record.
|
|
82
75
|
|
|
83
76
|
```python exec="1" source="material-block" session="getting-started"
|
|
84
|
-
out(sf.top(records, by="source"))
|
|
77
|
+
out(sf.top(records, by="source", n=5))
|
|
85
78
|
```
|
|
86
79
|
|
|
87
80
|
## Carry the work onward
|
|
@@ -89,7 +82,10 @@ out(sf.top(records, by="source"))
|
|
|
89
82
|
From the one schema the rest of the workflow follows. You can extract a clean DOI list, compare two harvests to see what changed, summarise growth over time and draw it, compare how sub-topics move within a literature, read fuller records, and export to a reference manager.
|
|
90
83
|
|
|
91
84
|
```python exec="1" source="material-block" session="getting-started"
|
|
92
|
-
|
|
85
|
+
dois = sf.extract_dois(records)
|
|
86
|
+
out((len(dois), dois[:3]))
|
|
93
87
|
```
|
|
94
88
|
|
|
89
|
+
The list is 127 entries long rather than 138, because the eleven records without a DOI are dropped rather than carried through as blanks.
|
|
90
|
+
|
|
95
91
|
Each of these has its own guide. [Building a reference set](building-a-reference-set.md) covers DOIs and export to BibTeX and RIS. [Tracking change over time](tracking-literature-change.md) covers comparing two retrievals. [Analysing a literature](analysing-a-literature.md) covers trends, top sources and authors, and abstracts. [Comparing topics](comparing-topics.md) covers the share of a reference literature each sub-topic holds over time. [Author keywords and references](keywords-and-references.md) covers retrieving a document's own keywords and reference list, and assembling a minimal corpus from them, both at a materially different quota cost from an ordinary search. [The code-free app](the-app.md) does all of it through a browser tab, with no code at all.
|