scopusflow 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scopusflow/__init__.py +51 -0
- scopusflow/abstract.py +315 -0
- scopusflow/app.py +561 -0
- scopusflow/app_helpers.py +115 -0
- scopusflow/compare.py +123 -0
- scopusflow/corpus.py +92 -0
- scopusflow/count.py +39 -0
- scopusflow/diff.py +58 -0
- scopusflow/exceptions.py +18 -0
- scopusflow/export.py +138 -0
- scopusflow/fetch.py +169 -0
- scopusflow/plan.py +63 -0
- scopusflow/plots.py +344 -0
- scopusflow/py.typed +0 -0
- scopusflow/query.py +53 -0
- scopusflow/records.py +85 -0
- scopusflow/trend.py +55 -0
- scopusflow-0.1.0.dist-info/METADATA +146 -0
- scopusflow-0.1.0.dist-info/RECORD +22 -0
- scopusflow-0.1.0.dist-info/WHEEL +4 -0
- scopusflow-0.1.0.dist-info/entry_points.txt +2 -0
- scopusflow-0.1.0.dist-info/licenses/LICENSE +21 -0
scopusflow/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""scopusflow: a reproducible workflow layer over pybliometrics.
|
|
2
|
+
|
|
3
|
+
pybliometrics provides the retrieval primitives for the Scopus API; scopusflow
|
|
4
|
+
adds the workflow on top: reproducible search plans, a single stable record
|
|
5
|
+
schema, resumable checkpointed harvesting, and DOI change-tracking.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .abstract import scopus_abstract
|
|
11
|
+
from .compare import compare_topics
|
|
12
|
+
from .corpus import corpus
|
|
13
|
+
from .count import scopus_count
|
|
14
|
+
from .diff import diff_dois, extract_dois
|
|
15
|
+
from .exceptions import ScopusFlowForbiddenError
|
|
16
|
+
from .export import to_bibtex, to_ris
|
|
17
|
+
from .fetch import fetch_plan
|
|
18
|
+
from .plan import PlanCell, SearchPlan
|
|
19
|
+
from .plots import plot_comparison, plot_top, plot_trend
|
|
20
|
+
from .query import FIELD_TAGS, scopus_query, wrap_field
|
|
21
|
+
from .records import RECORD_COLUMNS, to_records, top
|
|
22
|
+
from .trend import scopus_trend, year_counts
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"SearchPlan",
|
|
28
|
+
"PlanCell",
|
|
29
|
+
"scopus_query",
|
|
30
|
+
"wrap_field",
|
|
31
|
+
"FIELD_TAGS",
|
|
32
|
+
"to_records",
|
|
33
|
+
"top",
|
|
34
|
+
"RECORD_COLUMNS",
|
|
35
|
+
"fetch_plan",
|
|
36
|
+
"extract_dois",
|
|
37
|
+
"diff_dois",
|
|
38
|
+
"year_counts",
|
|
39
|
+
"scopus_trend",
|
|
40
|
+
"scopus_count",
|
|
41
|
+
"compare_topics",
|
|
42
|
+
"scopus_abstract",
|
|
43
|
+
"ScopusFlowForbiddenError",
|
|
44
|
+
"corpus",
|
|
45
|
+
"to_bibtex",
|
|
46
|
+
"to_ris",
|
|
47
|
+
"plot_trend",
|
|
48
|
+
"plot_top",
|
|
49
|
+
"plot_comparison",
|
|
50
|
+
"__version__",
|
|
51
|
+
]
|
scopusflow/abstract.py
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
"""Batch abstract retrieval, resilient per id, mirroring ``scopus_abstract``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import pickle
|
|
7
|
+
import warnings
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .exceptions import ScopusFlowForbiddenError
|
|
13
|
+
|
|
14
|
+
#: Per-identifier progress is emitted on this logger; see fetch.py.
|
|
15
|
+
logger = logging.getLogger("scopusflow")
|
|
16
|
+
|
|
17
|
+
#: The stable column schema for retrieved abstracts.
|
|
18
|
+
ABSTRACT_COLUMNS = [
|
|
19
|
+
"scopus_id", "doi", "title", "abstract",
|
|
20
|
+
"publication", "date", "year", "citations",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
#: Map the user-facing ``by`` argument to a pybliometrics ``id_type``.
|
|
24
|
+
_ID_TYPES = {"doi": "doi", "eid": "eid", "scopus_id": "scopus_id"}
|
|
25
|
+
|
|
26
|
+
#: Values accepted by ``include``.
|
|
27
|
+
_KNOWN_INCLUDE = {"references", "keywords"}
|
|
28
|
+
|
|
29
|
+
#: Abstract Retrieval views that carry a bibliography.
|
|
30
|
+
_VIEWS_WITH_REFERENCES = {"FULL", "REF"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _get(obj, name):
|
|
34
|
+
if isinstance(obj, dict):
|
|
35
|
+
return obj.get(name)
|
|
36
|
+
return getattr(obj, name, None)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _references_frame(refs) -> pd.DataFrame:
|
|
40
|
+
"""Normalise pybliometrics' own ``Reference`` namedtuples into a DataFrame,
|
|
41
|
+
one row per cited work, keeping pybliometrics' full native field set (a
|
|
42
|
+
richer shape than the R package's leaner equivalent; see
|
|
43
|
+
:func:`scopus_abstract`'s docstring for the field-by-field difference)."""
|
|
44
|
+
from pybliometrics.scopus import Reference
|
|
45
|
+
|
|
46
|
+
if not refs:
|
|
47
|
+
return pd.DataFrame(columns=list(Reference._fields))
|
|
48
|
+
return pd.DataFrame([r._asdict() for r in refs])
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _abstract_row(obj, include: tuple[str, ...] = ()) -> dict:
|
|
52
|
+
"""Normalise a single ``AbstractRetrieval``-like object (or dict) into the
|
|
53
|
+
stable :data:`ABSTRACT_COLUMNS` mapping; offline, no network. When
|
|
54
|
+
"references" is included, a mismatch between the number of references
|
|
55
|
+
returned and the document's own reported ``refcount`` is warned about,
|
|
56
|
+
since the list may be an incomplete page rather than the whole
|
|
57
|
+
bibliography."""
|
|
58
|
+
eid = _get(obj, "eid")
|
|
59
|
+
scopus_id = str(eid).split("2-s2.0-")[-1] if eid else pd.NA
|
|
60
|
+
date = _get(obj, "coverDate")
|
|
61
|
+
head = str(date)[:4] if date else ""
|
|
62
|
+
year = int(head) if head.isdigit() else pd.NA
|
|
63
|
+
cited = _get(obj, "citedby_count")
|
|
64
|
+
citations = int(cited) if cited not in (None, "") else pd.NA
|
|
65
|
+
row = {
|
|
66
|
+
"scopus_id": scopus_id,
|
|
67
|
+
"doi": _get(obj, "doi"),
|
|
68
|
+
"title": _get(obj, "title"),
|
|
69
|
+
"abstract": _get(obj, "description"),
|
|
70
|
+
"publication": _get(obj, "publicationName"),
|
|
71
|
+
"date": date,
|
|
72
|
+
"year": year,
|
|
73
|
+
"citations": citations,
|
|
74
|
+
}
|
|
75
|
+
if "keywords" in include:
|
|
76
|
+
kw = _get(obj, "authkeywords")
|
|
77
|
+
row["authkeywords"] = "; ".join(kw) if kw else pd.NA
|
|
78
|
+
if "references" in include:
|
|
79
|
+
refs = _references_frame(_get(obj, "references"))
|
|
80
|
+
row["references"] = refs
|
|
81
|
+
refcount = _get(obj, "refcount")
|
|
82
|
+
try:
|
|
83
|
+
expected = int(refcount) if refcount not in (None, "") else None
|
|
84
|
+
except (TypeError, ValueError):
|
|
85
|
+
expected = None
|
|
86
|
+
if expected is not None and len(refs) != expected:
|
|
87
|
+
ident = _get(obj, "doi") or _get(obj, "eid") or "this document"
|
|
88
|
+
warnings.warn(
|
|
89
|
+
f"{len(refs)} reference(s) returned for {ident}, which reports "
|
|
90
|
+
f"refcount={expected}; the list may be an incomplete page "
|
|
91
|
+
"rather than the whole bibliography.",
|
|
92
|
+
stacklevel=2,
|
|
93
|
+
)
|
|
94
|
+
return row
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _safe_filename(ident: str) -> str:
|
|
98
|
+
"""A filesystem-safe cache key: non-alphanumerics become "_", keeping the
|
|
99
|
+
result human-decipherable (unlike a hash), since collisions between
|
|
100
|
+
distinct real identifiers are effectively impossible."""
|
|
101
|
+
return "".join(c if c.isalnum() else "_" for c in ident)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _abstract_cache_name(view: str, include: tuple[str, ...], ident: str) -> str:
|
|
105
|
+
"""The per-identifier checkpoint filename. Both ``view`` and ``include``
|
|
106
|
+
are part of the key, since both change what a row carries: a resumed run
|
|
107
|
+
with a different ``include`` must refetch rather than silently reuse a
|
|
108
|
+
leaner cached row."""
|
|
109
|
+
incl = "-".join(sorted(include)) if include else "plain"
|
|
110
|
+
return f"id-{view}-{incl}-{_safe_filename(ident)}.pkl"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _find_abstract_checkpoint(
|
|
114
|
+
cache: Path, view: str, include: tuple[str, ...], ident: str
|
|
115
|
+
) -> Path | None:
|
|
116
|
+
candidate = cache / _abstract_cache_name(view, include, ident)
|
|
117
|
+
return candidate if candidate.exists() else None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _write_abstract_checkpoint(
|
|
121
|
+
row: dict, cache: Path, view: str, include: tuple[str, ...], ident: str
|
|
122
|
+
) -> None:
|
|
123
|
+
# Pickled, not parquet/csv like fetch_plan()'s per-cell checkpoints: a row
|
|
124
|
+
# here can carry a nested DataFrame in its "references" entry, which
|
|
125
|
+
# parquet and csv cannot hold in a single cell but pickle handles
|
|
126
|
+
# directly, the same way the R package's per-identifier cache relies on
|
|
127
|
+
# RDS to hold an R list-column.
|
|
128
|
+
with open(cache / _abstract_cache_name(view, include, ident), "wb") as f:
|
|
129
|
+
pickle.dump(row, f)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def scopus_abstract(
|
|
133
|
+
ids,
|
|
134
|
+
by: str = "doi",
|
|
135
|
+
view: str = "META_ABS",
|
|
136
|
+
include: tuple[str, ...] = (),
|
|
137
|
+
cache_dir: str | None = None,
|
|
138
|
+
resume: bool = True,
|
|
139
|
+
**kwargs,
|
|
140
|
+
) -> pd.DataFrame:
|
|
141
|
+
"""Retrieve abstracts for one or many ids, resilient per id.
|
|
142
|
+
|
|
143
|
+
``by`` selects the lookup type ("doi", "eid" or "scopus_id"). Any id that
|
|
144
|
+
fails is warned about and yields an all-NA row that still records the id.
|
|
145
|
+
``view`` defaults to "META_ABS" (pybliometrics' own default, and the R
|
|
146
|
+
package's effective default), which carries the abstract text; the lighter
|
|
147
|
+
"META" omits it and leaves the ``abstract`` column empty. ``include``,
|
|
148
|
+
when unused, leaves the column set exactly as before.
|
|
149
|
+
|
|
150
|
+
``include`` names extra fields to retrieve in the same request:
|
|
151
|
+
"references" and/or "keywords". Both require Abstract Retrieval's "FULL"
|
|
152
|
+
or "REF" view (see ``view``), an entitlement separate from ordinary
|
|
153
|
+
abstract access and from Scopus Search access, and that, per Elsevier's
|
|
154
|
+
own documentation, some fields (notably author keywords) may need to be
|
|
155
|
+
requested from your Scopus/Elsevier account contact even when the view
|
|
156
|
+
itself is otherwise accessible. In development, against a live key with
|
|
157
|
+
full Abstract Retrieval access, "FULL" returned a complete, correctly
|
|
158
|
+
counted reference list for every document tried, while "REF" returned the
|
|
159
|
+
identical, complete list in one case but a truncated (paginated) subset in
|
|
160
|
+
another on an otherwise identical request made moments apart; "FULL" is
|
|
161
|
+
recommended when your entitlement allows it, and a mismatch between the
|
|
162
|
+
number of references returned and the document's own reported reference
|
|
163
|
+
count (``refcount``) is warned about, since the list may be an incomplete
|
|
164
|
+
page rather than the whole bibliography.
|
|
165
|
+
|
|
166
|
+
When "keywords" is included, an ``authkeywords`` column is added: the
|
|
167
|
+
document's author-supplied keywords, joined with "; ", or ``NA`` when the
|
|
168
|
+
document has none, or when the API omits the field for a given key's
|
|
169
|
+
entitlement. In this package's own development testing, against a live,
|
|
170
|
+
otherwise fully-entitled key, this field did not populate for documents
|
|
171
|
+
that do carry author keywords in Scopus itself, so an all-``NA`` column
|
|
172
|
+
is more likely an entitlement gap worth raising with your Scopus/Elsevier
|
|
173
|
+
account contact than genuinely absent data.
|
|
174
|
+
|
|
175
|
+
When "references" is included, a ``references`` column is added: one
|
|
176
|
+
DataFrame per document (one row per cited work), using pybliometrics' own
|
|
177
|
+
native field set (``position``, ``id``, ``doi``, ``title``, ``authors``,
|
|
178
|
+
``authors_auid``, ``authors_affiliationid``, ``sourcetitle``,
|
|
179
|
+
``publicationyear``, ``coverDate``, ``volume``, ``issue``, ``first``,
|
|
180
|
+
``last``, ``citedbycount``, ``type``, ``text``, ``fulltext``). This is a
|
|
181
|
+
richer shape than the R package's own ``references`` list-column, which
|
|
182
|
+
carries a leaner, hand-picked field set (``position``, ``id``, ``doi``,
|
|
183
|
+
``title``, ``authors``, ``source``, ``year``, ``citedbycount``); see the R
|
|
184
|
+
package's ``scopus_abstract()`` documentation for that shape. A document
|
|
185
|
+
with no resolvable references yields a zero-row DataFrame, not ``NA``, so
|
|
186
|
+
the column can always be unnested.
|
|
187
|
+
|
|
188
|
+
``cache_dir`` and ``resume`` checkpoint per identifier, the way
|
|
189
|
+
:func:`scopusflow.fetch.fetch_plan` checkpoints per cell, pickled rather
|
|
190
|
+
than written as parquet/csv, since a row here can carry a nested
|
|
191
|
+
DataFrame. Worth setting whenever ``include`` is used: Abstract Retrieval
|
|
192
|
+
draws on its own weekly quota, smaller than and separate from Search's,
|
|
193
|
+
and every identifier costs its own request, so re-running an interrupted
|
|
194
|
+
batch without a cache re-spends quota already spent. Relying on
|
|
195
|
+
pybliometrics' own on-disk response cache (its ``refresh`` parameter,
|
|
196
|
+
keyed by identifier and view under its configured cache directory) is
|
|
197
|
+
enough to avoid repeat network calls for the *same* identifier across
|
|
198
|
+
script runs; this checkpoint is for batch-level progress and resumability
|
|
199
|
+
across *many* identifiers, a separate concern.
|
|
200
|
+
|
|
201
|
+
The number of Abstract Retrieval requests made, and the most recently
|
|
202
|
+
parsed remaining-quota figure (from pybliometrics'
|
|
203
|
+
``get_key_remaining_quota()``), are attached as ``result.attrs["n_requests"]``
|
|
204
|
+
and ``result.attrs["quota"]``, since this is a materially more expensive
|
|
205
|
+
operation than a search call.
|
|
206
|
+
|
|
207
|
+
A 403 (an entitlement gate, most often on the requested view or field)
|
|
208
|
+
raises :class:`scopusflow.exceptions.ScopusFlowForbiddenError` and stops
|
|
209
|
+
the batch immediately, naming the view and identifier, rather than
|
|
210
|
+
repeating the identical failure for every remaining identifier:
|
|
211
|
+
entitlement is an account-level property, not a per-document one, so it
|
|
212
|
+
will not succeed on retry.
|
|
213
|
+
"""
|
|
214
|
+
if by not in _ID_TYPES:
|
|
215
|
+
raise ValueError("by must be one of 'doi', 'eid', 'scopus_id'.")
|
|
216
|
+
include = tuple(include) if include else ()
|
|
217
|
+
if not set(include) <= _KNOWN_INCLUDE:
|
|
218
|
+
raise ValueError("include must be made up of 'references' and/or 'keywords'.")
|
|
219
|
+
if "references" in include and view not in _VIEWS_WITH_REFERENCES:
|
|
220
|
+
raise ValueError('include="references" needs view="FULL" or view="REF".')
|
|
221
|
+
id_type = _ID_TYPES[by]
|
|
222
|
+
id_column = "doi" if by == "doi" else "scopus_id"
|
|
223
|
+
|
|
224
|
+
if isinstance(ids, str):
|
|
225
|
+
ids = [ids]
|
|
226
|
+
|
|
227
|
+
columns = list(ABSTRACT_COLUMNS)
|
|
228
|
+
if "keywords" in include:
|
|
229
|
+
columns = [*columns, "authkeywords"]
|
|
230
|
+
if "references" in include:
|
|
231
|
+
columns = [*columns, "references"]
|
|
232
|
+
|
|
233
|
+
cache = Path(cache_dir) if cache_dir else None
|
|
234
|
+
if cache is not None:
|
|
235
|
+
cache.mkdir(parents=True, exist_ok=True)
|
|
236
|
+
|
|
237
|
+
# Resolve the dependency once: a missing pybliometrics is a setup error and
|
|
238
|
+
# must surface clearly, not masquerade as every id failing to retrieve.
|
|
239
|
+
from pybliometrics.scopus import AbstractRetrieval # lazy; needs a key
|
|
240
|
+
try:
|
|
241
|
+
# A separate, defensive import: pybliometrics.exception is an internal
|
|
242
|
+
# module, not part of pybliometrics.scopus's own public surface, so a
|
|
243
|
+
# minimal test double or an unexpected future reorganisation should
|
|
244
|
+
# degrade to generic exception handling below rather than breaking
|
|
245
|
+
# every call to this function.
|
|
246
|
+
from pybliometrics.exception import Scopus403Error
|
|
247
|
+
except ImportError:
|
|
248
|
+
class Scopus403Error(Exception): # never actually raised; see above
|
|
249
|
+
pass
|
|
250
|
+
|
|
251
|
+
n_requests = 0
|
|
252
|
+
quota = None
|
|
253
|
+
rows = []
|
|
254
|
+
for i, ident in enumerate(ids, start=1):
|
|
255
|
+
checkpoint = (
|
|
256
|
+
_find_abstract_checkpoint(cache, view, include, ident)
|
|
257
|
+
if cache is not None else None
|
|
258
|
+
)
|
|
259
|
+
if checkpoint is not None and resume:
|
|
260
|
+
logger.info("%d/%d: %s loaded from cache.", i, len(ids), ident)
|
|
261
|
+
with open(checkpoint, "rb") as f:
|
|
262
|
+
rows.append(pickle.load(f))
|
|
263
|
+
continue
|
|
264
|
+
|
|
265
|
+
logger.info("Retrieving %d/%d: %s", i, len(ids), ident)
|
|
266
|
+
try:
|
|
267
|
+
ab = AbstractRetrieval(ident, id_type=id_type, view=view, **kwargs)
|
|
268
|
+
n_requests += 1
|
|
269
|
+
# Optional: an object without these methods (a plain dict, or a
|
|
270
|
+
# minimal stand-in) simply reports no quota, rather than failing
|
|
271
|
+
# the whole retrieval over a metadata nicety.
|
|
272
|
+
get_quota = getattr(ab, "get_key_remaining_quota", None)
|
|
273
|
+
remaining = get_quota() if callable(get_quota) else None
|
|
274
|
+
if remaining is not None:
|
|
275
|
+
quota = {"remaining": remaining, "reset": ab.get_key_reset_time()}
|
|
276
|
+
row = _abstract_row(ab, include=include)
|
|
277
|
+
except Scopus403Error as exc:
|
|
278
|
+
n_requests += 1
|
|
279
|
+
remaining_ids = len(ids) - i
|
|
280
|
+
# The FULL/REF alternative is only sensible advice when the failed
|
|
281
|
+
# view was one of the two reference-carrying views; a 403 on a
|
|
282
|
+
# plain META/META_ABS retrieval is an ordinary entitlement gap.
|
|
283
|
+
alternative = (
|
|
284
|
+
' or, if you have not already, try the other of "FULL"/"REF"'
|
|
285
|
+
if view in _VIEWS_WITH_REFERENCES else ""
|
|
286
|
+
)
|
|
287
|
+
raise ScopusFlowForbiddenError(
|
|
288
|
+
f'Abstract Retrieval refused view="{view}" (HTTP 403) for {ident!r}. '
|
|
289
|
+
"This usually means your Scopus API key's entitlement does not cover "
|
|
290
|
+
"the requested view or field; contact your Scopus/Elsevier account "
|
|
291
|
+
f"holder or institutional administrator to request access{alternative}. "
|
|
292
|
+
"Stopping rather "
|
|
293
|
+
f"than repeating the same failure for the remaining {remaining_ids} "
|
|
294
|
+
"identifier(s) (this entitlement is an account-level property, not a "
|
|
295
|
+
"per-document one, so it will not succeed on retry)."
|
|
296
|
+
) from exc
|
|
297
|
+
except Exception: # one bad id must not sink the batch
|
|
298
|
+
n_requests += 1
|
|
299
|
+
warnings.warn(
|
|
300
|
+
f"Could not retrieve abstract for {ident!r}; recording NA row.",
|
|
301
|
+
stacklevel=2,
|
|
302
|
+
)
|
|
303
|
+
row = {col: pd.NA for col in columns}
|
|
304
|
+
row[id_column] = ident
|
|
305
|
+
if "references" in include:
|
|
306
|
+
row["references"] = _references_frame(None)
|
|
307
|
+
|
|
308
|
+
if cache is not None:
|
|
309
|
+
_write_abstract_checkpoint(row, cache, view, include, ident)
|
|
310
|
+
rows.append(row)
|
|
311
|
+
|
|
312
|
+
out = pd.DataFrame(rows, columns=columns)
|
|
313
|
+
out.attrs["n_requests"] = n_requests
|
|
314
|
+
out.attrs["quota"] = quota
|
|
315
|
+
return out
|