scopusflow 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scopusflow/__init__.py ADDED
@@ -0,0 +1,51 @@
1
+ """scopusflow: a reproducible workflow layer over pybliometrics.
2
+
3
+ pybliometrics provides the retrieval primitives for the Scopus API; scopusflow
4
+ adds the workflow on top: reproducible search plans, a single stable record
5
+ schema, resumable checkpointed harvesting, and DOI change-tracking.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from .abstract import scopus_abstract
11
+ from .compare import compare_topics
12
+ from .corpus import corpus
13
+ from .count import scopus_count
14
+ from .diff import diff_dois, extract_dois
15
+ from .exceptions import ScopusFlowForbiddenError
16
+ from .export import to_bibtex, to_ris
17
+ from .fetch import fetch_plan
18
+ from .plan import PlanCell, SearchPlan
19
+ from .plots import plot_comparison, plot_top, plot_trend
20
+ from .query import FIELD_TAGS, scopus_query, wrap_field
21
+ from .records import RECORD_COLUMNS, to_records, top
22
+ from .trend import scopus_trend, year_counts
23
+
24
+ __version__ = "0.1.0"
25
+
26
+ __all__ = [
27
+ "SearchPlan",
28
+ "PlanCell",
29
+ "scopus_query",
30
+ "wrap_field",
31
+ "FIELD_TAGS",
32
+ "to_records",
33
+ "top",
34
+ "RECORD_COLUMNS",
35
+ "fetch_plan",
36
+ "extract_dois",
37
+ "diff_dois",
38
+ "year_counts",
39
+ "scopus_trend",
40
+ "scopus_count",
41
+ "compare_topics",
42
+ "scopus_abstract",
43
+ "ScopusFlowForbiddenError",
44
+ "corpus",
45
+ "to_bibtex",
46
+ "to_ris",
47
+ "plot_trend",
48
+ "plot_top",
49
+ "plot_comparison",
50
+ "__version__",
51
+ ]
scopusflow/abstract.py ADDED
@@ -0,0 +1,315 @@
1
+ """Batch abstract retrieval, resilient per id, mirroring ``scopus_abstract``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import pickle
7
+ import warnings
8
+ from pathlib import Path
9
+
10
+ import pandas as pd
11
+
12
+ from .exceptions import ScopusFlowForbiddenError
13
+
14
+ #: Per-identifier progress is emitted on this logger; see fetch.py.
15
+ logger = logging.getLogger("scopusflow")
16
+
17
+ #: The stable column schema for retrieved abstracts.
18
+ ABSTRACT_COLUMNS = [
19
+ "scopus_id", "doi", "title", "abstract",
20
+ "publication", "date", "year", "citations",
21
+ ]
22
+
23
+ #: Map the user-facing ``by`` argument to a pybliometrics ``id_type``.
24
+ _ID_TYPES = {"doi": "doi", "eid": "eid", "scopus_id": "scopus_id"}
25
+
26
+ #: Values accepted by ``include``.
27
+ _KNOWN_INCLUDE = {"references", "keywords"}
28
+
29
+ #: Abstract Retrieval views that carry a bibliography.
30
+ _VIEWS_WITH_REFERENCES = {"FULL", "REF"}
31
+
32
+
33
+ def _get(obj, name):
34
+ if isinstance(obj, dict):
35
+ return obj.get(name)
36
+ return getattr(obj, name, None)
37
+
38
+
39
+ def _references_frame(refs) -> pd.DataFrame:
40
+ """Normalise pybliometrics' own ``Reference`` namedtuples into a DataFrame,
41
+ one row per cited work, keeping pybliometrics' full native field set (a
42
+ richer shape than the R package's leaner equivalent; see
43
+ :func:`scopus_abstract`'s docstring for the field-by-field difference)."""
44
+ from pybliometrics.scopus import Reference
45
+
46
+ if not refs:
47
+ return pd.DataFrame(columns=list(Reference._fields))
48
+ return pd.DataFrame([r._asdict() for r in refs])
49
+
50
+
51
+ def _abstract_row(obj, include: tuple[str, ...] = ()) -> dict:
52
+ """Normalise a single ``AbstractRetrieval``-like object (or dict) into the
53
+ stable :data:`ABSTRACT_COLUMNS` mapping; offline, no network. When
54
+ "references" is included, a mismatch between the number of references
55
+ returned and the document's own reported ``refcount`` is warned about,
56
+ since the list may be an incomplete page rather than the whole
57
+ bibliography."""
58
+ eid = _get(obj, "eid")
59
+ scopus_id = str(eid).split("2-s2.0-")[-1] if eid else pd.NA
60
+ date = _get(obj, "coverDate")
61
+ head = str(date)[:4] if date else ""
62
+ year = int(head) if head.isdigit() else pd.NA
63
+ cited = _get(obj, "citedby_count")
64
+ citations = int(cited) if cited not in (None, "") else pd.NA
65
+ row = {
66
+ "scopus_id": scopus_id,
67
+ "doi": _get(obj, "doi"),
68
+ "title": _get(obj, "title"),
69
+ "abstract": _get(obj, "description"),
70
+ "publication": _get(obj, "publicationName"),
71
+ "date": date,
72
+ "year": year,
73
+ "citations": citations,
74
+ }
75
+ if "keywords" in include:
76
+ kw = _get(obj, "authkeywords")
77
+ row["authkeywords"] = "; ".join(kw) if kw else pd.NA
78
+ if "references" in include:
79
+ refs = _references_frame(_get(obj, "references"))
80
+ row["references"] = refs
81
+ refcount = _get(obj, "refcount")
82
+ try:
83
+ expected = int(refcount) if refcount not in (None, "") else None
84
+ except (TypeError, ValueError):
85
+ expected = None
86
+ if expected is not None and len(refs) != expected:
87
+ ident = _get(obj, "doi") or _get(obj, "eid") or "this document"
88
+ warnings.warn(
89
+ f"{len(refs)} reference(s) returned for {ident}, which reports "
90
+ f"refcount={expected}; the list may be an incomplete page "
91
+ "rather than the whole bibliography.",
92
+ stacklevel=2,
93
+ )
94
+ return row
95
+
96
+
97
+ def _safe_filename(ident: str) -> str:
98
+ """A filesystem-safe cache key: non-alphanumerics become "_", keeping the
99
+ result human-decipherable (unlike a hash), since collisions between
100
+ distinct real identifiers are effectively impossible."""
101
+ return "".join(c if c.isalnum() else "_" for c in ident)
102
+
103
+
104
+ def _abstract_cache_name(view: str, include: tuple[str, ...], ident: str) -> str:
105
+ """The per-identifier checkpoint filename. Both ``view`` and ``include``
106
+ are part of the key, since both change what a row carries: a resumed run
107
+ with a different ``include`` must refetch rather than silently reuse a
108
+ leaner cached row."""
109
+ incl = "-".join(sorted(include)) if include else "plain"
110
+ return f"id-{view}-{incl}-{_safe_filename(ident)}.pkl"
111
+
112
+
113
+ def _find_abstract_checkpoint(
114
+ cache: Path, view: str, include: tuple[str, ...], ident: str
115
+ ) -> Path | None:
116
+ candidate = cache / _abstract_cache_name(view, include, ident)
117
+ return candidate if candidate.exists() else None
118
+
119
+
120
+ def _write_abstract_checkpoint(
121
+ row: dict, cache: Path, view: str, include: tuple[str, ...], ident: str
122
+ ) -> None:
123
+ # Pickled, not parquet/csv like fetch_plan()'s per-cell checkpoints: a row
124
+ # here can carry a nested DataFrame in its "references" entry, which
125
+ # parquet and csv cannot hold in a single cell but pickle handles
126
+ # directly, the same way the R package's per-identifier cache relies on
127
+ # RDS to hold an R list-column.
128
+ with open(cache / _abstract_cache_name(view, include, ident), "wb") as f:
129
+ pickle.dump(row, f)
130
+
131
+
132
+ def scopus_abstract(
133
+ ids,
134
+ by: str = "doi",
135
+ view: str = "META_ABS",
136
+ include: tuple[str, ...] = (),
137
+ cache_dir: str | None = None,
138
+ resume: bool = True,
139
+ **kwargs,
140
+ ) -> pd.DataFrame:
141
+ """Retrieve abstracts for one or many ids, resilient per id.
142
+
143
+ ``by`` selects the lookup type ("doi", "eid" or "scopus_id"). Any id that
144
+ fails is warned about and yields an all-NA row that still records the id.
145
+ ``view`` defaults to "META_ABS" (pybliometrics' own default, and the R
146
+ package's effective default), which carries the abstract text; the lighter
147
+ "META" omits it and leaves the ``abstract`` column empty. ``include``,
148
+ when unused, leaves the column set exactly as before.
149
+
150
+ ``include`` names extra fields to retrieve in the same request:
151
+ "references" and/or "keywords". Both require Abstract Retrieval's "FULL"
152
+ or "REF" view (see ``view``), an entitlement separate from ordinary
153
+ abstract access and from Scopus Search access, and that, per Elsevier's
154
+ own documentation, some fields (notably author keywords) may need to be
155
+ requested from your Scopus/Elsevier account contact even when the view
156
+ itself is otherwise accessible. In development, against a live key with
157
+ full Abstract Retrieval access, "FULL" returned a complete, correctly
158
+ counted reference list for every document tried, while "REF" returned the
159
+ identical, complete list in one case but a truncated (paginated) subset in
160
+ another on an otherwise identical request made moments apart; "FULL" is
161
+ recommended when your entitlement allows it, and a mismatch between the
162
+ number of references returned and the document's own reported reference
163
+ count (``refcount``) is warned about, since the list may be an incomplete
164
+ page rather than the whole bibliography.
165
+
166
+ When "keywords" is included, an ``authkeywords`` column is added: the
167
+ document's author-supplied keywords, joined with "; ", or ``NA`` when the
168
+ document has none, or when the API omits the field for a given key's
169
+ entitlement. In this package's own development testing, against a live,
170
+ otherwise fully-entitled key, this field did not populate for documents
171
+ that do carry author keywords in Scopus itself, so an all-``NA`` column
172
+ is more likely an entitlement gap worth raising with your Scopus/Elsevier
173
+ account contact than genuinely absent data.
174
+
175
+ When "references" is included, a ``references`` column is added: one
176
+ DataFrame per document (one row per cited work), using pybliometrics' own
177
+ native field set (``position``, ``id``, ``doi``, ``title``, ``authors``,
178
+ ``authors_auid``, ``authors_affiliationid``, ``sourcetitle``,
179
+ ``publicationyear``, ``coverDate``, ``volume``, ``issue``, ``first``,
180
+ ``last``, ``citedbycount``, ``type``, ``text``, ``fulltext``). This is a
181
+ richer shape than the R package's own ``references`` list-column, which
182
+ carries a leaner, hand-picked field set (``position``, ``id``, ``doi``,
183
+ ``title``, ``authors``, ``source``, ``year``, ``citedbycount``); see the R
184
+ package's ``scopus_abstract()`` documentation for that shape. A document
185
+ with no resolvable references yields a zero-row DataFrame, not ``NA``, so
186
+ the column can always be unnested.
187
+
188
+ ``cache_dir`` and ``resume`` checkpoint per identifier, the way
189
+ :func:`scopusflow.fetch.fetch_plan` checkpoints per cell, pickled rather
190
+ than written as parquet/csv, since a row here can carry a nested
191
+ DataFrame. Worth setting whenever ``include`` is used: Abstract Retrieval
192
+ draws on its own weekly quota, smaller than and separate from Search's,
193
+ and every identifier costs its own request, so re-running an interrupted
194
+ batch without a cache re-spends quota already spent. Relying on
195
+ pybliometrics' own on-disk response cache (its ``refresh`` parameter,
196
+ keyed by identifier and view under its configured cache directory) is
197
+ enough to avoid repeat network calls for the *same* identifier across
198
+ script runs; this checkpoint is for batch-level progress and resumability
199
+ across *many* identifiers, a separate concern.
200
+
201
+ The number of Abstract Retrieval requests made, and the most recently
202
+ parsed remaining-quota figure (from pybliometrics'
203
+ ``get_key_remaining_quota()``), are attached as ``result.attrs["n_requests"]``
204
+ and ``result.attrs["quota"]``, since this is a materially more expensive
205
+ operation than a search call.
206
+
207
+ A 403 (an entitlement gate, most often on the requested view or field)
208
+ raises :class:`scopusflow.exceptions.ScopusFlowForbiddenError` and stops
209
+ the batch immediately, naming the view and identifier, rather than
210
+ repeating the identical failure for every remaining identifier:
211
+ entitlement is an account-level property, not a per-document one, so it
212
+ will not succeed on retry.
213
+ """
214
+ if by not in _ID_TYPES:
215
+ raise ValueError("by must be one of 'doi', 'eid', 'scopus_id'.")
216
+ include = tuple(include) if include else ()
217
+ if not set(include) <= _KNOWN_INCLUDE:
218
+ raise ValueError("include must be made up of 'references' and/or 'keywords'.")
219
+ if "references" in include and view not in _VIEWS_WITH_REFERENCES:
220
+ raise ValueError('include="references" needs view="FULL" or view="REF".')
221
+ id_type = _ID_TYPES[by]
222
+ id_column = "doi" if by == "doi" else "scopus_id"
223
+
224
+ if isinstance(ids, str):
225
+ ids = [ids]
226
+
227
+ columns = list(ABSTRACT_COLUMNS)
228
+ if "keywords" in include:
229
+ columns = [*columns, "authkeywords"]
230
+ if "references" in include:
231
+ columns = [*columns, "references"]
232
+
233
+ cache = Path(cache_dir) if cache_dir else None
234
+ if cache is not None:
235
+ cache.mkdir(parents=True, exist_ok=True)
236
+
237
+ # Resolve the dependency once: a missing pybliometrics is a setup error and
238
+ # must surface clearly, not masquerade as every id failing to retrieve.
239
+ from pybliometrics.scopus import AbstractRetrieval # lazy; needs a key
240
+ try:
241
+ # A separate, defensive import: pybliometrics.exception is an internal
242
+ # module, not part of pybliometrics.scopus's own public surface, so a
243
+ # minimal test double or an unexpected future reorganisation should
244
+ # degrade to generic exception handling below rather than breaking
245
+ # every call to this function.
246
+ from pybliometrics.exception import Scopus403Error
247
+ except ImportError:
248
+ class Scopus403Error(Exception): # never actually raised; see above
249
+ pass
250
+
251
+ n_requests = 0
252
+ quota = None
253
+ rows = []
254
+ for i, ident in enumerate(ids, start=1):
255
+ checkpoint = (
256
+ _find_abstract_checkpoint(cache, view, include, ident)
257
+ if cache is not None else None
258
+ )
259
+ if checkpoint is not None and resume:
260
+ logger.info("%d/%d: %s loaded from cache.", i, len(ids), ident)
261
+ with open(checkpoint, "rb") as f:
262
+ rows.append(pickle.load(f))
263
+ continue
264
+
265
+ logger.info("Retrieving %d/%d: %s", i, len(ids), ident)
266
+ try:
267
+ ab = AbstractRetrieval(ident, id_type=id_type, view=view, **kwargs)
268
+ n_requests += 1
269
+ # Optional: an object without these methods (a plain dict, or a
270
+ # minimal stand-in) simply reports no quota, rather than failing
271
+ # the whole retrieval over a metadata nicety.
272
+ get_quota = getattr(ab, "get_key_remaining_quota", None)
273
+ remaining = get_quota() if callable(get_quota) else None
274
+ if remaining is not None:
275
+ quota = {"remaining": remaining, "reset": ab.get_key_reset_time()}
276
+ row = _abstract_row(ab, include=include)
277
+ except Scopus403Error as exc:
278
+ n_requests += 1
279
+ remaining_ids = len(ids) - i
280
+ # The FULL/REF alternative is only sensible advice when the failed
281
+ # view was one of the two reference-carrying views; a 403 on a
282
+ # plain META/META_ABS retrieval is an ordinary entitlement gap.
283
+ alternative = (
284
+ ' or, if you have not already, try the other of "FULL"/"REF"'
285
+ if view in _VIEWS_WITH_REFERENCES else ""
286
+ )
287
+ raise ScopusFlowForbiddenError(
288
+ f'Abstract Retrieval refused view="{view}" (HTTP 403) for {ident!r}. '
289
+ "This usually means your Scopus API key's entitlement does not cover "
290
+ "the requested view or field; contact your Scopus/Elsevier account "
291
+ f"holder or institutional administrator to request access{alternative}. "
292
+ "Stopping rather "
293
+ f"than repeating the same failure for the remaining {remaining_ids} "
294
+ "identifier(s) (this entitlement is an account-level property, not a "
295
+ "per-document one, so it will not succeed on retry)."
296
+ ) from exc
297
+ except Exception: # one bad id must not sink the batch
298
+ n_requests += 1
299
+ warnings.warn(
300
+ f"Could not retrieve abstract for {ident!r}; recording NA row.",
301
+ stacklevel=2,
302
+ )
303
+ row = {col: pd.NA for col in columns}
304
+ row[id_column] = ident
305
+ if "references" in include:
306
+ row["references"] = _references_frame(None)
307
+
308
+ if cache is not None:
309
+ _write_abstract_checkpoint(row, cache, view, include, ident)
310
+ rows.append(row)
311
+
312
+ out = pd.DataFrame(rows, columns=columns)
313
+ out.attrs["n_requests"] = n_requests
314
+ out.attrs["quota"] = quota
315
+ return out