unitoolkit 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ladok_vakt/__init__.py +7 -0
- ladok_vakt/__main__.py +51 -0
- ladok_vakt/core.py +553 -0
- ladok_vakt/login.py +232 -0
- tools.json +623 -0
- unitoolkit/__init__.py +1 -0
- unitoolkit/__main__.py +63 -0
- unitoolkit/agent.py +270 -0
- unitoolkit/browser.py +147 -0
- unitoolkit/canvas.py +189 -0
- unitoolkit/chainproxy.py +113 -0
- unitoolkit/citation.py +165 -0
- unitoolkit/databases.py +497 -0
- unitoolkit/docs.py +394 -0
- unitoolkit/fulltext.py +260 -0
- unitoolkit/koha.py +729 -0
- unitoolkit/mcp.py +85 -0
- unitoolkit/programme.py +147 -0
- unitoolkit/search.py +507 -0
- unitoolkit/serve.py +153 -0
- unitoolkit/sharepoint.py +86 -0
- unitoolkit/surface.py +86 -0
- unitoolkit/timeedit.py +209 -0
- unitoolkit/tools.py +555 -0
- unitoolkit-0.7.0.dist-info/METADATA +177 -0
- unitoolkit-0.7.0.dist-info/RECORD +30 -0
- unitoolkit-0.7.0.dist-info/WHEEL +5 -0
- unitoolkit-0.7.0.dist-info/entry_points.txt +4 -0
- unitoolkit-0.7.0.dist-info/licenses/LICENSE +21 -0
- unitoolkit-0.7.0.dist-info/top_level.txt +2 -0
unitoolkit/databases.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
"""Database access for SHH, through the library's OCLC EZproxy.
|
|
2
|
+
|
|
3
|
+
Every subscription database Sophiahemmet offers sits behind `shh.idm.oclc.org`, which hands
|
|
4
|
+
the visitor to the school's Microsoft Entra tenant. Once one sign-in has happened with
|
|
5
|
+
"stay signed in", the browser profile keeps an Entra session and EZproxy accepts it silently
|
|
6
|
+
afterwards, so a search needs no credentials at all. When the session has expired EZproxy
|
|
7
|
+
shows the Entra sign-in page, and every function here raises instead of pretending the
|
|
8
|
+
result set was empty.
|
|
9
|
+
|
|
10
|
+
What is read-only: searches and record pages. Nothing signs in, renews, or writes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
|
|
20
|
+
CDP = os.environ.get('UNITOOLKIT_CDP_SCHOOL', 'http://127.0.0.1:9233')
|
|
21
|
+
EZ = 'https://shh.idm.oclc.org/login?url='
|
|
22
|
+
EBSCO = 'https://research-ebsco-com.shh.idm.oclc.org'
|
|
23
|
+
EBSCO_COLLECTION = 'm7tdf3'
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DatabaseError(RuntimeError):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Databases the school subscribes to, with the EZproxy target each one needs.
|
|
31
|
+
# `ebsco` entries are searched through the EBSCOhost collection; the rest open their own site.
|
|
32
|
+
DATABASES = {
|
|
33
|
+
'cinahl': {'name': 'CINAHL', 'ebsco': 'cin20',
|
|
34
|
+
'url': EBSCO + '/c/%s/search/results?db=cin20&q=' % EBSCO_COLLECTION},
|
|
35
|
+
'psycinfo': {'name': 'PsycInfo', 'ebsco': 'psyh',
|
|
36
|
+
'url': EBSCO + '/c/%s/search/results?db=psyh&q=' % EBSCO_COLLECTION},
|
|
37
|
+
'cochrane': {'name': 'Cochrane Library', 'reader': 'cochrane',
|
|
38
|
+
'url': 'https://www.cochranelibrary.com/cdsr/reviews'},
|
|
39
|
+
'wos': {'name': 'Web of Science', 'reader': 'wos', 'url': 'https://webofknowledge.com/wos'},
|
|
40
|
+
'jcr': {'name': 'Journal Citation Reports', 'url': 'https://jcr.clarivate.com/jcr/home'},
|
|
41
|
+
'sage': {'name': 'Sage Research Methods', 'url': 'https://methods.sagepub.com/'},
|
|
42
|
+
'nese': {'name': 'Nationalencyklopedin', 'url': 'https://www.ne.se/'},
|
|
43
|
+
'ffhb': {'name': 'Författningshandbok (Liber)',
|
|
44
|
+
'url': 'https://online.liber.se/Home/LinkAccess/?acid=qQdnnEJcDpnaiECJYMw0BQ%3d%3d&'},
|
|
45
|
+
'pubmed': {'name': 'PubMed (EZproxy, for full-text links)',
|
|
46
|
+
'url': 'http://www.ncbi.nlm.nih.gov/pubmed/?otool=isesushlib'},
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
FREE = {
|
|
50
|
+
'pubmed_api': 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils',
|
|
51
|
+
'libris': 'https://libris.kb.se',
|
|
52
|
+
'diva': 'http://shh.diva-portal.org/smash/search.jsf',
|
|
53
|
+
'doaj': 'https://www.doaj.org/',
|
|
54
|
+
'eric': 'https://eric.ed.gov/',
|
|
55
|
+
'swepub': 'http://swepub.kb.se/form_extended.jsp',
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
# A result card on the EBSCOhost results page.
|
|
59
|
+
CARDS_JS = r"""(() => {
|
|
60
|
+
const cards = [...document.querySelectorAll('div[class*="search-result-item_search-result-item__"]')]
|
|
61
|
+
.filter(c => c.querySelector('a[class*="result-item-title_result-item-title"]'));
|
|
62
|
+
const total = (document.body.innerText.match(/([\d\s,.]+)\s+total results/i) || [])[1] || '';
|
|
63
|
+
return JSON.stringify({
|
|
64
|
+
total: total.trim(),
|
|
65
|
+
authenticated: /SOPHIAHEMMET/i.test(document.body.innerText),
|
|
66
|
+
cards: cards.map(c => {
|
|
67
|
+
const a = c.querySelector('a[class*="result-item-title_result-item-title"]');
|
|
68
|
+
const lines = (c.innerText || '').split('\n').map(s => s.trim()).filter(Boolean);
|
|
69
|
+
return {
|
|
70
|
+
title: (a ? a.innerText : '').trim(),
|
|
71
|
+
href: a ? a.getAttribute('href') : '',
|
|
72
|
+
lines: lines,
|
|
73
|
+
peer_reviewed: lines.includes('Peer reviewed') || /Peer reviewed/i.test(c.innerText),
|
|
74
|
+
full_text: /Full Text Finder|PDF Full Text|HTML Full Text/i.test(c.innerText),
|
|
75
|
+
};
|
|
76
|
+
})
|
|
77
|
+
});
|
|
78
|
+
})()"""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
DETAIL_JS = r"""(() => {
|
|
82
|
+
const t = (document.body.innerText || '');
|
|
83
|
+
const flat = t.replace(/\s+/g, ' ');
|
|
84
|
+
const blocks = [];
|
|
85
|
+
const abs = t.match(/Abstract\s*\n([^\n][\s\S]{80,3000})/);
|
|
86
|
+
if (abs) blocks.push(abs[1].split(/(?:Subjects|DOI|PMID|Author Affiliations)/)[0].trim());
|
|
87
|
+
const pick = (re) => (flat.match(re) || [])[1] || '';
|
|
88
|
+
return JSON.stringify({
|
|
89
|
+
title: (document.querySelector('h1') || {}).innerText || '',
|
|
90
|
+
abstract: (blocks[0] || '').slice(0, 4000),
|
|
91
|
+
doi: pick(/DOI\s*:?\s*(10\.[^\s,;]+)/i),
|
|
92
|
+
pmid: pick(/PMID\s*:?\s*(\d+)/),
|
|
93
|
+
authors: pick(/Authors\s+(.{0,400}?)(?:Source|Abstract|Journal|Publication)/),
|
|
94
|
+
source_line: pick(/Source\s+(.{0,200}?)(?:Abstract|Subjects|DOI)/),
|
|
95
|
+
authenticated: /SOPHIAHEMMET/i.test(t),
|
|
96
|
+
login_wall: /Sign in to your account|Logga in med/i.test(t),
|
|
97
|
+
});
|
|
98
|
+
})()"""
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class _Tab:
|
|
102
|
+
"""One CDP page target, kept for the length of a call."""
|
|
103
|
+
|
|
104
|
+
def __init__(self, url=None):
|
|
105
|
+
self.url = url
|
|
106
|
+
|
|
107
|
+
def __enter__(self):
|
|
108
|
+
self.loop = asyncio.new_event_loop()
|
|
109
|
+
self._run(self._open())
|
|
110
|
+
return self
|
|
111
|
+
|
|
112
|
+
def __exit__(self, *exc):
|
|
113
|
+
try:
|
|
114
|
+
try:
|
|
115
|
+
self.loop.run_until_complete(self.call('Target.closeTarget', {'targetId': self.target}))
|
|
116
|
+
except Exception:
|
|
117
|
+
pass
|
|
118
|
+
self.loop.run_until_complete(self.ws.close())
|
|
119
|
+
finally:
|
|
120
|
+
self.loop.close()
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
def _run(self, coro):
|
|
124
|
+
return self.loop.run_until_complete(coro)
|
|
125
|
+
|
|
126
|
+
async def _open(self):
|
|
127
|
+
import websockets
|
|
128
|
+
try:
|
|
129
|
+
ver = json.load(urllib.request.urlopen(CDP + '/json/version', timeout=5))
|
|
130
|
+
except Exception:
|
|
131
|
+
raise DatabaseError(
|
|
132
|
+
'the school browser is not running on %s; start it with agent_chrome.sh' % CDP)
|
|
133
|
+
self.ws = await websockets.connect(ver['webSocketDebuggerUrl'], max_size=None)
|
|
134
|
+
self.n = 0
|
|
135
|
+
self.target = (await self.call('Target.createTarget', {'url': self.url or 'about:blank'}))['targetId']
|
|
136
|
+
self.sid = (await self.call('Target.attachToTarget',
|
|
137
|
+
{'targetId': self.target, 'flatten': True}))['sessionId']
|
|
138
|
+
await self.call('Runtime.enable', {}, self.sid)
|
|
139
|
+
|
|
140
|
+
async def call(self, method, params=None, sid=None):
|
|
141
|
+
self.n += 1
|
|
142
|
+
msg = {'id': self.n, 'method': method, 'params': params or {}}
|
|
143
|
+
if sid:
|
|
144
|
+
msg['sessionId'] = sid
|
|
145
|
+
await self.ws.send(json.dumps(msg))
|
|
146
|
+
while True:
|
|
147
|
+
r = json.loads(await self.ws.recv())
|
|
148
|
+
if r.get('id') == self.n:
|
|
149
|
+
if 'error' in r:
|
|
150
|
+
raise DatabaseError('%s: %s' % (method, r['error'].get('message')))
|
|
151
|
+
return r.get('result', r)
|
|
152
|
+
|
|
153
|
+
def goto(self, url, settle=3.0):
|
|
154
|
+
self._run(self.call('Page.navigate', {'url': url}, self.sid))
|
|
155
|
+
self._run(asyncio.sleep(settle))
|
|
156
|
+
|
|
157
|
+
def eval(self, expr):
|
|
158
|
+
r = self._run(self.call('Runtime.evaluate',
|
|
159
|
+
{'expression': expr, 'returnByValue': True, 'awaitPromise': True},
|
|
160
|
+
self.sid))
|
|
161
|
+
if r.get('exceptionDetails'):
|
|
162
|
+
raise DatabaseError('page script failed: %s' % r['exceptionDetails'].get('text'))
|
|
163
|
+
return (r.get('result') or {}).get('value')
|
|
164
|
+
|
|
165
|
+
def text(self):
|
|
166
|
+
return (self.eval('document.body.innerText') or '')
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _check_session(page):
|
|
170
|
+
"""EZproxy answers with the Entra sign-in page when the school session has gone."""
|
|
171
|
+
url = page.eval('location.href') or ''
|
|
172
|
+
body = page.text()
|
|
173
|
+
if 'login.microsoftonline.com' in url or 'Sign in to your account' in body:
|
|
174
|
+
raise DatabaseError(
|
|
175
|
+
'the school sign-in has expired: open %s in the school browser and sign in once '
|
|
176
|
+
'(Microsoft Entra, with MFA). After that EZproxy accepts the session silently.'
|
|
177
|
+
% (EZ + '%s' % EBSCO))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _split_authors(by):
|
|
181
|
+
by = re.sub(r'^\s*By:\s*', '', by or '')
|
|
182
|
+
by = re.split(r'(?:In:|Subjects:)|\s{2,}', by)[0] # a short record puts fields on one line
|
|
183
|
+
by = re.sub(r'\s*\+\s*\d+\s*more\s*$', '', by)
|
|
184
|
+
out = []
|
|
185
|
+
for name in re.split(r'\s*[;]\s*', by):
|
|
186
|
+
name = name.strip(' .,')
|
|
187
|
+
if not name:
|
|
188
|
+
continue
|
|
189
|
+
out.append(name)
|
|
190
|
+
return out
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
_MONTH = r'(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*'
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def parse_source(source):
|
|
197
|
+
"""'Psychiatry International, Oct2026, volume 7, issue 5, page 235 • CINAHL'"""
|
|
198
|
+
s = re.sub(r'^(?:In:)\s*', '', (source or '').strip())
|
|
199
|
+
s = re.split(r'\s*\u2022\s*', s)[0].strip()
|
|
200
|
+
s = re.sub(r'\s*\(\d+p\)\s*$', '', s) # "(10p)" page count
|
|
201
|
+
s = re.sub(r'(CINAHL|PsycINFO|MEDLINE|Academic Search\w*)\s*$', '', s).strip(' .,')
|
|
202
|
+
s = re.sub(r'\s*\(\d{4,5}-?[\dxX]*\)', '', s) # journal ISSN in brackets, not a year
|
|
203
|
+
out = {'raw': s}
|
|
204
|
+
m = re.search(r'(?<!\d)(19|20)\d{2}(?!\d)', s)
|
|
205
|
+
if m:
|
|
206
|
+
out['year'] = m.group(0)
|
|
207
|
+
m = re.search(r'volume\s+([\w-]+)', s)
|
|
208
|
+
if m:
|
|
209
|
+
out['volume'] = m.group(1)
|
|
210
|
+
m = re.search(r'issue\s+([\w-]+)', s)
|
|
211
|
+
if m:
|
|
212
|
+
out['issue'] = m.group(1)
|
|
213
|
+
m = re.search(r'pages?\s+([\w\-–]+)', s, re.I)
|
|
214
|
+
if m:
|
|
215
|
+
out['pages'] = m.group(1)
|
|
216
|
+
out['journal'] = re.split(
|
|
217
|
+
r',\s*(?:%s\s*)?(?:19|20)\d{2}|,\s*(?:19|20)\d{2}|,\s*volume' % _MONTH, s)[0].strip(' .,')
|
|
218
|
+
return {k: v for k, v in out.items() if v}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _normalise_query(query):
|
|
222
|
+
"""Give a bare multi-word query AND semantics.
|
|
223
|
+
|
|
224
|
+
EBSCO's URL parameter treats a space-separated string as one phrase, so "delirium older
|
|
225
|
+
adults" comes back with a single hit while the same words joined by AND return the
|
|
226
|
+
expected set. Keep quotes, braces and existing operators untouched, since those carry
|
|
227
|
+
the author's intent.
|
|
228
|
+
"""
|
|
229
|
+
q = (query or '').strip()
|
|
230
|
+
if not q or re.search(r'"|\bAND\b|\bOR\b|\bNOT\b|\bN[0-9]\b|[()]', q):
|
|
231
|
+
return q
|
|
232
|
+
words = q.split()
|
|
233
|
+
if len(words) < 2:
|
|
234
|
+
return q
|
|
235
|
+
return ' AND '.join(words)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def search(database, query, limit=10):
|
|
239
|
+
"""Run a search in one database and return citation-shaped results."""
|
|
240
|
+
key = (database or '').strip().lower()
|
|
241
|
+
if key not in DATABASES:
|
|
242
|
+
raise DatabaseError('unknown database %r; known: %s' % (database, ', '.join(sorted(DATABASES))))
|
|
243
|
+
db = DATABASES[key]
|
|
244
|
+
if db.get('reader') == 'cochrane':
|
|
245
|
+
return _search_cochrane(query, limit)
|
|
246
|
+
if db.get('reader') == 'wos':
|
|
247
|
+
return _search_wos(query, limit)
|
|
248
|
+
if not db.get('ebsco'):
|
|
249
|
+
raise DatabaseError('%s has no structured search here; open it with open_database()'
|
|
250
|
+
% db['name'])
|
|
251
|
+
import urllib.parse
|
|
252
|
+
# %20, not '+': EBSCO reads a literal plus as an operator and a two-word query then
|
|
253
|
+
# matches nothing at all.
|
|
254
|
+
url = db['url'] + urllib.parse.quote(_normalise_query(query), safe='') + '&limiters=None'
|
|
255
|
+
with _Tab(url) as page:
|
|
256
|
+
for _ in range(12):
|
|
257
|
+
page._run(asyncio.sleep(3))
|
|
258
|
+
try:
|
|
259
|
+
raw = json.loads(page.eval(CARDS_JS) or '{}')
|
|
260
|
+
except DatabaseError:
|
|
261
|
+
continue
|
|
262
|
+
if raw.get('cards'):
|
|
263
|
+
break
|
|
264
|
+
_check_session(page)
|
|
265
|
+
if 'No results' in page.text():
|
|
266
|
+
return {'database': db['name'], 'query': query, 'total': '0', 'results': []}
|
|
267
|
+
else:
|
|
268
|
+
_check_session(page)
|
|
269
|
+
raise DatabaseError('no result cards appeared for %r in %s' % (query, db['name']))
|
|
270
|
+
out = []
|
|
271
|
+
for c in raw['cards'][:limit]:
|
|
272
|
+
lines = c.get('lines') or []
|
|
273
|
+
by = next((l for l in lines if l.lower().startswith('by:')), '')
|
|
274
|
+
# A short record puts the source on the same line as the authors: "By:A;BIn:Journal, 2026, ..."
|
|
275
|
+
inline_src = ''
|
|
276
|
+
if 'In:' in by:
|
|
277
|
+
by, inline_src = by.split('In:', 1)
|
|
278
|
+
inline_src = 'In:' + inline_src
|
|
279
|
+
src_line = next((l for l in lines if l.lower().startswith('in:')), '') or inline_src
|
|
280
|
+
if not src_line:
|
|
281
|
+
src_line = next((l for l in lines
|
|
282
|
+
if not l.lower().startswith('by:')
|
|
283
|
+
and re.search(r'\bvolume\s+\d', l) and len(l) < 220), '')
|
|
284
|
+
subj = next((l for l in lines if l.lower().startswith('subjects:')), '')
|
|
285
|
+
subj = re.split(r'\+\d+ more', subj)[0]
|
|
286
|
+
src = parse_source(src_line)
|
|
287
|
+
out.append({
|
|
288
|
+
'title': c.get('title', ''),
|
|
289
|
+
'authors': _split_authors(by),
|
|
290
|
+
'journal': src.get('journal', ''),
|
|
291
|
+
'year': src.get('year', ''),
|
|
292
|
+
'volume': src.get('volume', ''),
|
|
293
|
+
'issue': src.get('issue', ''),
|
|
294
|
+
'pages': src.get('pages', ''),
|
|
295
|
+
'subjects': re.sub(r'^Subjects:\s*', '', subj).strip(' ;'),
|
|
296
|
+
'peer_reviewed': c.get('peer_reviewed', False),
|
|
297
|
+
'full_text': c.get('full_text', False),
|
|
298
|
+
'url': (EBSCO + c['href']) if c.get('href', '').startswith('/') else c.get('href', ''),
|
|
299
|
+
'database': db['name'],
|
|
300
|
+
})
|
|
301
|
+
return {'database': db['name'], 'query': query, 'total': raw.get('total', ''), 'results': out}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def detail(url):
|
|
305
|
+
"""Open one record and return its abstract and identifiers."""
|
|
306
|
+
with _Tab(url) as page:
|
|
307
|
+
for _ in range(6):
|
|
308
|
+
page._run(asyncio.sleep(2))
|
|
309
|
+
if 'Abstract' in page.text() or 'Subjects' in page.text():
|
|
310
|
+
break
|
|
311
|
+
_check_session(page)
|
|
312
|
+
return json.loads(page.eval(DETAIL_JS) or '{}')
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def status(database='cinahl'):
|
|
316
|
+
"""Is the school session still good for this database?"""
|
|
317
|
+
key = (database or '').strip().lower()
|
|
318
|
+
db = DATABASES.get(key)
|
|
319
|
+
if not db:
|
|
320
|
+
raise DatabaseError('unknown database %r' % database)
|
|
321
|
+
with _Tab() as page:
|
|
322
|
+
page.goto(EZ + db['url'], settle=4)
|
|
323
|
+
page._run(asyncio.sleep(3))
|
|
324
|
+
url = page.eval('location.href') or ''
|
|
325
|
+
body = page.text()
|
|
326
|
+
signed_out = 'login.microsoftonline.com' in url or 'Sign in to your account' in body
|
|
327
|
+
return {
|
|
328
|
+
'database': db['name'],
|
|
329
|
+
'authenticated': not signed_out,
|
|
330
|
+
'landed_on': url.split('?')[0][:120],
|
|
331
|
+
'institution': 'Sophiahemmet Högskola' if 'SOPHIAHEMMET' in body.upper() else '',
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def open_database(database):
|
|
336
|
+
"""Open a database in the school browser and report where it landed."""
|
|
337
|
+
key = (database or '').strip().lower()
|
|
338
|
+
db = DATABASES.get(key)
|
|
339
|
+
if not db:
|
|
340
|
+
raise DatabaseError('unknown database %r; known: %s' % (database, ', '.join(sorted(DATABASES))))
|
|
341
|
+
with _Tab(EZ + db['url']) as page:
|
|
342
|
+
page._run(asyncio.sleep(4))
|
|
343
|
+
url = page.eval('location.href') or ''
|
|
344
|
+
body = page.text()
|
|
345
|
+
return {
|
|
346
|
+
'database': db['name'],
|
|
347
|
+
'landed_on': url.split('?')[0][:120],
|
|
348
|
+
'authenticated': 'login.microsoftonline.com' not in url,
|
|
349
|
+
'hint': ('open in the school browser and sign in once' if 'login.microsoftonline.com' in url
|
|
350
|
+
else ''),
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def catalogue():
|
|
355
|
+
"""The databases the school lists, with what each is good for."""
|
|
356
|
+
return [
|
|
357
|
+
{'key': k, 'name': v['name'], 'searchable': bool(v.get('ebsco') or v.get('reader')),
|
|
358
|
+
'behind_ezproxy': True}
|
|
359
|
+
for k, v in DATABASES.items()
|
|
360
|
+
]
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
# ---- Cochrane Library -------------------------------------------------------------------
|
|
364
|
+
# The URL parameters on /search are ignored unless the session came from the form, and Enter
|
|
365
|
+
# in the box does nothing: set the value through the native setter, then click .searchByBtn.
|
|
366
|
+
COCHRANE = 'https://www-cochranelibrary-com.shh.idm.oclc.org'
|
|
367
|
+
|
|
368
|
+
COCHRANE_JS = r"""(() => {
|
|
369
|
+
const b = document.body.innerText;
|
|
370
|
+
const total = ((b.match(/(\d[\d,]*)\s*\n?\s*Cochrane Reviews/) || [])[1] || '').replace(/,/g, '');
|
|
371
|
+
return JSON.stringify({total: total, url: location.href, items:
|
|
372
|
+
[...document.querySelectorAll('.search-results-item')].map(e => {
|
|
373
|
+
const a = e.querySelector('a[href*="/cdsr/doi/"]');
|
|
374
|
+
const lines = e.innerText.split('\n').map(s => s.trim()).filter(Boolean);
|
|
375
|
+
return {title: a ? a.innerText.trim() : '', href: a ? a.getAttribute('href') : '', lines: lines};
|
|
376
|
+
})});
|
|
377
|
+
})()"""
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _search_cochrane(query, limit):
|
|
381
|
+
with _Tab(COCHRANE + '/cdsr/reviews') as page:
|
|
382
|
+
for _ in range(12): # the search box is rendered late
|
|
383
|
+
page._run(asyncio.sleep(2))
|
|
384
|
+
try:
|
|
385
|
+
if page.eval("!!document.querySelector('#searchText') && !!document.querySelector('.searchByBtn')"):
|
|
386
|
+
break
|
|
387
|
+
except DatabaseError:
|
|
388
|
+
continue
|
|
389
|
+
_check_session(page)
|
|
390
|
+
done = page.eval("""(q => {
|
|
391
|
+
const i = document.querySelector('#searchText');
|
|
392
|
+
if (!i) return 'no box';
|
|
393
|
+
Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(i, q);
|
|
394
|
+
i.dispatchEvent(new Event('input', {bubbles: true}));
|
|
395
|
+
i.dispatchEvent(new Event('change', {bubbles: true}));
|
|
396
|
+
const b = document.querySelector('.searchByBtn');
|
|
397
|
+
if (!b) return 'no button';
|
|
398
|
+
b.click(); return 'ok';
|
|
399
|
+
})(%s)""" % json.dumps(query))
|
|
400
|
+
if done != 'ok':
|
|
401
|
+
raise DatabaseError('Cochrane search form changed: %s' % done)
|
|
402
|
+
raw = {}
|
|
403
|
+
for _ in range(8):
|
|
404
|
+
page._run(asyncio.sleep(3))
|
|
405
|
+
try:
|
|
406
|
+
raw = json.loads(page.eval(COCHRANE_JS) or '{}')
|
|
407
|
+
except DatabaseError:
|
|
408
|
+
continue
|
|
409
|
+
hl = urllib.parse.unquote(raw.get('items', [{}])[0].get('href', '') if raw.get('items') else '')
|
|
410
|
+
fresh = all(w.lower() in hl.lower() for w in re.findall(r'\w{4,}', query)[:3]) if hl else False
|
|
411
|
+
if raw.get('url', '').rstrip('/').endswith('/search') and raw.get('items') and fresh:
|
|
412
|
+
break
|
|
413
|
+
out = []
|
|
414
|
+
for it in (raw.get('items') or [])[:limit]:
|
|
415
|
+
m = re.search(r'/cdsr/doi/(10\.[^/]+/[^/]+)/', it.get('href') or '')
|
|
416
|
+
doi = m.group(1) if m else ''
|
|
417
|
+
lines = [l for l in it.get('lines') or [] if not re.fullmatch(r'\d+', l)]
|
|
418
|
+
title = it.get('title') or (lines[0] if lines else '')
|
|
419
|
+
rest = [l for l in lines if l != title]
|
|
420
|
+
authors = [a.strip() for a in (rest[0] if rest else '').split(',') if a.strip()]
|
|
421
|
+
date = next((l for l in rest if re.search(r'\b(19|20)\d{2}\b', l)), '')
|
|
422
|
+
kind = next((l for l in rest if re.search(r'Intervention|Diagnostic|Prognosis|Overview|Methodology|Qualitative', l)), '')
|
|
423
|
+
out.append({
|
|
424
|
+
'title': title, 'authors': authors, 'journal': 'Cochrane Database of Systematic Reviews',
|
|
425
|
+
'year': (re.search(r'(19|20)\d{2}', date) or [''])[0] if date else '',
|
|
426
|
+
'review_type': kind, 'doi': doi, 'peer_reviewed': True, 'full_text': True,
|
|
427
|
+
'url': COCHRANE + (it.get('href') or ''), 'database': 'Cochrane Library',
|
|
428
|
+
})
|
|
429
|
+
return {'database': 'Cochrane Library', 'query': query, 'total': raw.get('total', ''),
|
|
430
|
+
'results': out}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
# ---- Web of Science ---------------------------------------------------------------------
|
|
434
|
+
# The summary page takes the query as JSON in the URL (`queryJson`), which skips the Angular
|
|
435
|
+
# search form entirely. Records are <app-record>; the card has no DOI, so ask Crossref by
|
|
436
|
+
# title when a DOI is needed (doi_for_title).
|
|
437
|
+
WOS = 'https://www-webofscience-com.shh.idm.oclc.org'
|
|
438
|
+
|
|
439
|
+
WOS_JS = r"""(() => {
|
|
440
|
+
const b = document.body.innerText;
|
|
441
|
+
return JSON.stringify({
|
|
442
|
+
total: ((b.match(/([\d,]+)\s+results/) || [])[1] || '').replace(/,/g, ''),
|
|
443
|
+
records: [...document.querySelectorAll('app-record')].map(r => {
|
|
444
|
+
const a = r.querySelector('a[data-ta="summary-record-title-link"]');
|
|
445
|
+
const cites = r.querySelector('a[data-ta="stat-number-citation-related-count"]');
|
|
446
|
+
return {
|
|
447
|
+
title: a ? a.innerText.trim() : '',
|
|
448
|
+
href: a ? a.getAttribute('href') : '',
|
|
449
|
+
authors: [...r.querySelectorAll('a[data-ta*="DisplayName-author"]')].map(x => x.innerText.trim()),
|
|
450
|
+
lines: r.innerText.split('\n').map(s => s.trim()).filter(Boolean),
|
|
451
|
+
cited_by: cites ? cites.innerText.trim() : '',
|
|
452
|
+
};
|
|
453
|
+
})});
|
|
454
|
+
})()"""
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _search_wos(query, limit, field='TS'):
|
|
458
|
+
import urllib.parse
|
|
459
|
+
qj = json.dumps([{'rowBoolean': None, 'rowField': field, 'rowText': query}])
|
|
460
|
+
url = WOS + '/wos/woscc/general-summary?queryJson=' + urllib.parse.quote(qj)
|
|
461
|
+
raw = {}
|
|
462
|
+
with _Tab(url) as page:
|
|
463
|
+
for _ in range(10):
|
|
464
|
+
page._run(asyncio.sleep(4))
|
|
465
|
+
try:
|
|
466
|
+
raw = json.loads(page.eval(WOS_JS) or '{}')
|
|
467
|
+
except DatabaseError:
|
|
468
|
+
continue
|
|
469
|
+
if raw.get('records'):
|
|
470
|
+
break
|
|
471
|
+
_check_session(page)
|
|
472
|
+
if re.search(r'Your search found no results|0 results', page.text()):
|
|
473
|
+
break
|
|
474
|
+
out = []
|
|
475
|
+
for r in (raw.get('records') or [])[:limit]:
|
|
476
|
+
lines = r.get('lines') or []
|
|
477
|
+
title = r.get('title', '')
|
|
478
|
+
i = lines.index(title) if title in lines else 1
|
|
479
|
+
tail = lines[i + 1:]
|
|
480
|
+
date = next((l for l in tail if re.fullmatch(r'(?:[A-Z][a-z]{2}\s+)?(?:\d{1,2}\s+)?(19|20)\d{2}', l)), '')
|
|
481
|
+
journal = next((re.sub(r'arrow_drop_down$', '', l).strip() for l in tail
|
|
482
|
+
if l.endswith('arrow_drop_down')), '')
|
|
483
|
+
vol = next((l for l in tail if re.match(r'^\d+(\(\w+\))?(,\s*pp?\.)?', l) and len(l) < 40), '')
|
|
484
|
+
m = re.match(r'^(\d+)(?:\(([^)]+)\))?(?:,\s*pp?\.\s*([\w\-–]+))?', vol)
|
|
485
|
+
authors = r.get('authors') or []
|
|
486
|
+
if not authors and len(tail) > 0:
|
|
487
|
+
authors = [a.strip() for a in re.split(r';|\band\b', tail[0]) if a.strip()]
|
|
488
|
+
out.append({
|
|
489
|
+
'title': title, 'authors': authors, 'journal': journal.title() if journal.isupper() else journal,
|
|
490
|
+
'year': (re.search(r'(19|20)\d{2}', date) or [''])[0] if date else '',
|
|
491
|
+
'volume': m.group(1) if m else '', 'issue': (m.group(2) or '') if m else '',
|
|
492
|
+
'pages': (m.group(3) or '') if m else '', 'cited_by': r.get('cited_by', ''),
|
|
493
|
+
'free_full_text': any('Free Full Text' in l for l in lines),
|
|
494
|
+
'url': WOS + (r.get('href') or ''), 'database': 'Web of Science',
|
|
495
|
+
})
|
|
496
|
+
return {'database': 'Web of Science', 'query': query, 'total': raw.get('total', ''),
|
|
497
|
+
'results': out}
|