unitoolkit 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ladok_vakt/__init__.py +7 -0
- ladok_vakt/__main__.py +51 -0
- ladok_vakt/core.py +553 -0
- ladok_vakt/login.py +232 -0
- tools.json +623 -0
- unitoolkit/__init__.py +1 -0
- unitoolkit/__main__.py +63 -0
- unitoolkit/agent.py +270 -0
- unitoolkit/browser.py +147 -0
- unitoolkit/canvas.py +189 -0
- unitoolkit/chainproxy.py +113 -0
- unitoolkit/citation.py +165 -0
- unitoolkit/databases.py +497 -0
- unitoolkit/docs.py +394 -0
- unitoolkit/fulltext.py +260 -0
- unitoolkit/koha.py +729 -0
- unitoolkit/mcp.py +85 -0
- unitoolkit/programme.py +147 -0
- unitoolkit/search.py +507 -0
- unitoolkit/serve.py +153 -0
- unitoolkit/sharepoint.py +86 -0
- unitoolkit/surface.py +86 -0
- unitoolkit/timeedit.py +209 -0
- unitoolkit/tools.py +555 -0
- unitoolkit-0.7.0.dist-info/METADATA +177 -0
- unitoolkit-0.7.0.dist-info/RECORD +30 -0
- unitoolkit-0.7.0.dist-info/WHEEL +5 -0
- unitoolkit-0.7.0.dist-info/entry_points.txt +4 -0
- unitoolkit-0.7.0.dist-info/licenses/LICENSE +21 -0
- unitoolkit-0.7.0.dist-info/top_level.txt +2 -0
unitoolkit/chainproxy.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Local chaining proxy for the library browser: `python -m unitoolkit.chainproxy`.
|
|
3
|
+
|
|
4
|
+
Chrome cannot take user:pass in --proxy-server for https, so this listens on 127.0.0.1:8118
|
|
5
|
+
and adds the upstream Proxy-Authorization itself. Only needed where the library drops your
|
|
6
|
+
own IP; most students reach their library directly and never start it.
|
|
7
|
+
|
|
8
|
+
Upstream: CHAIN_UPSTREAM_HOST/PORT/USER (default Apify residential, sticky session, since
|
|
9
|
+
Koha binds a login to the client IP). Password from CHAIN_UPSTREAM_PASSWORD_FILE (default
|
|
10
|
+
~/.secrets/APIFY_PROXY_PASSWORD), read at start-up, never logged.
|
|
11
|
+
"""
|
|
12
|
+
import base64
|
|
13
|
+
import os
|
|
14
|
+
import select
|
|
15
|
+
import socket
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
|
|
19
|
+
LISTEN = ('127.0.0.1', int(os.environ.get('CHAIN_PORT', '8118')))
|
|
20
|
+
UP_HOST = os.environ.get('CHAIN_UPSTREAM_HOST', 'proxy.apify.com')
|
|
21
|
+
UP_PORT = int(os.environ.get('CHAIN_UPSTREAM_PORT', '8000'))
|
|
22
|
+
# Sticky session: Koha binds the login to the client IP, so every request in the flow
|
|
23
|
+
# must leave through the same residential exit. Apify keeps a session ~30 minutes.
|
|
24
|
+
UP_USER = os.environ.get('CHAIN_UPSTREAM_USER', 'groups-RESIDENTIAL,session-kohavakt')
|
|
25
|
+
PW_FILE = os.path.expanduser(os.environ.get('CHAIN_UPSTREAM_PASSWORD_FILE',
|
|
26
|
+
'~/.secrets/APIFY_PROXY_PASSWORD'))
|
|
27
|
+
AUTH = '' # set in main(): importing this module must not need the password
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _auth():
|
|
31
|
+
with open(PW_FILE) as f:
|
|
32
|
+
pw = f.read().strip()
|
|
33
|
+
return 'Basic ' + base64.b64encode(f'{UP_USER}:{pw}'.encode()).decode()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def pipe(a, b):
|
|
37
|
+
try:
|
|
38
|
+
while True:
|
|
39
|
+
r, _, _ = select.select([a, b], [], [], 60)
|
|
40
|
+
if not r:
|
|
41
|
+
break
|
|
42
|
+
for s in r:
|
|
43
|
+
data = s.recv(65536)
|
|
44
|
+
if not data:
|
|
45
|
+
return
|
|
46
|
+
(b if s is a else a).sendall(data)
|
|
47
|
+
except Exception:
|
|
48
|
+
pass
|
|
49
|
+
finally:
|
|
50
|
+
for s in (a, b):
|
|
51
|
+
try:
|
|
52
|
+
s.close()
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def handle(client):
|
|
58
|
+
try:
|
|
59
|
+
head = b''
|
|
60
|
+
while b'\r\n\r\n' not in head:
|
|
61
|
+
chunk = client.recv(4096)
|
|
62
|
+
if not chunk:
|
|
63
|
+
return client.close()
|
|
64
|
+
head += chunk
|
|
65
|
+
lines = head.split(b'\r\n')
|
|
66
|
+
method, target, version = lines[0].split(b' ')
|
|
67
|
+
up = socket.create_connection((UP_HOST, UP_PORT), timeout=30)
|
|
68
|
+
if method.upper() == b'CONNECT':
|
|
69
|
+
req = (b'CONNECT ' + target + b' HTTP/1.1\r\nHost: ' + target +
|
|
70
|
+
b'\r\nProxy-Authorization: ' + AUTH.encode() + b'\r\n\r\n')
|
|
71
|
+
up.sendall(req)
|
|
72
|
+
resp = b''
|
|
73
|
+
while b'\r\n\r\n' not in resp:
|
|
74
|
+
chunk = up.recv(4096)
|
|
75
|
+
if not chunk:
|
|
76
|
+
return client.close()
|
|
77
|
+
resp += chunk
|
|
78
|
+
status = resp.split(b'\r\n')[0]
|
|
79
|
+
if b' 200' not in status:
|
|
80
|
+
client.sendall(b'HTTP/1.1 502 Bad Gateway\r\n\r\n')
|
|
81
|
+
print('upstream refused:', status[:60], file=sys.stderr)
|
|
82
|
+
return client.close()
|
|
83
|
+
client.sendall(b'HTTP/1.1 200 Connection established\r\n\r\n')
|
|
84
|
+
pipe(client, up)
|
|
85
|
+
else:
|
|
86
|
+
rest = head.split(b'\r\n\r\n', 1)[1]
|
|
87
|
+
extra = b''.join(b'\r\n' + ln for ln in lines[1:] if ln and not ln.lower().startswith(b'proxy-'))
|
|
88
|
+
up.sendall(lines[0] + extra + b'\r\nProxy-Authorization: ' + AUTH.encode() +
|
|
89
|
+
b'\r\n\r\n' + rest)
|
|
90
|
+
pipe(client, up)
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
print('handler error:', type(exc).__name__, file=sys.stderr)
|
|
93
|
+
try:
|
|
94
|
+
client.close()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def main():
|
|
100
|
+
global AUTH
|
|
101
|
+
AUTH = _auth()
|
|
102
|
+
srv = socket.socket()
|
|
103
|
+
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
104
|
+
srv.bind(LISTEN)
|
|
105
|
+
srv.listen(64)
|
|
106
|
+
print('chaining proxy on %s:%d -> %s:%d (%s)' % (LISTEN[0], LISTEN[1], UP_HOST, UP_PORT, UP_USER))
|
|
107
|
+
while True:
|
|
108
|
+
c, _ = srv.accept()
|
|
109
|
+
threading.Thread(target=handle, args=(c,), daemon=True).start()
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
if __name__ == '__main__':
|
|
113
|
+
main()
|
unitoolkit/citation.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Citations: APA 7 references from a DOI, a DOI from a title, and one search across databases.
|
|
2
|
+
|
|
3
|
+
APA comes from the DOI registry itself (content negotiation on doi.org, which Crossref and
|
|
4
|
+
DataCite both answer), not from scraped search cards, so authors, volume and pages are the
|
|
5
|
+
publisher's own metadata. Two gotchas fixed here: Crossref lowercases Cochrane DOIs in the
|
|
6
|
+
URL it returns, and it adds "Portico." (an archive, not part of the reference) for some
|
|
7
|
+
Wiley titles. Swedish APA (svensk APA 7) differs only in "&" vs "och" and "red." vs "Ed.";
|
|
8
|
+
pass lang='sv' for those.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
|
|
16
|
+
UA = {'User-Agent': 'unitoolkit (https://github.com/vhsgreed/shh-study-kit; mailto:unitoolkit@users.noreply.github.com)'} # mailto = Crossref's polite pool
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _get(url, accept, timeout=20, tries=4):
|
|
20
|
+
"""GET with backoff on 429/503: Crossref throttles bursts, and one throttled lookup
|
|
21
|
+
must not take down a whole multi-database search."""
|
|
22
|
+
import time
|
|
23
|
+
import urllib.error
|
|
24
|
+
delay = 1.5
|
|
25
|
+
for attempt in range(tries):
|
|
26
|
+
req = urllib.request.Request(url, headers=dict(UA, Accept=accept))
|
|
27
|
+
try:
|
|
28
|
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
29
|
+
return r.read().decode('utf-8', 'replace')
|
|
30
|
+
except urllib.error.HTTPError as e:
|
|
31
|
+
if e.code not in (429, 503) or attempt == tries - 1:
|
|
32
|
+
raise
|
|
33
|
+
wait = e.headers.get('Retry-After')
|
|
34
|
+
time.sleep(float(wait) if wait and wait.isdigit() else delay)
|
|
35
|
+
delay *= 2
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def clean_doi(doi):
|
|
39
|
+
return re.sub(r'^(https?://(dx\.)?doi\.org/|doi:\s*)', '', (doi or '').strip(), flags=re.I)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def apa(doi, lang='en'):
|
|
43
|
+
"""APA 7 reference for a DOI, from the registry's own metadata."""
|
|
44
|
+
doi = clean_doi(doi)
|
|
45
|
+
if not doi.startswith('10.'):
|
|
46
|
+
raise ValueError('not a DOI: %r' % doi)
|
|
47
|
+
ref = _get('https://doi.org/' + urllib.parse.quote(doi, safe='/'),
|
|
48
|
+
'text/x-bibliography; style=apa; locale=en-US').strip()
|
|
49
|
+
ref = re.sub(r'\s+Portico\.(?=\s+https://doi)', '', ref) # archive, not a source
|
|
50
|
+
ref = re.sub(r'https://doi\.org/\S+$', 'https://doi.org/' + doi, ref) # keep the DOI's case
|
|
51
|
+
if lang == 'sv':
|
|
52
|
+
ref = re.sub(r',? & ', ' & ', ref) # svensk APA keeps "&" inside the reference
|
|
53
|
+
ref = ref.replace('(Ed.)', '(Red.)').replace('(Eds.)', '(Red.)')
|
|
54
|
+
return ref
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def doi_for_title(title, author=''):
|
|
58
|
+
"""Best Crossref match for a title, with a score so a weak match is visible."""
|
|
59
|
+
q = urllib.parse.urlencode({'query.bibliographic': title, 'query.author': author, 'rows': 1})
|
|
60
|
+
j = json.loads(_get('https://api.crossref.org/works?' + q, 'application/json'))
|
|
61
|
+
items = (j.get('message') or {}).get('items') or []
|
|
62
|
+
if not items:
|
|
63
|
+
return {'doi': '', 'score': 0}
|
|
64
|
+
it = items[0]
|
|
65
|
+
found = (it.get('title') or [''])[0]
|
|
66
|
+
a = re.sub(r'\W+', ' ', title.lower()).split()
|
|
67
|
+
b = re.sub(r'\W+', ' ', found.lower()).split()
|
|
68
|
+
overlap = len(set(a) & set(b)) / max(1, len(set(a)))
|
|
69
|
+
return {'doi': it.get('DOI', ''), 'title': found, 'match': round(overlap, 2),
|
|
70
|
+
'confident': overlap >= 0.8}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
_STOP = set('a an and the of in on for to with by at from or as is are was were be vs och i av för på med som en ett att'.split())
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _terms(text):
|
|
77
|
+
return {w for w in re.findall(r'[a-zåäö0-9]+', (text or '').lower()) if w not in _STOP and len(w) > 2}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def relevance(query, row):
|
|
81
|
+
"""Share of the query's words found in the title (plus subjects when the card has them).
|
|
82
|
+
|
|
83
|
+
Database ranking is not comparable across databases, and Cochrane's own order puts
|
|
84
|
+
off-topic reviews first for multi-word queries. A crude, visible score is better than
|
|
85
|
+
trusting three different black boxes.
|
|
86
|
+
"""
|
|
87
|
+
q = _terms(query)
|
|
88
|
+
if not q:
|
|
89
|
+
return 0.0
|
|
90
|
+
hay = _terms((row.get('title') or '') + ' ' + (row.get('subjects') or ''))
|
|
91
|
+
stems = {w[:6] for w in hay}
|
|
92
|
+
hit = sum(1 for w in q if w in hay or w[:6] in stems)
|
|
93
|
+
return round(hit / len(q), 2)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _total(v):
|
|
97
|
+
digits = re.sub(r'\D', '', str(v or ''))
|
|
98
|
+
return int(digits) if digits else None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _key(r):
|
|
102
|
+
if r.get('doi'):
|
|
103
|
+
return 'doi:' + r['doi'].lower()
|
|
104
|
+
return 't:' + re.sub(r'\W+', '', (r.get('title') or '').lower())[:80]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def find_sources(query, databases=('cinahl', 'pubmed', 'cochrane'), limit=5, with_apa=True):
|
|
108
|
+
"""One query across several databases, deduplicated, each hit with an APA reference.
|
|
109
|
+
|
|
110
|
+
This is the call a student's "find me three sources on X" should become. PubMed goes
|
|
111
|
+
through the free E-utilities; the rest through the school's EZproxy.
|
|
112
|
+
"""
|
|
113
|
+
from . import databases as dbs
|
|
114
|
+
seen, out, totals, errors = set(), [], {}, {}
|
|
115
|
+
for name in databases:
|
|
116
|
+
try:
|
|
117
|
+
if name == 'pubmed':
|
|
118
|
+
from .tools import call
|
|
119
|
+
res = call('pubmed', {'query': query, 'max': limit})
|
|
120
|
+
hits = res if isinstance(res, list) else (res or {}).get('results', [])
|
|
121
|
+
totals['PubMed'] = None
|
|
122
|
+
rows = [{'title': (h.get('title') or '').rstrip('.'), 'authors': h.get('authors') or [],
|
|
123
|
+
'journal': h.get('journal') or '', 'year': str(h.get('year') or ''),
|
|
124
|
+
'doi': h.get('doi') or '', 'pmid': h.get('pmid') or '',
|
|
125
|
+
'database': 'PubMed'} for h in (hits or [])]
|
|
126
|
+
else:
|
|
127
|
+
# ask for more than we keep: the relevance re-rank needs candidates, since
|
|
128
|
+
# Cochrane's own order puts off-topic reviews first
|
|
129
|
+
res = dbs.search(name, query, limit=max(limit * 4, 20) if name == 'cochrane' else limit)
|
|
130
|
+
totals[res['database']] = _total(res.get('total'))
|
|
131
|
+
rows = res['results']
|
|
132
|
+
except Exception as e: # one database failing must not hide the rest
|
|
133
|
+
errors[name] = str(e)[:200]
|
|
134
|
+
continue
|
|
135
|
+
for r in rows:
|
|
136
|
+
r['relevance'] = relevance(query, r)
|
|
137
|
+
# keep the database's order among equals, but never let an off-topic hit outrank
|
|
138
|
+
rows = sorted(rows, key=lambda r: -r['relevance'])
|
|
139
|
+
for r in rows[:limit]:
|
|
140
|
+
if not r.get('doi') and r.get('title') and with_apa:
|
|
141
|
+
try:
|
|
142
|
+
m = doi_for_title(r['title'], (r.get('authors') or [''])[0].split(',')[0])
|
|
143
|
+
except Exception as e:
|
|
144
|
+
m = {}
|
|
145
|
+
r['doi_lookup_error'] = str(e)[:100]
|
|
146
|
+
if m.get('confident'):
|
|
147
|
+
r['doi'] = m['doi']
|
|
148
|
+
k = _key(r)
|
|
149
|
+
if k in seen:
|
|
150
|
+
for o in out:
|
|
151
|
+
if _key(o) == k:
|
|
152
|
+
o.setdefault('also_in', []).append(r.get('database'))
|
|
153
|
+
continue
|
|
154
|
+
seen.add(k)
|
|
155
|
+
if with_apa and r.get('doi'):
|
|
156
|
+
try:
|
|
157
|
+
r['apa'] = apa(r['doi'])
|
|
158
|
+
except Exception as e:
|
|
159
|
+
r['apa_error'] = str(e)[:120]
|
|
160
|
+
out.append(r)
|
|
161
|
+
out.sort(key=lambda r: -r.get('relevance', 0))
|
|
162
|
+
return {'query': query, 'totals': totals, 'results': out, 'errors': errors,
|
|
163
|
+
'note': ('APA from the DOI registry keeps the publisher title case; APA 7 wants '
|
|
164
|
+
'sentence case for article titles. Fix by hand, since proper nouns and '
|
|
165
|
+
'acronyms cannot be lowercased safely by rule.')}
|