unitoolkit 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env python3
2
+ """Local chaining proxy for the library browser: `python -m unitoolkit.chainproxy`.
3
+
4
+ Chrome cannot take user:pass in --proxy-server for https, so this listens on 127.0.0.1:8118
5
+ and adds the upstream Proxy-Authorization itself. Only needed where the library drops your
6
+ own IP; most students reach their library directly and never start it.
7
+
8
+ Upstream: CHAIN_UPSTREAM_HOST/PORT/USER (default Apify residential, sticky session, since
9
+ Koha binds a login to the client IP). Password from CHAIN_UPSTREAM_PASSWORD_FILE (default
10
+ ~/.secrets/APIFY_PROXY_PASSWORD), read at start-up, never logged.
11
+ """
12
+ import base64
13
+ import os
14
+ import select
15
+ import socket
16
+ import sys
17
+ import threading
18
+
19
+ LISTEN = ('127.0.0.1', int(os.environ.get('CHAIN_PORT', '8118')))
20
+ UP_HOST = os.environ.get('CHAIN_UPSTREAM_HOST', 'proxy.apify.com')
21
+ UP_PORT = int(os.environ.get('CHAIN_UPSTREAM_PORT', '8000'))
22
+ # Sticky session: Koha binds the login to the client IP, so every request in the flow
23
+ # must leave through the same residential exit. Apify keeps a session ~30 minutes.
24
+ UP_USER = os.environ.get('CHAIN_UPSTREAM_USER', 'groups-RESIDENTIAL,session-kohavakt')
25
+ PW_FILE = os.path.expanduser(os.environ.get('CHAIN_UPSTREAM_PASSWORD_FILE',
26
+ '~/.secrets/APIFY_PROXY_PASSWORD'))
27
+ AUTH = '' # set in main(): importing this module must not need the password
28
+
29
+
30
+ def _auth():
31
+ with open(PW_FILE) as f:
32
+ pw = f.read().strip()
33
+ return 'Basic ' + base64.b64encode(f'{UP_USER}:{pw}'.encode()).decode()
34
+
35
+
36
+ def pipe(a, b):
37
+ try:
38
+ while True:
39
+ r, _, _ = select.select([a, b], [], [], 60)
40
+ if not r:
41
+ break
42
+ for s in r:
43
+ data = s.recv(65536)
44
+ if not data:
45
+ return
46
+ (b if s is a else a).sendall(data)
47
+ except Exception:
48
+ pass
49
+ finally:
50
+ for s in (a, b):
51
+ try:
52
+ s.close()
53
+ except Exception:
54
+ pass
55
+
56
+
57
+ def handle(client):
58
+ try:
59
+ head = b''
60
+ while b'\r\n\r\n' not in head:
61
+ chunk = client.recv(4096)
62
+ if not chunk:
63
+ return client.close()
64
+ head += chunk
65
+ lines = head.split(b'\r\n')
66
+ method, target, version = lines[0].split(b' ')
67
+ up = socket.create_connection((UP_HOST, UP_PORT), timeout=30)
68
+ if method.upper() == b'CONNECT':
69
+ req = (b'CONNECT ' + target + b' HTTP/1.1\r\nHost: ' + target +
70
+ b'\r\nProxy-Authorization: ' + AUTH.encode() + b'\r\n\r\n')
71
+ up.sendall(req)
72
+ resp = b''
73
+ while b'\r\n\r\n' not in resp:
74
+ chunk = up.recv(4096)
75
+ if not chunk:
76
+ return client.close()
77
+ resp += chunk
78
+ status = resp.split(b'\r\n')[0]
79
+ if b' 200' not in status:
80
+ client.sendall(b'HTTP/1.1 502 Bad Gateway\r\n\r\n')
81
+ print('upstream refused:', status[:60], file=sys.stderr)
82
+ return client.close()
83
+ client.sendall(b'HTTP/1.1 200 Connection established\r\n\r\n')
84
+ pipe(client, up)
85
+ else:
86
+ rest = head.split(b'\r\n\r\n', 1)[1]
87
+ extra = b''.join(b'\r\n' + ln for ln in lines[1:] if ln and not ln.lower().startswith(b'proxy-'))
88
+ up.sendall(lines[0] + extra + b'\r\nProxy-Authorization: ' + AUTH.encode() +
89
+ b'\r\n\r\n' + rest)
90
+ pipe(client, up)
91
+ except Exception as exc:
92
+ print('handler error:', type(exc).__name__, file=sys.stderr)
93
+ try:
94
+ client.close()
95
+ except Exception:
96
+ pass
97
+
98
+
99
+ def main():
100
+ global AUTH
101
+ AUTH = _auth()
102
+ srv = socket.socket()
103
+ srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
104
+ srv.bind(LISTEN)
105
+ srv.listen(64)
106
+ print('chaining proxy on %s:%d -> %s:%d (%s)' % (LISTEN[0], LISTEN[1], UP_HOST, UP_PORT, UP_USER))
107
+ while True:
108
+ c, _ = srv.accept()
109
+ threading.Thread(target=handle, args=(c,), daemon=True).start()
110
+
111
+
112
+ if __name__ == '__main__':
113
+ main()
unitoolkit/citation.py ADDED
@@ -0,0 +1,165 @@
1
+ """Citations: APA 7 references from a DOI, a DOI from a title, and one search across databases.
2
+
3
+ APA comes from the DOI registry itself (content negotiation on doi.org, which Crossref and
4
+ DataCite both answer), not from scraped search cards, so authors, volume and pages are the
5
+ publisher's own metadata. Two gotchas fixed here: Crossref lowercases Cochrane DOIs in the
6
+ URL it returns, and it adds "Portico." (an archive, not part of the reference) for some
7
+ Wiley titles. Swedish APA (svensk APA 7) differs only in "&" vs "och" and "red." vs "Ed.";
8
+ pass lang='sv' for those.
9
+ """
10
+
11
+ import json
12
+ import re
13
+ import urllib.parse
14
+ import urllib.request
15
+
16
+ UA = {'User-Agent': 'unitoolkit (https://github.com/vhsgreed/shh-study-kit; mailto:unitoolkit@users.noreply.github.com)'} # mailto = Crossref's polite pool
17
+
18
+
19
+ def _get(url, accept, timeout=20, tries=4):
20
+ """GET with backoff on 429/503: Crossref throttles bursts, and one throttled lookup
21
+ must not take down a whole multi-database search."""
22
+ import time
23
+ import urllib.error
24
+ delay = 1.5
25
+ for attempt in range(tries):
26
+ req = urllib.request.Request(url, headers=dict(UA, Accept=accept))
27
+ try:
28
+ with urllib.request.urlopen(req, timeout=timeout) as r:
29
+ return r.read().decode('utf-8', 'replace')
30
+ except urllib.error.HTTPError as e:
31
+ if e.code not in (429, 503) or attempt == tries - 1:
32
+ raise
33
+ wait = e.headers.get('Retry-After')
34
+ time.sleep(float(wait) if wait and wait.isdigit() else delay)
35
+ delay *= 2
36
+
37
+
38
+ def clean_doi(doi):
39
+ return re.sub(r'^(https?://(dx\.)?doi\.org/|doi:\s*)', '', (doi or '').strip(), flags=re.I)
40
+
41
+
42
+ def apa(doi, lang='en'):
43
+ """APA 7 reference for a DOI, from the registry's own metadata."""
44
+ doi = clean_doi(doi)
45
+ if not doi.startswith('10.'):
46
+ raise ValueError('not a DOI: %r' % doi)
47
+ ref = _get('https://doi.org/' + urllib.parse.quote(doi, safe='/'),
48
+ 'text/x-bibliography; style=apa; locale=en-US').strip()
49
+ ref = re.sub(r'\s+Portico\.(?=\s+https://doi)', '', ref) # archive, not a source
50
+ ref = re.sub(r'https://doi\.org/\S+$', 'https://doi.org/' + doi, ref) # keep the DOI's case
51
+ if lang == 'sv':
52
+ ref = re.sub(r',? & ', ' & ', ref) # svensk APA keeps "&" inside the reference
53
+ ref = ref.replace('(Ed.)', '(Red.)').replace('(Eds.)', '(Red.)')
54
+ return ref
55
+
56
+
57
+ def doi_for_title(title, author=''):
58
+ """Best Crossref match for a title, with a score so a weak match is visible."""
59
+ q = urllib.parse.urlencode({'query.bibliographic': title, 'query.author': author, 'rows': 1})
60
+ j = json.loads(_get('https://api.crossref.org/works?' + q, 'application/json'))
61
+ items = (j.get('message') or {}).get('items') or []
62
+ if not items:
63
+ return {'doi': '', 'score': 0}
64
+ it = items[0]
65
+ found = (it.get('title') or [''])[0]
66
+ a = re.sub(r'\W+', ' ', title.lower()).split()
67
+ b = re.sub(r'\W+', ' ', found.lower()).split()
68
+ overlap = len(set(a) & set(b)) / max(1, len(set(a)))
69
+ return {'doi': it.get('DOI', ''), 'title': found, 'match': round(overlap, 2),
70
+ 'confident': overlap >= 0.8}
71
+
72
+
73
+ _STOP = set('a an and the of in on for to with by at from or as is are was were be vs och i av för på med som en ett att'.split())
74
+
75
+
76
+ def _terms(text):
77
+ return {w for w in re.findall(r'[a-zåäö0-9]+', (text or '').lower()) if w not in _STOP and len(w) > 2}
78
+
79
+
80
+ def relevance(query, row):
81
+ """Share of the query's words found in the title (plus subjects when the card has them).
82
+
83
+ Database ranking is not comparable across databases, and Cochrane's own order puts
84
+ off-topic reviews first for multi-word queries. A crude, visible score is better than
85
+ trusting three different black boxes.
86
+ """
87
+ q = _terms(query)
88
+ if not q:
89
+ return 0.0
90
+ hay = _terms((row.get('title') or '') + ' ' + (row.get('subjects') or ''))
91
+ stems = {w[:6] for w in hay}
92
+ hit = sum(1 for w in q if w in hay or w[:6] in stems)
93
+ return round(hit / len(q), 2)
94
+
95
+
96
+ def _total(v):
97
+ digits = re.sub(r'\D', '', str(v or ''))
98
+ return int(digits) if digits else None
99
+
100
+
101
+ def _key(r):
102
+ if r.get('doi'):
103
+ return 'doi:' + r['doi'].lower()
104
+ return 't:' + re.sub(r'\W+', '', (r.get('title') or '').lower())[:80]
105
+
106
+
107
+ def find_sources(query, databases=('cinahl', 'pubmed', 'cochrane'), limit=5, with_apa=True):
108
+ """One query across several databases, deduplicated, each hit with an APA reference.
109
+
110
+ This is the call a student's "find me three sources on X" should become. PubMed goes
111
+ through the free E-utilities; the rest through the school's EZproxy.
112
+ """
113
+ from . import databases as dbs
114
+ seen, out, totals, errors = set(), [], {}, {}
115
+ for name in databases:
116
+ try:
117
+ if name == 'pubmed':
118
+ from .tools import call
119
+ res = call('pubmed', {'query': query, 'max': limit})
120
+ hits = res if isinstance(res, list) else (res or {}).get('results', [])
121
+ totals['PubMed'] = None
122
+ rows = [{'title': (h.get('title') or '').rstrip('.'), 'authors': h.get('authors') or [],
123
+ 'journal': h.get('journal') or '', 'year': str(h.get('year') or ''),
124
+ 'doi': h.get('doi') or '', 'pmid': h.get('pmid') or '',
125
+ 'database': 'PubMed'} for h in (hits or [])]
126
+ else:
127
+ # ask for more than we keep: the relevance re-rank needs candidates, since
128
+ # Cochrane's own order puts off-topic reviews first
129
+ res = dbs.search(name, query, limit=max(limit * 4, 20) if name == 'cochrane' else limit)
130
+ totals[res['database']] = _total(res.get('total'))
131
+ rows = res['results']
132
+ except Exception as e: # one database failing must not hide the rest
133
+ errors[name] = str(e)[:200]
134
+ continue
135
+ for r in rows:
136
+ r['relevance'] = relevance(query, r)
137
+ # keep the database's order among equals, but never let an off-topic hit outrank
138
+ rows = sorted(rows, key=lambda r: -r['relevance'])
139
+ for r in rows[:limit]:
140
+ if not r.get('doi') and r.get('title') and with_apa:
141
+ try:
142
+ m = doi_for_title(r['title'], (r.get('authors') or [''])[0].split(',')[0])
143
+ except Exception as e:
144
+ m = {}
145
+ r['doi_lookup_error'] = str(e)[:100]
146
+ if m.get('confident'):
147
+ r['doi'] = m['doi']
148
+ k = _key(r)
149
+ if k in seen:
150
+ for o in out:
151
+ if _key(o) == k:
152
+ o.setdefault('also_in', []).append(r.get('database'))
153
+ continue
154
+ seen.add(k)
155
+ if with_apa and r.get('doi'):
156
+ try:
157
+ r['apa'] = apa(r['doi'])
158
+ except Exception as e:
159
+ r['apa_error'] = str(e)[:120]
160
+ out.append(r)
161
+ out.sort(key=lambda r: -r.get('relevance', 0))
162
+ return {'query': query, 'totals': totals, 'results': out, 'errors': errors,
163
+ 'note': ('APA from the DOI registry keeps the publisher title case; APA 7 wants '
164
+ 'sentence case for article titles. Fix by hand, since proper nouns and '
165
+ 'acronyms cannot be lowercased safely by rule.')}