litdb 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litdb/__init__.py +1 -0
- litdb/bibtex.py +182 -0
- litdb/cli.py +1510 -0
- litdb/coa.py +152 -0
- litdb/db.py +362 -0
- litdb/gpt.py +150 -0
- litdb/lab.py +23 -0
- litdb/openalex.py +81 -0
- litdb/pdf.py +19 -0
- litdb/utils.py +64 -0
- litdb-2.0.0.dist-info/METADATA +804 -0
- litdb-2.0.0.dist-info/RECORD +15 -0
- litdb-2.0.0.dist-info/WHEEL +5 -0
- litdb-2.0.0.dist-info/entry_points.txt +2 -0
- litdb-2.0.0.dist-info/top_level.txt +1 -0
litdb/cli.py
ADDED
|
@@ -0,0 +1,1510 @@
|
|
|
1
|
+
"""CLI for litdb.
|
|
2
|
+
|
|
3
|
+
The main command is litdb. There are subcommands for the actions.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import datetime
|
|
8
|
+
import json
|
|
9
|
+
import pathlib
|
|
10
|
+
import tempfile
|
|
11
|
+
import sys
|
|
12
|
+
import warnings
|
|
13
|
+
|
|
14
|
+
import bs4
|
|
15
|
+
import click
|
|
16
|
+
import dateparser
|
|
17
|
+
from docx import Document
|
|
18
|
+
from IPython import get_ipython
|
|
19
|
+
from IPython.display import display, HTML
|
|
20
|
+
from jinja2 import Template
|
|
21
|
+
from more_itertools import batched
|
|
22
|
+
import nbformat
|
|
23
|
+
from nbconvert import MarkdownExporter
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from pptx import Presentation
|
|
27
|
+
import requests
|
|
28
|
+
from rich import print as richprint
|
|
29
|
+
from rich.console import Console
|
|
30
|
+
from sentence_transformers import SentenceTransformer
|
|
31
|
+
import tabulate
|
|
32
|
+
from tqdm import tqdm
|
|
33
|
+
import webbrowser
|
|
34
|
+
|
|
35
|
+
from .utils import get_config, init_litdb
|
|
36
|
+
from .db import get_db, add_source, add_work, add_author, update_filter, add_bibtex
|
|
37
|
+
from .openalex import get_data, get_text
|
|
38
|
+
from .pdf import add_pdf
|
|
39
|
+
from .bibtex import dump_bibtex
|
|
40
|
+
from .gpt import gpt
|
|
41
|
+
|
|
42
|
+
warnings.filterwarnings("ignore")
|
|
43
|
+
|
|
44
|
+
db = get_db()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@click.group()
|
|
48
|
+
def cli():
|
|
49
|
+
"""Group command for litdb."""
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@cli.command()
|
|
54
|
+
def init():
|
|
55
|
+
"""Initialize a litdb directory in the current working directory."""
|
|
56
|
+
init_litdb()
|
|
57
|
+
print(f"Initialized in {os.getcwd()}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
#################
|
|
61
|
+
# Add functions #
|
|
62
|
+
#################
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@cli.command()
|
|
66
|
+
@click.argument("sources", nargs=-1)
|
|
67
|
+
@click.option("--references", is_flag=True, help="Add references too.")
|
|
68
|
+
@click.option("--related", is_flag=True, help="Add related too.")
|
|
69
|
+
@click.option("--citing", is_flag=True, help="Add citing too.")
|
|
70
|
+
@click.option("--all", is_flag=True, help="Add references, related and citing.")
|
|
71
|
+
@click.option("-t", "--tag", "tags", multiple=True)
|
|
72
|
+
def add(
|
|
73
|
+
sources,
|
|
74
|
+
references=False,
|
|
75
|
+
citing=False,
|
|
76
|
+
related=False,
|
|
77
|
+
all=False,
|
|
78
|
+
verbose=False,
|
|
79
|
+
tags=None,
|
|
80
|
+
):
|
|
81
|
+
"""Add WIDS to the db.
|
|
82
|
+
|
|
83
|
+
REFERENCES, RELATED, CITING are flags to also add those for DOI sources. ALL
|
|
84
|
+
is shorthand for all of those.
|
|
85
|
+
|
|
86
|
+
SOURCES can be one or more of a doi or orcid, a pdf path, a url, bibtex
|
|
87
|
+
file, or other kind of file assumed to be text.
|
|
88
|
+
|
|
89
|
+
TAGS is a list of tags to add to the source.
|
|
90
|
+
|
|
91
|
+
These are one time additions.
|
|
92
|
+
|
|
93
|
+
"""
|
|
94
|
+
for source in tqdm(sources):
|
|
95
|
+
# a work
|
|
96
|
+
if source.startswith("10.") or "doi.org" in source:
|
|
97
|
+
if source.startswith("10."):
|
|
98
|
+
source = f"https://doi.org/{source}"
|
|
99
|
+
|
|
100
|
+
if all:
|
|
101
|
+
references, citing, related = True, True, True
|
|
102
|
+
|
|
103
|
+
add_work(source, references, citing, related)
|
|
104
|
+
|
|
105
|
+
# works from an author
|
|
106
|
+
elif "orcid" in source or source.lower().startswith("https://openalex.org/a"):
|
|
107
|
+
add_author(source)
|
|
108
|
+
|
|
109
|
+
# a bibtex file
|
|
110
|
+
elif source.endswith(".bib"):
|
|
111
|
+
add_bibtex(source)
|
|
112
|
+
|
|
113
|
+
# pdf
|
|
114
|
+
elif source.endswith(".pdf"):
|
|
115
|
+
source = os.path.abspath(source)
|
|
116
|
+
add_pdf(source)
|
|
117
|
+
|
|
118
|
+
# docx
|
|
119
|
+
elif source.endswith(".docx"):
|
|
120
|
+
source = os.path.abspath(source)
|
|
121
|
+
doc = Document(source)
|
|
122
|
+
add_source(source, "\n".join([para.text for para in doc.paragraphs]))
|
|
123
|
+
|
|
124
|
+
# pptx
|
|
125
|
+
elif source.endswith(".pptx"):
|
|
126
|
+
source = os.path.abspath(source)
|
|
127
|
+
prs = Presentation(source)
|
|
128
|
+
text = []
|
|
129
|
+
for slide in prs.slides:
|
|
130
|
+
for shape in slide.shapes:
|
|
131
|
+
if hasattr(shape, "text"):
|
|
132
|
+
text.append(shape.text)
|
|
133
|
+
add_source(source, "\n".join(text))
|
|
134
|
+
|
|
135
|
+
# local html
|
|
136
|
+
elif not source.startswith("http") and source.endswith(".html"):
|
|
137
|
+
source = os.path.abspath(source)
|
|
138
|
+
with open(source) as f:
|
|
139
|
+
text = f.read()
|
|
140
|
+
soup = bs4.BeautifulSoup(text, features="lxml")
|
|
141
|
+
add_source(source, soup.get_text())
|
|
142
|
+
|
|
143
|
+
# a url
|
|
144
|
+
elif source.startswith("http"):
|
|
145
|
+
soup = bs4.BeautifulSoup(requests.get(source).text)
|
|
146
|
+
add_source(source, soup.get_text())
|
|
147
|
+
|
|
148
|
+
# ipynb
|
|
149
|
+
elif source.endswith(".ipynb"):
|
|
150
|
+
source = os.path.abspath(source)
|
|
151
|
+
with open(source) as f:
|
|
152
|
+
notebook = nbformat.read(f, as_version=4)
|
|
153
|
+
|
|
154
|
+
# Create a Markdown exporter
|
|
155
|
+
markdown_exporter = MarkdownExporter()
|
|
156
|
+
|
|
157
|
+
# Convert the notebook to Markdown
|
|
158
|
+
(body, resources) = markdown_exporter.from_notebook_node(notebook)
|
|
159
|
+
|
|
160
|
+
add_source(source, body)
|
|
161
|
+
|
|
162
|
+
# assume it is text
|
|
163
|
+
else:
|
|
164
|
+
source = os.path.abspath(source)
|
|
165
|
+
with open(source) as f:
|
|
166
|
+
text = f.read()
|
|
167
|
+
add_source(source, text)
|
|
168
|
+
|
|
169
|
+
if tags:
|
|
170
|
+
with click.Context(add_tag) as ctx:
|
|
171
|
+
ctx.invoke(add_tag, sources=sources, tags=tags)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@cli.command()
|
|
175
|
+
@click.argument("query", nargs=-1)
|
|
176
|
+
@click.option("--references", is_flag=True)
|
|
177
|
+
@click.option("--related", is_flag=True)
|
|
178
|
+
@click.option("--citing", is_flag=True)
|
|
179
|
+
def crossref(query, references, related, citing):
|
|
180
|
+
"""Add entries to litdb from a crossref query."""
|
|
181
|
+
query = " ".join(query)
|
|
182
|
+
resp = requests.get("https://api.crossref.org/works", params={"query": query})
|
|
183
|
+
|
|
184
|
+
if resp.status_code == 200:
|
|
185
|
+
data = resp.json()
|
|
186
|
+
for i, item in enumerate(data["message"]["items"]):
|
|
187
|
+
authors = ", ".join(
|
|
188
|
+
[f'{au["given"]} {au["family"]}' for au in item.get("author", [])]
|
|
189
|
+
)
|
|
190
|
+
source = " ".join(item.get("container-title", ["no source"]))
|
|
191
|
+
published = item.get("published", {}) or {}
|
|
192
|
+
year = published.get("date-parts", [["no year"]])[0][0]
|
|
193
|
+
title = item.get("title", ["no title"])
|
|
194
|
+
richprint(
|
|
195
|
+
f'{i}. {" ".join(title)}, {authors}, {source} ({year}), https://doi.org/{item["DOI"]}.'
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
toadd = input("Enter space separated numbers to add, or return to quit. ")
|
|
199
|
+
|
|
200
|
+
if toadd:
|
|
201
|
+
toadd = [int(x) for x in toadd.split(" ")]
|
|
202
|
+
dois = [
|
|
203
|
+
"https://doi.org/" + data["message"]["items"][i]["DOI"] for i in toadd
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
with click.Context(add) as ctx:
|
|
207
|
+
ctx.invoke(
|
|
208
|
+
add,
|
|
209
|
+
sources=dois,
|
|
210
|
+
related=related,
|
|
211
|
+
references=references,
|
|
212
|
+
citing=citing,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@cli.command()
|
|
217
|
+
@click.argument("sources", nargs=-1)
|
|
218
|
+
def index(sources):
|
|
219
|
+
"""Index the directories in SOURCES.
|
|
220
|
+
|
|
221
|
+
SOURCES is a list of directories.
|
|
222
|
+
"""
|
|
223
|
+
for directory in sources:
|
|
224
|
+
directory = pathlib.Path(directory).resolve()
|
|
225
|
+
for fname in directory.rglob("*"):
|
|
226
|
+
# for f in files:
|
|
227
|
+
if fname.suffix in [
|
|
228
|
+
".pdf",
|
|
229
|
+
".docx",
|
|
230
|
+
".pptx",
|
|
231
|
+
".org",
|
|
232
|
+
".md",
|
|
233
|
+
".html",
|
|
234
|
+
".bib",
|
|
235
|
+
".ipynb",
|
|
236
|
+
]:
|
|
237
|
+
fname = str(fname)
|
|
238
|
+
|
|
239
|
+
# skip files we already have
|
|
240
|
+
if db.execute(
|
|
241
|
+
"""select source from sources where source = ?""", (fname,)
|
|
242
|
+
).fetchone():
|
|
243
|
+
continue
|
|
244
|
+
|
|
245
|
+
with click.Context(add) as ctx:
|
|
246
|
+
print(fname)
|
|
247
|
+
ctx.invoke(add, sources=[fname])
|
|
248
|
+
print(f"Adding {fname}")
|
|
249
|
+
|
|
250
|
+
richprint(f"Added {fname}")
|
|
251
|
+
|
|
252
|
+
last_updated = datetime.date.today().strftime("%Y-%m-%d")
|
|
253
|
+
|
|
254
|
+
directory = str(directory) # we need strings for the db
|
|
255
|
+
if db.execute(
|
|
256
|
+
"""select path from directories where path = ?""", (directory,)
|
|
257
|
+
).fetchone():
|
|
258
|
+
print(f"Updating {directory}")
|
|
259
|
+
db.execute(
|
|
260
|
+
"""update directories set last_updated = ?
|
|
261
|
+
where path = ?""",
|
|
262
|
+
(last_updated, directory),
|
|
263
|
+
)
|
|
264
|
+
else:
|
|
265
|
+
print(f"Inserting {directory}: {last_updated}")
|
|
266
|
+
db.execute(
|
|
267
|
+
"""insert into directories(path, last_updated)
|
|
268
|
+
values (?, ?)""",
|
|
269
|
+
(directory, last_updated),
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
db.commit()
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@cli.command()
|
|
276
|
+
def reindex():
|
|
277
|
+
"""Reindex saved directories."""
|
|
278
|
+
for (directory,) in db.execute("""select path from directories""").fetchall():
|
|
279
|
+
print(f"Reindexing {directory}")
|
|
280
|
+
index([directory])
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
###########
|
|
284
|
+
# Tagging #
|
|
285
|
+
###########
|
|
286
|
+
@cli.command()
|
|
287
|
+
@click.argument("sources", nargs=-1)
|
|
288
|
+
@click.option("-t", "--tag", "tags", multiple=True)
|
|
289
|
+
def add_tag(sources, tags):
|
|
290
|
+
"""Add tags to sources.
|
|
291
|
+
|
|
292
|
+
It is a little annoying to add multiple tags. It looks like this.
|
|
293
|
+
litdb add-tag source -t tag1 -t tag2
|
|
294
|
+
"""
|
|
295
|
+
for source in sources:
|
|
296
|
+
# Get source id
|
|
297
|
+
(source_id,) = db.execute(
|
|
298
|
+
"select rowid from sources where source = ?", (source,)
|
|
299
|
+
).fetchone()
|
|
300
|
+
|
|
301
|
+
for tag in tags:
|
|
302
|
+
# get tag id
|
|
303
|
+
tag_id = db.execute(
|
|
304
|
+
"select rowid from tags where tag = ?", (tag,)
|
|
305
|
+
).fetchone()
|
|
306
|
+
|
|
307
|
+
if not tag_id:
|
|
308
|
+
c = db.execute("insert into tags(tag) values (?)", (tag,))
|
|
309
|
+
tag_id = c.lastrowid
|
|
310
|
+
db.commit()
|
|
311
|
+
else:
|
|
312
|
+
# we get a tuple in the first query
|
|
313
|
+
(tag_id,) = tag_id
|
|
314
|
+
|
|
315
|
+
# Now add a tag
|
|
316
|
+
db.execute(
|
|
317
|
+
"insert into source_tag(source_id, tag_id) values (?, ?)",
|
|
318
|
+
(source_id, tag_id),
|
|
319
|
+
)
|
|
320
|
+
db.commit()
|
|
321
|
+
|
|
322
|
+
print(f"Tagged {source} with {tag}")
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@cli.command()
|
|
326
|
+
@click.argument("sources", nargs=-1)
|
|
327
|
+
@click.option("-t", "--tag", "tags", multiple=True)
|
|
328
|
+
def rm_tag(sources, tags):
|
|
329
|
+
"""Remove tags from sources.
|
|
330
|
+
|
|
331
|
+
It is a little annoying to remove multiple tags. It looks like this.
|
|
332
|
+
litdb rm-tag source -t tag1 -t tag2
|
|
333
|
+
"""
|
|
334
|
+
for source in sources:
|
|
335
|
+
# Get source id
|
|
336
|
+
(source_id,) = db.execute(
|
|
337
|
+
"select rowid from sources where source = ?", (source,)
|
|
338
|
+
).fetchone()
|
|
339
|
+
|
|
340
|
+
for tag in tags:
|
|
341
|
+
# get tag id. Assume it exists?
|
|
342
|
+
(tag_id,) = db.execute(
|
|
343
|
+
"select rowid from tags where tag = ?", (tag,)
|
|
344
|
+
).fetchone()
|
|
345
|
+
|
|
346
|
+
c = db.execute(
|
|
347
|
+
"""delete from source_tag
|
|
348
|
+
where source_id = ? and tag_id = ?""",
|
|
349
|
+
(source_id, tag_id),
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
db.commit()
|
|
353
|
+
print(f"Deleted {c.rowcount} rows ({tag} from {source}")
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
@cli.command()
|
|
357
|
+
@click.argument("tags", nargs=-1)
|
|
358
|
+
def delete_tag(tags):
|
|
359
|
+
"""Delete each tag.
|
|
360
|
+
|
|
361
|
+
This should also delete tags from sources by cascade.
|
|
362
|
+
"""
|
|
363
|
+
for tag in tags:
|
|
364
|
+
c = db.execute("delete from tags where tag = ?", (tag,))
|
|
365
|
+
print(f"Deleted {c.rowcount} rows ({tag})")
|
|
366
|
+
db.commit()
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@cli.command()
|
|
370
|
+
@click.argument("tags", nargs=-1)
|
|
371
|
+
@click.option("-f", "--fmt", default='{{ source }}\n{{ extra["citation"] }}')
|
|
372
|
+
def show_tag(tags, fmt):
|
|
373
|
+
"""Show entries with tags.
|
|
374
|
+
|
|
375
|
+
FMT is a jinja template for the output. You have variables of source, text
|
|
376
|
+
and extra.
|
|
377
|
+
|
|
378
|
+
I don't have good logic here, we just show all entries. I could probably get
|
|
379
|
+
some basic and logic with sets, but mostly I assume for now you only want
|
|
380
|
+
one tag, so this works. TODO: add something like boolean logic?
|
|
381
|
+
|
|
382
|
+
"""
|
|
383
|
+
template = Template(fmt)
|
|
384
|
+
for tag in tags:
|
|
385
|
+
for row in db.execute(
|
|
386
|
+
"""select
|
|
387
|
+
sources.source, sources.text, sources.extra
|
|
388
|
+
from sources
|
|
389
|
+
inner join source_tag on source_tag.source_id = sources.rowid
|
|
390
|
+
inner join tags on source_tag.tag_id = tags.rowid
|
|
391
|
+
where tags.tag = ?""",
|
|
392
|
+
(tag,),
|
|
393
|
+
).fetchall():
|
|
394
|
+
source, text, extra = row
|
|
395
|
+
extra = json.loads(extra)
|
|
396
|
+
richprint(template.render(**locals()))
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
@cli.command()
|
|
400
|
+
def list_tags():
|
|
401
|
+
"""Print defined tags."""
|
|
402
|
+
print("The following tags are defined.")
|
|
403
|
+
for (tag,) in db.execute("select tag from tags").fetchall():
|
|
404
|
+
print(tag)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
##########
|
|
408
|
+
# Review #
|
|
409
|
+
##########
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
@cli.command()
|
|
413
|
+
@click.option("-s", "--since", default="1 week ago")
|
|
414
|
+
@click.option("-f", "--fmt", default=None)
|
|
415
|
+
def review(since, fmt):
|
|
416
|
+
"""Review new entries added SINCE.
|
|
417
|
+
|
|
418
|
+
SINCE should be something dateparser can handle.
|
|
419
|
+
FMT is a jinja template for the output. Defaults to an org-mode template.
|
|
420
|
+
"""
|
|
421
|
+
|
|
422
|
+
since = dateparser.parse(since).strftime("%Y-%m-%d")
|
|
423
|
+
c = db.execute(
|
|
424
|
+
"""select source, text, extra from sources
|
|
425
|
+
where date(date_added) > ?""",
|
|
426
|
+
(since,),
|
|
427
|
+
).fetchall()
|
|
428
|
+
|
|
429
|
+
template = Template(
|
|
430
|
+
fmt
|
|
431
|
+
or """* {{ extra['display_name'] | replace("\n", " ") }}
|
|
432
|
+
:PROPERTIES:
|
|
433
|
+
:SOURCE: {{ source }}
|
|
434
|
+
:OPENALEX: {{ extra.get('id') }}
|
|
435
|
+
:YEAR: {{ extra.get('publication_year') }}
|
|
436
|
+
:REFERENCE_COUNT: {{ extra.get('referenced_works_count', 0) }}
|
|
437
|
+
:CITED_BY_COUNT: {{ extra.get('cited_by_count', 0) }}
|
|
438
|
+
:END:
|
|
439
|
+
|
|
440
|
+
{{ text }} litdb:{{ source }}
|
|
441
|
+
"""
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
for source, text, extra in c:
|
|
445
|
+
extra = json.loads(extra) or {}
|
|
446
|
+
print(template.render(**locals()))
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
#############
|
|
450
|
+
# Searching #
|
|
451
|
+
#############
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
@cli.command()
|
|
455
|
+
def screenshot():
|
|
456
|
+
"""Do vector search from text in a screenshot.
|
|
457
|
+
|
|
458
|
+
Use OCR to get text from an image on the clipboard (probably from a
|
|
459
|
+
screenshot) and do a vector search on the text.
|
|
460
|
+
"""
|
|
461
|
+
from PIL import ImageGrab
|
|
462
|
+
import pytesseract
|
|
463
|
+
|
|
464
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
465
|
+
|
|
466
|
+
# Grab the image from the clipboard
|
|
467
|
+
img = ImageGrab.grabclipboard()
|
|
468
|
+
|
|
469
|
+
if img:
|
|
470
|
+
text = pytesseract.image_to_string(img)
|
|
471
|
+
print(f'Searching for "{text}"')
|
|
472
|
+
vsearch(text)
|
|
473
|
+
else:
|
|
474
|
+
print("No image found in clipboard.")
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def record():
|
|
478
|
+
"""Make a recording to a tempfile.
|
|
479
|
+
|
|
480
|
+
Returns the audio filename it is recorded in.
|
|
481
|
+
"""
|
|
482
|
+
import pyaudio
|
|
483
|
+
import wave
|
|
484
|
+
import threading
|
|
485
|
+
|
|
486
|
+
# Parameters for recording
|
|
487
|
+
FORMAT = pyaudio.paInt16
|
|
488
|
+
CHANNELS = 2
|
|
489
|
+
RATE = 44100
|
|
490
|
+
CHUNK = 1024
|
|
491
|
+
|
|
492
|
+
_, audio_file = tempfile.mkstemp(suffix=".wav")
|
|
493
|
+
|
|
494
|
+
# Initialize PyAudio
|
|
495
|
+
audio = pyaudio.PyAudio()
|
|
496
|
+
|
|
497
|
+
# Function to record audio
|
|
498
|
+
def record_audio():
|
|
499
|
+
# Open stream
|
|
500
|
+
stream = audio.open(
|
|
501
|
+
format=FORMAT,
|
|
502
|
+
channels=CHANNELS,
|
|
503
|
+
rate=RATE,
|
|
504
|
+
input=True,
|
|
505
|
+
frames_per_buffer=CHUNK,
|
|
506
|
+
)
|
|
507
|
+
print("Recording... Press Enter to stop.")
|
|
508
|
+
|
|
509
|
+
frames = []
|
|
510
|
+
|
|
511
|
+
# Record until Enter is pressed
|
|
512
|
+
while not stop_recording.is_set():
|
|
513
|
+
data = stream.read(CHUNK)
|
|
514
|
+
frames.append(data)
|
|
515
|
+
|
|
516
|
+
# Stop and close the stream
|
|
517
|
+
stream.stop_stream()
|
|
518
|
+
stream.close()
|
|
519
|
+
|
|
520
|
+
# Save the recorded data as a WAV file
|
|
521
|
+
wf = wave.open(audio_file, "wb")
|
|
522
|
+
wf.setnchannels(CHANNELS)
|
|
523
|
+
wf.setsampwidth(audio.get_sample_size(FORMAT))
|
|
524
|
+
wf.setframerate(RATE)
|
|
525
|
+
wf.writeframes(b"".join(frames))
|
|
526
|
+
wf.close()
|
|
527
|
+
|
|
528
|
+
# Event to signal when to stop recording
|
|
529
|
+
stop_recording = threading.Event()
|
|
530
|
+
|
|
531
|
+
# Start recording in a separate thread
|
|
532
|
+
recording_thread = threading.Thread(target=record_audio)
|
|
533
|
+
recording_thread.start()
|
|
534
|
+
|
|
535
|
+
# Wait for Enter key press to stop recording
|
|
536
|
+
input()
|
|
537
|
+
stop_recording.set()
|
|
538
|
+
|
|
539
|
+
# Wait for the recording thread to finish
|
|
540
|
+
recording_thread.join()
|
|
541
|
+
|
|
542
|
+
# Terminate PyAudio
|
|
543
|
+
audio.terminate()
|
|
544
|
+
|
|
545
|
+
return audio_file
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
@cli.command()
|
|
549
|
+
@click.option("-p", "--playback", is_flag=True, help="Play audio back")
|
|
550
|
+
def audio(playback=False):
|
|
551
|
+
"""Record audio, convert it to text, and do a vector search on the text.
|
|
552
|
+
|
|
553
|
+
The idea is nice, but the quality of transcription for scientific words is
|
|
554
|
+
not that great. A better transcription library might make this more useful.
|
|
555
|
+
"""
|
|
556
|
+
import speech_recognition as sr
|
|
557
|
+
|
|
558
|
+
while True:
|
|
559
|
+
afile = record()
|
|
560
|
+
audio_file = sr.AudioFile(afile)
|
|
561
|
+
r = sr.Recognizer()
|
|
562
|
+
with audio_file as source:
|
|
563
|
+
audio = r.listen(source)
|
|
564
|
+
try:
|
|
565
|
+
text = r.recognize_sphinx(audio)
|
|
566
|
+
print("\n" + text + "\n")
|
|
567
|
+
except sr.UnknownValueError:
|
|
568
|
+
print("Could not understand audio.")
|
|
569
|
+
|
|
570
|
+
if playback:
|
|
571
|
+
import playsound
|
|
572
|
+
|
|
573
|
+
playsound.playsound(afile, block=True)
|
|
574
|
+
|
|
575
|
+
response = input("Is that what you want to search? ([y]/n/q): ")
|
|
576
|
+
if response.lower().startswith("q"):
|
|
577
|
+
return
|
|
578
|
+
elif response.lower().startswith("n"):
|
|
579
|
+
# record a new audio
|
|
580
|
+
continue
|
|
581
|
+
else:
|
|
582
|
+
# move on to searching
|
|
583
|
+
break
|
|
584
|
+
|
|
585
|
+
vsearch(text)
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
@cli.command()
|
|
589
|
+
@click.argument("query", nargs=-1)
|
|
590
|
+
@click.option("-n", default=3)
|
|
591
|
+
@click.option("-e", "--emacs", is_flag=True, default=False)
|
|
592
|
+
@click.option("-i", "--iterative", is_flag=True, default=False)
|
|
593
|
+
@click.option("-m", "--max-steps", default=None)
|
|
594
|
+
@click.option("-f", "--fmt", default=" {{ i }}. ({{ score|round(3) }}) {{ text }}\n\n")
|
|
595
|
+
@click.option("-x", "--cross-encode", is_flag=True, default=False)
|
|
596
|
+
def vsearch(query, n, emacs, fmt, cross_encode, iterative, max_steps):
|
|
597
|
+
"""Do a vector search on QUERY.
|
|
598
|
+
|
|
599
|
+
N is an integer for number of results to return
|
|
600
|
+
EMACS is a flag for changing the output format for emacs
|
|
601
|
+
|
|
602
|
+
FORMAT is a jinja template for the output. The variables you have access to
|
|
603
|
+
are i, source, text, extra, similarity.
|
|
604
|
+
|
|
605
|
+
CROSS_ENCODE is a boolean that resorts the results with a cross-encoder.
|
|
606
|
+
|
|
607
|
+
ITERATIVE is a boolean that expands the search by references, citations and
|
|
608
|
+
related articles from the top matches until you tell it to stop or reach
|
|
609
|
+
MAX_STEPS.
|
|
610
|
+
|
|
611
|
+
MAX_STEPS is the maximum number of iterations to take. If you set this, you
|
|
612
|
+
will not be prompted each time, it will just run those steps until nothing
|
|
613
|
+
better is found, or you reach the number.
|
|
614
|
+
|
|
615
|
+
"""
|
|
616
|
+
config = get_config()
|
|
617
|
+
query = " ".join(query)
|
|
618
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
619
|
+
emb = model.encode([query]).astype(np.float32).tobytes()
|
|
620
|
+
|
|
621
|
+
if iterative:
|
|
622
|
+
best = None
|
|
623
|
+
|
|
624
|
+
steps = 0
|
|
625
|
+
|
|
626
|
+
while True:
|
|
627
|
+
results = db.execute(
|
|
628
|
+
"""select
|
|
629
|
+
sources.source, sources.text,
|
|
630
|
+
sources.extra, vector_distance_cos(?, embedding) as d
|
|
631
|
+
from vector_top_k('embedding_idx', ?, ?)
|
|
632
|
+
join sources on sources.rowid = id
|
|
633
|
+
order by d""",
|
|
634
|
+
(emb, emb, n),
|
|
635
|
+
).fetchall()
|
|
636
|
+
|
|
637
|
+
for source, text, extra, d in results:
|
|
638
|
+
richprint(f"{d:1.3f}: {source}")
|
|
639
|
+
|
|
640
|
+
steps += 1
|
|
641
|
+
if steps == max_steps:
|
|
642
|
+
break
|
|
643
|
+
|
|
644
|
+
current = [x[0] for x in results] # sources
|
|
645
|
+
|
|
646
|
+
# This means no change
|
|
647
|
+
if current == best:
|
|
648
|
+
print("Nothing new was found")
|
|
649
|
+
break
|
|
650
|
+
|
|
651
|
+
if not max_steps and input(
|
|
652
|
+
"Search for better matches? ([y]/n)"
|
|
653
|
+
).lower().startswith("n"):
|
|
654
|
+
break
|
|
655
|
+
|
|
656
|
+
# something changed. add references and loop
|
|
657
|
+
best = current
|
|
658
|
+
for source in current:
|
|
659
|
+
add_work(source, True, True, True)
|
|
660
|
+
|
|
661
|
+
else:
|
|
662
|
+
c = db.execute(
|
|
663
|
+
"""select sources.source, sources.text,
|
|
664
|
+
sources.extra, vector_distance_cos(?, embedding)
|
|
665
|
+
from vector_top_k('embedding_idx', ?, ?)
|
|
666
|
+
join sources on sources.rowid = id""",
|
|
667
|
+
(emb, emb, n),
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
results = c.fetchall()
|
|
671
|
+
|
|
672
|
+
if cross_encode:
|
|
673
|
+
import torch
|
|
674
|
+
from sentence_transformers.cross_encoder import CrossEncoder
|
|
675
|
+
|
|
676
|
+
# I don't know why I have to set the activation function here, but the
|
|
677
|
+
# score is not 0..1 otherwise
|
|
678
|
+
ce = CrossEncoder(
|
|
679
|
+
config["embedding"]["cross-encoder"],
|
|
680
|
+
default_activation_function=torch.nn.Sigmoid(),
|
|
681
|
+
)
|
|
682
|
+
scores = ce.predict([[query, text] for _, text, _, _ in results])
|
|
683
|
+
# resort based on the scores
|
|
684
|
+
results = [results[i] for i in np.argsort(scores)]
|
|
685
|
+
|
|
686
|
+
if emacs:
|
|
687
|
+
tmpl = (
|
|
688
|
+
"( {% for source, text, extra, score in results %}"
|
|
689
|
+
'("({{ score|round(3) }}) {{ text }}" . "{{ source }}") '
|
|
690
|
+
"{% endfor %})"
|
|
691
|
+
)
|
|
692
|
+
template = Template(tmpl)
|
|
693
|
+
print(template.render(**locals()))
|
|
694
|
+
else:
|
|
695
|
+
for i, row in enumerate(results, 1):
|
|
696
|
+
source, text, extra, score = row
|
|
697
|
+
template = Template(fmt)
|
|
698
|
+
richprint(template.render(**locals()))
|
|
699
|
+
|
|
700
|
+
return results
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
@cli.command()
|
|
704
|
+
@click.argument("query", nargs=-1)
|
|
705
|
+
@click.option("-n", default=3)
|
|
706
|
+
@click.option(
|
|
707
|
+
"-f", "--fmt", default="{{ source }} ({{ score | round(3) }})\n{{ snippet }}"
|
|
708
|
+
)
|
|
709
|
+
def fulltext(query, n, fmt):
|
|
710
|
+
"""Perform a fulltext search on litdb."""
|
|
711
|
+
query = " ".join(query)
|
|
712
|
+
|
|
713
|
+
results = db.execute(
|
|
714
|
+
"""select
|
|
715
|
+
sources.source, sources.text, snippet(fulltext, 1, '', '', '', 16), sources.extra, bm25(fulltext)
|
|
716
|
+
from fulltext
|
|
717
|
+
inner join sources on fulltext.source = sources.source
|
|
718
|
+
where fulltext.text match ? order by rank limit ?""",
|
|
719
|
+
(query, n),
|
|
720
|
+
).fetchall()
|
|
721
|
+
|
|
722
|
+
for source, text, snippet, extra, score in results:
|
|
723
|
+
richprint(Template(fmt).render(**locals()))
|
|
724
|
+
|
|
725
|
+
return results
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
gpt = cli.command(gpt)
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
@cli.command()
|
|
732
|
+
@click.argument("source")
|
|
733
|
+
@click.option("-f", "--fmt", default=None)
|
|
734
|
+
@click.option("-e", "--emacs", is_flag=True, default=False)
|
|
735
|
+
@click.option("-n", default=3)
|
|
736
|
+
def similar(source, n, emacs, fmt):
|
|
737
|
+
"""Find N sources similar to SOURCE by vector similarity.
|
|
738
|
+
|
|
739
|
+
if EMACS is truthy, the output is lisp for Emacs to read.
|
|
740
|
+
FMT is a jinja template with access to source, text, and extra
|
|
741
|
+
"""
|
|
742
|
+
(emb,) = db.execute(
|
|
743
|
+
"""select embedding from sources where source = ?""", (source,)
|
|
744
|
+
).fetchone()
|
|
745
|
+
|
|
746
|
+
allrows = db.execute(
|
|
747
|
+
"""select sources.source, sources.text, sources.extra
|
|
748
|
+
from vector_top_k('embedding_idx', ?, ?)
|
|
749
|
+
join sources on sources.rowid = id""",
|
|
750
|
+
# we do n + 1 because the first entry is always the source
|
|
751
|
+
(emb, n + 1),
|
|
752
|
+
).fetchall()
|
|
753
|
+
|
|
754
|
+
rows = [(source, text, json.loads(extra)) for source, text, extra in allrows[1:]]
|
|
755
|
+
|
|
756
|
+
if emacs:
|
|
757
|
+
template = Template(
|
|
758
|
+
"({% for source, text, extra in rows %}"
|
|
759
|
+
' ("{{ extra.get("citation") or text }}" . "{{ source }}")'
|
|
760
|
+
" {% endfor %})"
|
|
761
|
+
)
|
|
762
|
+
print(template.render(**locals()))
|
|
763
|
+
else:
|
|
764
|
+
template = Template(fmt or "{{ i }}. {{ source }}\n {{text}}\n\n")
|
|
765
|
+
# print starting at index 1, the first item is always the source.
|
|
766
|
+
for i, row in enumerate(rows, 1):
|
|
767
|
+
source, text, extra = row
|
|
768
|
+
richprint(template.render(**locals()))
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
@cli.command()
|
|
772
|
+
@click.argument("vector_query")
|
|
773
|
+
@click.argument("text_query")
|
|
774
|
+
@click.option("-n", default=5)
|
|
775
|
+
@click.option(
|
|
776
|
+
"-f", "--fmt", default="{{ source }} ({{ score | round(3) }})\n{{ text }}\n\n"
|
|
777
|
+
)
|
|
778
|
+
def hybrid_search(vector_query, text_query, n, fmt):
|
|
779
|
+
"""Perform a hybrid vector and full text search.
|
|
780
|
+
|
|
781
|
+
VECTOR_QUERY: The query to do vector search on
|
|
782
|
+
TEXT_QUERY: The query for full text search
|
|
783
|
+
N is an integer number of documents to return.
|
|
784
|
+
FMT is a jinja template.
|
|
785
|
+
"""
|
|
786
|
+
# Get vector results and score
|
|
787
|
+
with click.Context(vsearch) as ctx:
|
|
788
|
+
# source, text, extra, score
|
|
789
|
+
vresults = ctx.invoke(vsearch, query=vector_query.split(" "), n=n, fmt="")
|
|
790
|
+
vscores = [(result[0], result[3]) for result in vresults]
|
|
791
|
+
|
|
792
|
+
# Get text results and score
|
|
793
|
+
with click.Context(fulltext) as ctx:
|
|
794
|
+
tresults = ctx.invoke(fulltext, query=text_query.split(" "), n=n, fmt="")
|
|
795
|
+
# I think sqlite makes scores negative to sort them the way they want. I
|
|
796
|
+
# reverse this here.
|
|
797
|
+
tscores = [(result[0], -result[-1]) for result in tresults]
|
|
798
|
+
|
|
799
|
+
# Normalize scores
|
|
800
|
+
minv, maxv = min([x[1] for x in vscores]), max([x[1] for x in vscores])
|
|
801
|
+
mint, maxt = min([x[1] for x in tscores]), max([x[1] for x in tscores])
|
|
802
|
+
|
|
803
|
+
vscores = {oaid: (score - minv) / (maxv - minv) for oaid, score in vscores}
|
|
804
|
+
tscores = {oaid: (score - minv) / (maxv - minv) for oaid, score in tscores}
|
|
805
|
+
|
|
806
|
+
combined_scores = {}
|
|
807
|
+
for oaid in set(vscores.keys()).union(tscores.keys()):
|
|
808
|
+
vscore = vscores.get(oaid, 0)
|
|
809
|
+
tscore = tscores.get(oaid, 0)
|
|
810
|
+
cscore = 1 / (1 + vscore) + 1 / (1 + tscore)
|
|
811
|
+
combined_scores[oaid] = cscore
|
|
812
|
+
|
|
813
|
+
sorted_results = sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)
|
|
814
|
+
results = []
|
|
815
|
+
for oaid, score in sorted_results:
|
|
816
|
+
c = db.execute(
|
|
817
|
+
"select source, text, extra from sources where source = ?", (oaid,)
|
|
818
|
+
)
|
|
819
|
+
row = c.fetchone()
|
|
820
|
+
results += [[*row, score]]
|
|
821
|
+
|
|
822
|
+
for row in results:
|
|
823
|
+
source, text, extra, score = row
|
|
824
|
+
richprint(Template(fmt).render(**locals()))
|
|
825
|
+
|
|
826
|
+
return results
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
@cli.command()
|
|
830
|
+
@click.argument("query", nargs=-1)
|
|
831
|
+
@click.option("-g", "--google", is_flag=True)
|
|
832
|
+
@click.option("-gs", "--google-scholar", is_flag=True)
|
|
833
|
+
@click.option("-pm", "--pubmed", is_flag=True)
|
|
834
|
+
@click.option("-ar", "--arxiv", is_flag=True)
|
|
835
|
+
@click.option("-cr", "--chemrxiv", is_flag=True)
|
|
836
|
+
@click.option("-br", "--biorxiv", is_flag=True)
|
|
837
|
+
@click.option("-a", "--all", is_flag=True)
|
|
838
|
+
def web(query, google, google_scholar, pubmed, arxiv, chemrxiv, biorxiv, all):
|
|
839
|
+
"""Open a web search for QUERY.
|
|
840
|
+
|
|
841
|
+
We always do an OpenAlex search.
|
|
842
|
+
|
|
843
|
+
If these are true, we also search here
|
|
844
|
+
GOOGLE
|
|
845
|
+
GOOGLE_SCHOLAR
|
|
846
|
+
PUBMED
|
|
847
|
+
ARXIV
|
|
848
|
+
CHEMRXIV
|
|
849
|
+
BIORXIV
|
|
850
|
+
"""
|
|
851
|
+
query = " ".join(query)
|
|
852
|
+
|
|
853
|
+
if all:
|
|
854
|
+
google, google_scholar = (
|
|
855
|
+
True,
|
|
856
|
+
True,
|
|
857
|
+
)
|
|
858
|
+
pubmed, arxiv, chemrxiv, biorxiv = True, True, True, True
|
|
859
|
+
|
|
860
|
+
# This is to avoid some huggingface/tokenizer warning. I don't know why we
|
|
861
|
+
# need to do it, but this code forks the process, and triggers that warning.
|
|
862
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
863
|
+
|
|
864
|
+
oa = f"https://openalex.org/works?filter=default.search:{query}"
|
|
865
|
+
webbrowser.open(oa)
|
|
866
|
+
|
|
867
|
+
if google:
|
|
868
|
+
url = f"https://www.google.com/search?q={query}"
|
|
869
|
+
webbrowser.open(url)
|
|
870
|
+
|
|
871
|
+
if google_scholar:
|
|
872
|
+
url = f"https://scholar.google.com/scholar?q={query}"
|
|
873
|
+
webbrowser.open(url)
|
|
874
|
+
|
|
875
|
+
if pubmed:
|
|
876
|
+
url = f"https://pubmed-ncbi-nlm-nih-gov.cmu.idm.oclc.org/?term={query}"
|
|
877
|
+
webbrowser.open(url)
|
|
878
|
+
|
|
879
|
+
if arxiv:
|
|
880
|
+
url = f"https://arxiv.org/search/?query={query}"
|
|
881
|
+
webbrowser.open(url)
|
|
882
|
+
|
|
883
|
+
if chemrxiv:
|
|
884
|
+
url = f"https://chemrxiv.org/engage/chemrxiv/search-dashboard?text={query}"
|
|
885
|
+
webbrowser.open(url)
|
|
886
|
+
|
|
887
|
+
if biorxiv:
|
|
888
|
+
url = f"https://www.biorxiv.org/search/{query}"
|
|
889
|
+
webbrowser.open(url)
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
###########
|
|
893
|
+
# Filters #
|
|
894
|
+
###########
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
@cli.command()
|
|
898
|
+
@click.argument("_filter")
|
|
899
|
+
@click.option("-d", "--description")
|
|
900
|
+
def add_filter(_filter, description=None):
|
|
901
|
+
"""Add an OpenAlex FILTER.
|
|
902
|
+
|
|
903
|
+
This does not run the filter right away. You need
|
|
904
|
+
to manually update the filters later.
|
|
905
|
+
"""
|
|
906
|
+
db.execute(
|
|
907
|
+
"insert into queries(filter, description) values (?, ?)", (_filter, description)
|
|
908
|
+
)
|
|
909
|
+
db.commit()
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
@cli.command()
|
|
913
|
+
@click.argument("_filter")
|
|
914
|
+
def rm_filter(_filter):
|
|
915
|
+
"""Remove an OpenAlex FILTER."""
|
|
916
|
+
db.execute("delete from queries where filter = ?", (_filter,))
|
|
917
|
+
db.commit()
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
update_filter_fmt = """** {{ extra['display_name'] | replace("\n", "") | replace("\r", "") }}
|
|
921
|
+
:PROPERTIES:
|
|
922
|
+
:SOURCE: {{ source }}
|
|
923
|
+
:REFERENCE_COUNT: {{ extra.get('referenced_works_count', 0) }}
|
|
924
|
+
:CITED_BY_COUNT: {{ extra.get('cited_by_count', 0) }}
|
|
925
|
+
:END:
|
|
926
|
+
|
|
927
|
+
litdb:{{ source }}
|
|
928
|
+
|
|
929
|
+
{{ text }}
|
|
930
|
+
|
|
931
|
+
"""
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
@cli.command()
|
|
935
|
+
@click.option("-f", "--fmt", default=update_filter_fmt)
|
|
936
|
+
@click.option("-s", "--silent", is_flag=True, default=False)
|
|
937
|
+
def update_filters(fmt, silent):
|
|
938
|
+
"""Update litdb using a filter with works from a created date."""
|
|
939
|
+
filters = db.execute("""select filter, description, last_updated from queries""")
|
|
940
|
+
for f, description, last_updated in filters.fetchall():
|
|
941
|
+
results = update_filter(f, last_updated, silent)
|
|
942
|
+
if results:
|
|
943
|
+
richprint(f"* {description}")
|
|
944
|
+
for result in results:
|
|
945
|
+
source, text, extra = result
|
|
946
|
+
richprint(Template(fmt).render(**locals()))
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
list_filter_fmt = (
|
|
950
|
+
'{{ "{:3d}".format(rowid) }}.'
|
|
951
|
+
' {{ "{:30s}".format(description'
|
|
952
|
+
' or "None") }} {{ f }}'
|
|
953
|
+
" ({{ last_updated }})"
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
@cli.command()
|
|
958
|
+
@click.option("-f", "--fmt", default=list_filter_fmt)
|
|
959
|
+
def list_filters(fmt):
|
|
960
|
+
"""List the filters.
|
|
961
|
+
|
|
962
|
+
FMT is a jinja template with access to the variables rowid, f, description
|
|
963
|
+
and last_updated. f is the filter string.
|
|
964
|
+
|
|
965
|
+
You can dump the filters to stdout like this.
|
|
966
|
+
|
|
967
|
+
> litdb list-filters -f 'litdb add-filter {{ f }}'
|
|
968
|
+
|
|
969
|
+
You could use that to send a list of your filters to someone, or to recreate
|
|
970
|
+
a db somewhere else.
|
|
971
|
+
"""
|
|
972
|
+
filters = db.execute(
|
|
973
|
+
"""select rowid, filter, description, last_updated
|
|
974
|
+
from queries"""
|
|
975
|
+
)
|
|
976
|
+
for rowid, f, description, last_updated in filters.fetchall():
|
|
977
|
+
richprint(Template(fmt).render(**locals()))
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
######################
|
|
981
|
+
# OpenAlex searching #
|
|
982
|
+
######################
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
@cli.command()
|
|
986
|
+
@click.argument("query", nargs=-1)
|
|
987
|
+
@click.option("-f", "--filter", "_filter", is_flag=True, default=False)
|
|
988
|
+
@click.option("-e", "--endpoint", default="works")
|
|
989
|
+
@click.option("--sample", default=-1)
|
|
990
|
+
@click.option("--per-page", default=5)
|
|
991
|
+
def openalex(query, _filter, endpoint, sample, per_page):
|
|
992
|
+
"""Run an openalex query on FILTER.
|
|
993
|
+
|
|
994
|
+
ENDPOINT should be one of works, authors, or another entity.
|
|
995
|
+
|
|
996
|
+
This does not add anything to your database. It is to help you find starting
|
|
997
|
+
points.
|
|
998
|
+
|
|
999
|
+
To search text:
|
|
1000
|
+
litdb openalex "circular polymer"
|
|
1001
|
+
|
|
1002
|
+
To find a journal id with a specific filter
|
|
1003
|
+
litdb openalex -e sources -f "display_name.search:Digital Discovery"
|
|
1004
|
+
|
|
1005
|
+
"""
|
|
1006
|
+
config = get_config()
|
|
1007
|
+
url = f"https://api.openalex.org/{endpoint}"
|
|
1008
|
+
|
|
1009
|
+
if isinstance(query, tuple):
|
|
1010
|
+
query = " ".join(query)
|
|
1011
|
+
if not _filter:
|
|
1012
|
+
query = f"default.search:{query}"
|
|
1013
|
+
|
|
1014
|
+
print(query)
|
|
1015
|
+
|
|
1016
|
+
next_cursor = "*"
|
|
1017
|
+
params = {
|
|
1018
|
+
"email": config["openalex"]["email"],
|
|
1019
|
+
"filter": query,
|
|
1020
|
+
"cursor": next_cursor,
|
|
1021
|
+
"per_page": per_page,
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
if api_key := config["openalex"].get("api_key"):
|
|
1025
|
+
params.update(api_key=api_key)
|
|
1026
|
+
|
|
1027
|
+
if sample > 0:
|
|
1028
|
+
del params["sort"] # incompatible param with sample
|
|
1029
|
+
params.update(sample=sample)
|
|
1030
|
+
|
|
1031
|
+
while next_cursor:
|
|
1032
|
+
resp = requests.get(url, params)
|
|
1033
|
+
if resp.status_code != 200:
|
|
1034
|
+
print(resp.url)
|
|
1035
|
+
print(resp.text)
|
|
1036
|
+
return
|
|
1037
|
+
|
|
1038
|
+
data = resp.json()
|
|
1039
|
+
next_cursor = data["meta"]["next_cursor"]
|
|
1040
|
+
params.update(cursor=next_cursor)
|
|
1041
|
+
|
|
1042
|
+
for result in data["results"]:
|
|
1043
|
+
# Note sometimes there is an exception from bad markup in strings
|
|
1044
|
+
richprint(get_text(result))
|
|
1045
|
+
print()
|
|
1046
|
+
|
|
1047
|
+
if next_cursor:
|
|
1048
|
+
if input("Continue? (y/n)").lower().startswith("n"):
|
|
1049
|
+
return
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
########################################
|
|
1053
|
+
# Convenience functions to add filters #
|
|
1054
|
+
########################################
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
@cli.command()
|
|
1058
|
+
@click.argument("name", nargs=-1)
|
|
1059
|
+
def author_search(name):
|
|
1060
|
+
"""Search OpenAlex for name.
|
|
1061
|
+
|
|
1062
|
+
Uses the autocomplete endpoint to find an author's orcid.
|
|
1063
|
+
"""
|
|
1064
|
+
auname = " ".join(name)
|
|
1065
|
+
|
|
1066
|
+
url = "https://api.openalex.org/autocomplete/authors"
|
|
1067
|
+
|
|
1068
|
+
from .openalex import get_data
|
|
1069
|
+
|
|
1070
|
+
data = get_data(url, params={"q": auname})
|
|
1071
|
+
|
|
1072
|
+
for result in data["results"]:
|
|
1073
|
+
richprint(
|
|
1074
|
+
f'- {result["display_name"]}\n {result["hint"]} '
|
|
1075
|
+
f'{result["external_id"]}\n\n'
|
|
1076
|
+
)
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
@cli.command()
|
|
1080
|
+
@click.argument("orcids", nargs=-1)
|
|
1081
|
+
@click.option("-r", "--remove", is_flag=True, help="remove")
|
|
1082
|
+
def follow(orcids, remove=False):
|
|
1083
|
+
"""Add a filter to follow orcid."""
|
|
1084
|
+
for orcid in orcids:
|
|
1085
|
+
if not orcid.startswith("http"):
|
|
1086
|
+
orcid = f"https://orcid.org/{orcid}"
|
|
1087
|
+
|
|
1088
|
+
# Seems like we should get their articles first.
|
|
1089
|
+
add_author(orcid)
|
|
1090
|
+
|
|
1091
|
+
f = f"author.orcid:{orcid}"
|
|
1092
|
+
|
|
1093
|
+
if remove:
|
|
1094
|
+
c = db.execute("""delete from queries where filter = ?""", (f,))
|
|
1095
|
+
db.commit()
|
|
1096
|
+
richprint(f"{c.rowcount} rows removed")
|
|
1097
|
+
return
|
|
1098
|
+
|
|
1099
|
+
url = f"https://api.openalex.org/authors/{orcid}"
|
|
1100
|
+
data = get_data(url)
|
|
1101
|
+
name = data["display_name"]
|
|
1102
|
+
|
|
1103
|
+
today = datetime.date.today().strftime("%Y-%m-%d")
|
|
1104
|
+
db.execute(
|
|
1105
|
+
"""insert or ignore into
|
|
1106
|
+
queries(filter, description, last_updated)
|
|
1107
|
+
values (?, ?, ?)""",
|
|
1108
|
+
(f, name, today),
|
|
1109
|
+
)
|
|
1110
|
+
|
|
1111
|
+
richprint(f"Following {name}: {orcid}")
|
|
1112
|
+
db.commit()
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
@cli.command()
|
|
1116
|
+
@click.argument("query", nargs=-1)
|
|
1117
|
+
@click.option("-r", "--remove", is_flag=True, help="remove")
|
|
1118
|
+
def watch(query, remove=False):
|
|
1119
|
+
"""Create a watch on query.
|
|
1120
|
+
|
|
1121
|
+
QUERY: string, a filter for openalex.
|
|
1122
|
+
REMOVE: a flag to remove the query.
|
|
1123
|
+
"""
|
|
1124
|
+
# First, we should make sure the filter is valid
|
|
1125
|
+
query = " ".join(query)
|
|
1126
|
+
|
|
1127
|
+
if remove:
|
|
1128
|
+
c = db.execute("""delete from queries where filter = ?""", (query,))
|
|
1129
|
+
db.commit()
|
|
1130
|
+
richprint(f"{c.rowcount} rows removed")
|
|
1131
|
+
return
|
|
1132
|
+
|
|
1133
|
+
url = "https://api.openalex.org/works"
|
|
1134
|
+
|
|
1135
|
+
data = get_data(url, params={"filter": query})
|
|
1136
|
+
if len(data["results"]) == 0:
|
|
1137
|
+
richprint(f"Sorry, {query} does not seem valid.")
|
|
1138
|
+
|
|
1139
|
+
if remove:
|
|
1140
|
+
c = db.execute("""delete from queries where filter = ?""", (query,))
|
|
1141
|
+
richprint(f"Deleted {c.rowcount} rows")
|
|
1142
|
+
db.commit()
|
|
1143
|
+
else:
|
|
1144
|
+
c = db.execute(
|
|
1145
|
+
"""insert or ignore into queries(filter, description)
|
|
1146
|
+
values (?, ?)""",
|
|
1147
|
+
(query,),
|
|
1148
|
+
)
|
|
1149
|
+
richprint(f"Added {c.rowcount} rows")
|
|
1150
|
+
db.commit()
|
|
1151
|
+
richprint(f"Watching {query}")
|
|
1152
|
+
|
|
1153
|
+
|
|
1154
|
+
@cli.command()
|
|
1155
|
+
@click.argument("doi")
|
|
1156
|
+
@click.option("-r", "--remove", is_flag=True, help="remove")
|
|
1157
|
+
def citing(doi, remove=False):
|
|
1158
|
+
"""Create a watch for articles that cite doi.
|
|
1159
|
+
|
|
1160
|
+
REMOVE is a flag to remove the doi.
|
|
1161
|
+
"""
|
|
1162
|
+
url = "https://api.openalex.org/works"
|
|
1163
|
+
|
|
1164
|
+
# We need an OpenAlex id
|
|
1165
|
+
f = f"doi:{doi}"
|
|
1166
|
+
|
|
1167
|
+
data = get_data(url, params={"filter": f})
|
|
1168
|
+
if len(data["results"]) == 0:
|
|
1169
|
+
richprint(f"Sorry, {doi} does not seem valid.")
|
|
1170
|
+
|
|
1171
|
+
wid = data["results"][0]["id"]
|
|
1172
|
+
|
|
1173
|
+
if remove:
|
|
1174
|
+
c = db.execute("""delete from queries where filter = ?""", (f"cites:{wid}",))
|
|
1175
|
+
db.commit()
|
|
1176
|
+
richprint(f"Deleted {c.rowcount} rows")
|
|
1177
|
+
else:
|
|
1178
|
+
c = db.execute(
|
|
1179
|
+
"""insert or ignore into queries(filter, description)
|
|
1180
|
+
values (?, ?)""",
|
|
1181
|
+
(f"cites:{wid}", f"Citing papers for {doi}"),
|
|
1182
|
+
)
|
|
1183
|
+
|
|
1184
|
+
db.commit()
|
|
1185
|
+
richprint(f"Added {c.rowcount} rows")
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
@cli.command()
|
|
1189
|
+
@click.argument("doi")
|
|
1190
|
+
@click.option("-r", "--remove", is_flag=True, help="remove")
|
|
1191
|
+
def related(doi, remove=False):
|
|
1192
|
+
"""Create a watch for articles that are related to doi.
|
|
1193
|
+
|
|
1194
|
+
REMOVE is a flag to remove the doi from queries.
|
|
1195
|
+
"""
|
|
1196
|
+
url = "https://api.openalex.org/works"
|
|
1197
|
+
|
|
1198
|
+
# We need an OpenAlex id
|
|
1199
|
+
f = f"doi:{doi}"
|
|
1200
|
+
|
|
1201
|
+
data = get_data(url, params={"filter": f})
|
|
1202
|
+
if len(data["results"]) == 0:
|
|
1203
|
+
richprint(f"Sorry, {doi} does not seem valid.")
|
|
1204
|
+
|
|
1205
|
+
wid = data["results"][0]["id"]
|
|
1206
|
+
|
|
1207
|
+
if remove:
|
|
1208
|
+
c = db.execute(
|
|
1209
|
+
"""delete from queries where filter = ?""", (f"related_to:{wid}",)
|
|
1210
|
+
)
|
|
1211
|
+
db.commit()
|
|
1212
|
+
richprint(f"Deleted {c.rowcount} rows")
|
|
1213
|
+
else:
|
|
1214
|
+
c = db.execute(
|
|
1215
|
+
"""insert or ignore into queries(filter, description)
|
|
1216
|
+
values (?, ?)""",
|
|
1217
|
+
(f"related_to:{wid}", f"Related papers for {doi}"),
|
|
1218
|
+
)
|
|
1219
|
+
|
|
1220
|
+
db.commit()
|
|
1221
|
+
richprint(f"Added {c.rowcount} rows")
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
#############
|
|
1225
|
+
# Utilities #
|
|
1226
|
+
#############
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
@cli.command()
|
|
1230
|
+
@click.argument("sources", nargs=-1)
|
|
1231
|
+
def bibtex(sources):
|
|
1232
|
+
"""Generate bibtex entries for sources."""
|
|
1233
|
+
if not sources:
|
|
1234
|
+
sources = sys.stdin.read().strip().split()
|
|
1235
|
+
|
|
1236
|
+
for source in sources:
|
|
1237
|
+
(work,) = db.execute(
|
|
1238
|
+
"""select extra from sources where source = ?""", (source,)
|
|
1239
|
+
).fetchone()
|
|
1240
|
+
richprint(dump_bibtex(json.loads(work)))
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
@cli.command()
|
|
1244
|
+
@click.argument("sources", nargs=-1)
|
|
1245
|
+
def citation(sources):
|
|
1246
|
+
"""Generate citation strings for sources."""
|
|
1247
|
+
if not sources:
|
|
1248
|
+
sources = sys.stdin.read().strip().split()
|
|
1249
|
+
|
|
1250
|
+
for i, source in enumerate(sources):
|
|
1251
|
+
(_citation,) = db.execute(
|
|
1252
|
+
"""select json_extract(extra, '$.citation')
|
|
1253
|
+
from sources where source = ?""",
|
|
1254
|
+
(source,),
|
|
1255
|
+
).fetchone()
|
|
1256
|
+
richprint(f"{i + 1:2d}. {_citation}")
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
@cli.command()
|
|
1260
|
+
@click.argument("doi")
|
|
1261
|
+
def unpaywall(doi):
|
|
1262
|
+
"""Use unpaywall to find PDFs for doi."""
|
|
1263
|
+
config = get_config()
|
|
1264
|
+
url = f"https://api.unpaywall.org/v2/{doi}"
|
|
1265
|
+
params = {"email": config["openalex"]["email"]}
|
|
1266
|
+
|
|
1267
|
+
resp = requests.get(url, params)
|
|
1268
|
+
if resp.status_code == 200:
|
|
1269
|
+
data = resp.json()
|
|
1270
|
+
richprint(f'{data["title"]}, {data.get("journal_name") or ""}')
|
|
1271
|
+
richprint(f'Is open access: {data.get("is_oa", False)}')
|
|
1272
|
+
|
|
1273
|
+
for loc in data.get("oa_locations", []):
|
|
1274
|
+
richprint(loc.get("url_for_pdf") or loc.get("url_for_landing_page"))
|
|
1275
|
+
else:
|
|
1276
|
+
richprint(f"{doi} not found in unpaywall")
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
@cli.command()
|
|
1280
|
+
def about():
|
|
1281
|
+
"""Summary statistics of your db."""
|
|
1282
|
+
config = get_config()
|
|
1283
|
+
dbf = os.path.join(config["root"], "litdb.libsql")
|
|
1284
|
+
|
|
1285
|
+
richprint(f"Your database is located at {dbf}")
|
|
1286
|
+
kb = 1024
|
|
1287
|
+
mb = 1024 * kb
|
|
1288
|
+
gb = 1024 * mb
|
|
1289
|
+
richprint(f"Database size: {os.path.getsize(dbf) / gb:1.2f} GB")
|
|
1290
|
+
db = get_db()
|
|
1291
|
+
(nsources,) = db.execute("select count(source) from sources").fetchone()
|
|
1292
|
+
richprint(f"You have {nsources} sources")
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
@cli.command()
|
|
1296
|
+
@click.argument("sql")
|
|
1297
|
+
def sql(sql):
|
|
1298
|
+
"""Run the SQL command on the db."""
|
|
1299
|
+
for row in db.execute(sql).fetchall():
|
|
1300
|
+
richprint(row)
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
@cli.command()
|
|
1304
|
+
@click.argument("sources", nargs=-1)
|
|
1305
|
+
@click.option("-f", "--fmt", default="{{ source }}\n{{ text }}")
|
|
1306
|
+
def show(sources, fmt):
|
|
1307
|
+
"""Show the source.
|
|
1308
|
+
|
|
1309
|
+
FMT is a jinja template with access to source, text, extra for each arg.
|
|
1310
|
+
"""
|
|
1311
|
+
for src in sources:
|
|
1312
|
+
result = db.execute(
|
|
1313
|
+
"""select source, text, extra from
|
|
1314
|
+
sources where source = ?""",
|
|
1315
|
+
(src,),
|
|
1316
|
+
).fetchone()
|
|
1317
|
+
|
|
1318
|
+
if result:
|
|
1319
|
+
source, text, extra = result
|
|
1320
|
+
extra = json.loads(extra)
|
|
1321
|
+
richprint(Template(fmt).render(**locals()))
|
|
1322
|
+
else:
|
|
1323
|
+
print(f"Nothing found for {src}")
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
@cli.command(name="open")
|
|
1327
|
+
@click.argument("source")
|
|
1328
|
+
def visit(source):
|
|
1329
|
+
"""Open source."""
|
|
1330
|
+
if source.startswith("http"):
|
|
1331
|
+
webbrowser.open(source, new=2)
|
|
1332
|
+
elif source.endswith(".pdf"):
|
|
1333
|
+
webbrowser.open(f"file://{source}")
|
|
1334
|
+
else:
|
|
1335
|
+
webbrowser.open(f"file://{source}")
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
@cli.command()
|
|
1339
|
+
def update_embeddings():
|
|
1340
|
+
"""Update all the embeddings in your db.
|
|
1341
|
+
|
|
1342
|
+
The only reason you would do this is if you change the embedding model, or
|
|
1343
|
+
the way the chunks are sized in your config.
|
|
1344
|
+
|
|
1345
|
+
"""
|
|
1346
|
+
config = get_config()
|
|
1347
|
+
db = get_db()
|
|
1348
|
+
from sentence_transformers import SentenceTransformer
|
|
1349
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
1350
|
+
|
|
1351
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
1352
|
+
splitter = RecursiveCharacterTextSplitter(
|
|
1353
|
+
chunk_size=config["embedding"]["chunk_size"],
|
|
1354
|
+
chunk_overlap=config["embedding"]["chunk_overlap"],
|
|
1355
|
+
)
|
|
1356
|
+
|
|
1357
|
+
_, dim = model.encode(["test"]).shape
|
|
1358
|
+
|
|
1359
|
+
# The point of this is to avoid deleting the database.
|
|
1360
|
+
db.execute("drop index if exists embedding_idx")
|
|
1361
|
+
db.execute("alter table sources drop embedding")
|
|
1362
|
+
db.execute(f"alter table sources add column embedding F32_BLOB({dim})")
|
|
1363
|
+
db.commit()
|
|
1364
|
+
|
|
1365
|
+
for rowid, text in db.execute("select rowid, text from sources").fetchall():
|
|
1366
|
+
chunks = splitter.split_text(text)
|
|
1367
|
+
embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
|
|
1368
|
+
|
|
1369
|
+
c = db.execute(
|
|
1370
|
+
"update sources set embedding = ? where rowid = ?", (embedding, rowid)
|
|
1371
|
+
)
|
|
1372
|
+
print(rowid, c.rowcount)
|
|
1373
|
+
|
|
1374
|
+
# I don't know why this has to be here. I had it above, and no updates were
|
|
1375
|
+
# happening.
|
|
1376
|
+
db.execute(
|
|
1377
|
+
"""create index if not exists embedding_idx ON sources (libsql_vector_idx(embedding))"""
|
|
1378
|
+
)
|
|
1379
|
+
db.commit()
|
|
1380
|
+
|
|
1381
|
+
|
|
1382
|
+
######################
|
|
1383
|
+
# Academic functions #
|
|
1384
|
+
######################
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
@cli.command()
|
|
1388
|
+
@click.argument("orcid")
|
|
1389
|
+
def coa(orcid):
|
|
1390
|
+
"""Generate Table 4 of Collaborators and Other Affiliations for NSF.
|
|
1391
|
+
|
|
1392
|
+
ORCID is an orcid URL for the user to generate the table for.
|
|
1393
|
+
The file is saved in {orcid}-{today}.xlsx.
|
|
1394
|
+
"""
|
|
1395
|
+
from .coa import get_coa
|
|
1396
|
+
|
|
1397
|
+
get_coa(orcid)
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
@cli.command()
|
|
1401
|
+
@click.argument("query", nargs=-1)
|
|
1402
|
+
@click.option("-n", default=5, help="Number of documents to use")
|
|
1403
|
+
def suggest_reviewers(query, n):
|
|
1404
|
+
"""Suggest reviewers for QUERY.
|
|
1405
|
+
|
|
1406
|
+
Use up to N similar documents. This is an iterative function, you will be
|
|
1407
|
+
prompted to expand the search.
|
|
1408
|
+
"""
|
|
1409
|
+
config = get_config()
|
|
1410
|
+
query = " ".join(query)
|
|
1411
|
+
|
|
1412
|
+
# This is a surprise. You can't just call the functions above! This is
|
|
1413
|
+
# apparently the way to do this.
|
|
1414
|
+
with click.Context(vsearch) as ctx:
|
|
1415
|
+
results = ctx.invoke(
|
|
1416
|
+
vsearch, query=query.split(" "), n=n, fmt="", iterative=True
|
|
1417
|
+
)
|
|
1418
|
+
|
|
1419
|
+
# Now collect the authors from the matching papers
|
|
1420
|
+
authors = []
|
|
1421
|
+
|
|
1422
|
+
for i, row in enumerate(results):
|
|
1423
|
+
source, citation, extra, distance = row
|
|
1424
|
+
|
|
1425
|
+
d = json.loads(extra)
|
|
1426
|
+
|
|
1427
|
+
for authorship in d["authorships"]:
|
|
1428
|
+
authors += [authorship["author"]["id"]]
|
|
1429
|
+
|
|
1430
|
+
# get the unique ones
|
|
1431
|
+
from collections import Counter
|
|
1432
|
+
|
|
1433
|
+
authors = Counter(authors)
|
|
1434
|
+
|
|
1435
|
+
# Get author information
|
|
1436
|
+
data = []
|
|
1437
|
+
|
|
1438
|
+
url = "https://api.openalex.org/authors/"
|
|
1439
|
+
# You can only filter on 50 ids at a time, so we hard code this limit here
|
|
1440
|
+
# and per page.
|
|
1441
|
+
for batch in batched(authors, 50):
|
|
1442
|
+
url = f'https://api.openalex.org/authors?filter=id:{"|".join(batch)}'
|
|
1443
|
+
|
|
1444
|
+
params = {"per-page": 50, "email": config["openalex"]["email"]}
|
|
1445
|
+
|
|
1446
|
+
r = get_data(url, params)
|
|
1447
|
+
|
|
1448
|
+
for d in r["results"]:
|
|
1449
|
+
lki = d.get("last_known_institutions", [])
|
|
1450
|
+
if lki == []:
|
|
1451
|
+
affils = d.get("affiliations", [])
|
|
1452
|
+
if len(affils) >= 0:
|
|
1453
|
+
lki = affils[0]["institution"]["display_name"]
|
|
1454
|
+
else:
|
|
1455
|
+
lki = "unknown"
|
|
1456
|
+
|
|
1457
|
+
else:
|
|
1458
|
+
if len(lki) >= 1:
|
|
1459
|
+
lki = lki[0].get("display_name")
|
|
1460
|
+
else:
|
|
1461
|
+
lki = "unknown"
|
|
1462
|
+
|
|
1463
|
+
row = [
|
|
1464
|
+
d["display_name"],
|
|
1465
|
+
authors[d["id"]],
|
|
1466
|
+
d["summary_stats"]["h_index"],
|
|
1467
|
+
d["id"],
|
|
1468
|
+
lki,
|
|
1469
|
+
]
|
|
1470
|
+
data += [row]
|
|
1471
|
+
|
|
1472
|
+
# Sort and display the results
|
|
1473
|
+
data.sort(key=lambda row: row[2], reverse=True)
|
|
1474
|
+
|
|
1475
|
+
if get_ipython():
|
|
1476
|
+
display(
|
|
1477
|
+
HTML(
|
|
1478
|
+
tabulate.tabulate(
|
|
1479
|
+
data,
|
|
1480
|
+
headers=["name", "# papers", "h-index", "oaid", "institution"],
|
|
1481
|
+
tablefmt="html",
|
|
1482
|
+
)
|
|
1483
|
+
)
|
|
1484
|
+
)
|
|
1485
|
+
for i, row in enumerate(results):
|
|
1486
|
+
source, citation, extra, distance = row
|
|
1487
|
+
richprint(f"{i + 1:2d}. {citation} (source)\n\n")
|
|
1488
|
+
|
|
1489
|
+
else:
|
|
1490
|
+
s = ["Potential reviewers"]
|
|
1491
|
+
s += [
|
|
1492
|
+
tabulate.tabulate(
|
|
1493
|
+
data,
|
|
1494
|
+
headers=["name", "# papers", "h-index", "oaid", "institution"],
|
|
1495
|
+
tablefmt="orgtbl",
|
|
1496
|
+
)
|
|
1497
|
+
]
|
|
1498
|
+
s += ["\n" + "From these papers:"]
|
|
1499
|
+
for i, row in enumerate(results):
|
|
1500
|
+
source, citation, extra, distance = row
|
|
1501
|
+
s += [f"{i + 1:2d}. {citation} (source)\n\n"]
|
|
1502
|
+
|
|
1503
|
+
console = Console(color_system="truecolor")
|
|
1504
|
+
with console.pager():
|
|
1505
|
+
for _s in s:
|
|
1506
|
+
console.print(_s)
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
if __name__ == "__main__":
|
|
1510
|
+
cli()
|