litdb 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
litdb/cli.py ADDED
@@ -0,0 +1,1510 @@
1
+ """CLI for litdb.
2
+
3
+ The main command is litdb. There are subcommands for the actions.
4
+ """
5
+
6
+ import os
7
+ import datetime
8
+ import json
9
+ import pathlib
10
+ import tempfile
11
+ import sys
12
+ import warnings
13
+
14
+ import bs4
15
+ import click
16
+ import dateparser
17
+ from docx import Document
18
+ from IPython import get_ipython
19
+ from IPython.display import display, HTML
20
+ from jinja2 import Template
21
+ from more_itertools import batched
22
+ import nbformat
23
+ from nbconvert import MarkdownExporter
24
+ import numpy as np
25
+
26
+ from pptx import Presentation
27
+ import requests
28
+ from rich import print as richprint
29
+ from rich.console import Console
30
+ from sentence_transformers import SentenceTransformer
31
+ import tabulate
32
+ from tqdm import tqdm
33
+ import webbrowser
34
+
35
+ from .utils import get_config, init_litdb
36
+ from .db import get_db, add_source, add_work, add_author, update_filter, add_bibtex
37
+ from .openalex import get_data, get_text
38
+ from .pdf import add_pdf
39
+ from .bibtex import dump_bibtex
40
+ from .gpt import gpt
41
+
42
+ warnings.filterwarnings("ignore")
43
+
44
+ db = get_db()
45
+
46
+
47
+ @click.group()
48
+ def cli():
49
+ """Group command for litdb."""
50
+ pass
51
+
52
+
53
+ @cli.command()
54
+ def init():
55
+ """Initialize a litdb directory in the current working directory."""
56
+ init_litdb()
57
+ print(f"Initialized in {os.getcwd()}")
58
+
59
+
60
+ #################
61
+ # Add functions #
62
+ #################
63
+
64
+
65
+ @cli.command()
66
+ @click.argument("sources", nargs=-1)
67
+ @click.option("--references", is_flag=True, help="Add references too.")
68
+ @click.option("--related", is_flag=True, help="Add related too.")
69
+ @click.option("--citing", is_flag=True, help="Add citing too.")
70
+ @click.option("--all", is_flag=True, help="Add references, related and citing.")
71
+ @click.option("-t", "--tag", "tags", multiple=True)
72
+ def add(
73
+ sources,
74
+ references=False,
75
+ citing=False,
76
+ related=False,
77
+ all=False,
78
+ verbose=False,
79
+ tags=None,
80
+ ):
81
+ """Add WIDS to the db.
82
+
83
+ REFERENCES, RELATED, CITING are flags to also add those for DOI sources. ALL
84
+ is shorthand for all of those.
85
+
86
+ SOURCES can be one or more of a doi or orcid, a pdf path, a url, bibtex
87
+ file, or other kind of file assumed to be text.
88
+
89
+ TAGS is a list of tags to add to the source.
90
+
91
+ These are one time additions.
92
+
93
+ """
94
+ for source in tqdm(sources):
95
+ # a work
96
+ if source.startswith("10.") or "doi.org" in source:
97
+ if source.startswith("10."):
98
+ source = f"https://doi.org/{source}"
99
+
100
+ if all:
101
+ references, citing, related = True, True, True
102
+
103
+ add_work(source, references, citing, related)
104
+
105
+ # works from an author
106
+ elif "orcid" in source or source.lower().startswith("https://openalex.org/a"):
107
+ add_author(source)
108
+
109
+ # a bibtex file
110
+ elif source.endswith(".bib"):
111
+ add_bibtex(source)
112
+
113
+ # pdf
114
+ elif source.endswith(".pdf"):
115
+ source = os.path.abspath(source)
116
+ add_pdf(source)
117
+
118
+ # docx
119
+ elif source.endswith(".docx"):
120
+ source = os.path.abspath(source)
121
+ doc = Document(source)
122
+ add_source(source, "\n".join([para.text for para in doc.paragraphs]))
123
+
124
+ # pptx
125
+ elif source.endswith(".pptx"):
126
+ source = os.path.abspath(source)
127
+ prs = Presentation(source)
128
+ text = []
129
+ for slide in prs.slides:
130
+ for shape in slide.shapes:
131
+ if hasattr(shape, "text"):
132
+ text.append(shape.text)
133
+ add_source(source, "\n".join(text))
134
+
135
+ # local html
136
+ elif not source.startswith("http") and source.endswith(".html"):
137
+ source = os.path.abspath(source)
138
+ with open(source) as f:
139
+ text = f.read()
140
+ soup = bs4.BeautifulSoup(text, features="lxml")
141
+ add_source(source, soup.get_text())
142
+
143
+ # a url
144
+ elif source.startswith("http"):
145
+ soup = bs4.BeautifulSoup(requests.get(source).text)
146
+ add_source(source, soup.get_text())
147
+
148
+ # ipynb
149
+ elif source.endswith(".ipynb"):
150
+ source = os.path.abspath(source)
151
+ with open(source) as f:
152
+ notebook = nbformat.read(f, as_version=4)
153
+
154
+ # Create a Markdown exporter
155
+ markdown_exporter = MarkdownExporter()
156
+
157
+ # Convert the notebook to Markdown
158
+ (body, resources) = markdown_exporter.from_notebook_node(notebook)
159
+
160
+ add_source(source, body)
161
+
162
+ # assume it is text
163
+ else:
164
+ source = os.path.abspath(source)
165
+ with open(source) as f:
166
+ text = f.read()
167
+ add_source(source, text)
168
+
169
+ if tags:
170
+ with click.Context(add_tag) as ctx:
171
+ ctx.invoke(add_tag, sources=sources, tags=tags)
172
+
173
+
174
+ @cli.command()
175
+ @click.argument("query", nargs=-1)
176
+ @click.option("--references", is_flag=True)
177
+ @click.option("--related", is_flag=True)
178
+ @click.option("--citing", is_flag=True)
179
+ def crossref(query, references, related, citing):
180
+ """Add entries to litdb from a crossref query."""
181
+ query = " ".join(query)
182
+ resp = requests.get("https://api.crossref.org/works", params={"query": query})
183
+
184
+ if resp.status_code == 200:
185
+ data = resp.json()
186
+ for i, item in enumerate(data["message"]["items"]):
187
+ authors = ", ".join(
188
+ [f'{au["given"]} {au["family"]}' for au in item.get("author", [])]
189
+ )
190
+ source = " ".join(item.get("container-title", ["no source"]))
191
+ published = item.get("published", {}) or {}
192
+ year = published.get("date-parts", [["no year"]])[0][0]
193
+ title = item.get("title", ["no title"])
194
+ richprint(
195
+ f'{i}. {" ".join(title)}, {authors}, {source} ({year}), https://doi.org/{item["DOI"]}.'
196
+ )
197
+
198
+ toadd = input("Enter space separated numbers to add, or return to quit. ")
199
+
200
+ if toadd:
201
+ toadd = [int(x) for x in toadd.split(" ")]
202
+ dois = [
203
+ "https://doi.org/" + data["message"]["items"][i]["DOI"] for i in toadd
204
+ ]
205
+
206
+ with click.Context(add) as ctx:
207
+ ctx.invoke(
208
+ add,
209
+ sources=dois,
210
+ related=related,
211
+ references=references,
212
+ citing=citing,
213
+ )
214
+
215
+
216
+ @cli.command()
217
+ @click.argument("sources", nargs=-1)
218
+ def index(sources):
219
+ """Index the directories in SOURCES.
220
+
221
+ SOURCES is a list of directories.
222
+ """
223
+ for directory in sources:
224
+ directory = pathlib.Path(directory).resolve()
225
+ for fname in directory.rglob("*"):
226
+ # for f in files:
227
+ if fname.suffix in [
228
+ ".pdf",
229
+ ".docx",
230
+ ".pptx",
231
+ ".org",
232
+ ".md",
233
+ ".html",
234
+ ".bib",
235
+ ".ipynb",
236
+ ]:
237
+ fname = str(fname)
238
+
239
+ # skip files we already have
240
+ if db.execute(
241
+ """select source from sources where source = ?""", (fname,)
242
+ ).fetchone():
243
+ continue
244
+
245
+ with click.Context(add) as ctx:
246
+ print(fname)
247
+ ctx.invoke(add, sources=[fname])
248
+ print(f"Adding {fname}")
249
+
250
+ richprint(f"Added {fname}")
251
+
252
+ last_updated = datetime.date.today().strftime("%Y-%m-%d")
253
+
254
+ directory = str(directory) # we need strings for the db
255
+ if db.execute(
256
+ """select path from directories where path = ?""", (directory,)
257
+ ).fetchone():
258
+ print(f"Updating {directory}")
259
+ db.execute(
260
+ """update directories set last_updated = ?
261
+ where path = ?""",
262
+ (last_updated, directory),
263
+ )
264
+ else:
265
+ print(f"Inserting {directory}: {last_updated}")
266
+ db.execute(
267
+ """insert into directories(path, last_updated)
268
+ values (?, ?)""",
269
+ (directory, last_updated),
270
+ )
271
+
272
+ db.commit()
273
+
274
+
275
+ @cli.command()
276
+ def reindex():
277
+ """Reindex saved directories."""
278
+ for (directory,) in db.execute("""select path from directories""").fetchall():
279
+ print(f"Reindexing {directory}")
280
+ index([directory])
281
+
282
+
283
+ ###########
284
+ # Tagging #
285
+ ###########
286
+ @cli.command()
287
+ @click.argument("sources", nargs=-1)
288
+ @click.option("-t", "--tag", "tags", multiple=True)
289
+ def add_tag(sources, tags):
290
+ """Add tags to sources.
291
+
292
+ It is a little annoying to add multiple tags. It looks like this.
293
+ litdb add-tag source -t tag1 -t tag2
294
+ """
295
+ for source in sources:
296
+ # Get source id
297
+ (source_id,) = db.execute(
298
+ "select rowid from sources where source = ?", (source,)
299
+ ).fetchone()
300
+
301
+ for tag in tags:
302
+ # get tag id
303
+ tag_id = db.execute(
304
+ "select rowid from tags where tag = ?", (tag,)
305
+ ).fetchone()
306
+
307
+ if not tag_id:
308
+ c = db.execute("insert into tags(tag) values (?)", (tag,))
309
+ tag_id = c.lastrowid
310
+ db.commit()
311
+ else:
312
+ # we get a tuple in the first query
313
+ (tag_id,) = tag_id
314
+
315
+ # Now add a tag
316
+ db.execute(
317
+ "insert into source_tag(source_id, tag_id) values (?, ?)",
318
+ (source_id, tag_id),
319
+ )
320
+ db.commit()
321
+
322
+ print(f"Tagged {source} with {tag}")
323
+
324
+
325
+ @cli.command()
326
+ @click.argument("sources", nargs=-1)
327
+ @click.option("-t", "--tag", "tags", multiple=True)
328
+ def rm_tag(sources, tags):
329
+ """Remove tags from sources.
330
+
331
+ It is a little annoying to remove multiple tags. It looks like this.
332
+ litdb rm-tag source -t tag1 -t tag2
333
+ """
334
+ for source in sources:
335
+ # Get source id
336
+ (source_id,) = db.execute(
337
+ "select rowid from sources where source = ?", (source,)
338
+ ).fetchone()
339
+
340
+ for tag in tags:
341
+ # get tag id. Assume it exists?
342
+ (tag_id,) = db.execute(
343
+ "select rowid from tags where tag = ?", (tag,)
344
+ ).fetchone()
345
+
346
+ c = db.execute(
347
+ """delete from source_tag
348
+ where source_id = ? and tag_id = ?""",
349
+ (source_id, tag_id),
350
+ )
351
+
352
+ db.commit()
353
+ print(f"Deleted {c.rowcount} rows ({tag} from {source}")
354
+
355
+
356
+ @cli.command()
357
+ @click.argument("tags", nargs=-1)
358
+ def delete_tag(tags):
359
+ """Delete each tag.
360
+
361
+ This should also delete tags from sources by cascade.
362
+ """
363
+ for tag in tags:
364
+ c = db.execute("delete from tags where tag = ?", (tag,))
365
+ print(f"Deleted {c.rowcount} rows ({tag})")
366
+ db.commit()
367
+
368
+
369
+ @cli.command()
370
+ @click.argument("tags", nargs=-1)
371
+ @click.option("-f", "--fmt", default='{{ source }}\n{{ extra["citation"] }}')
372
+ def show_tag(tags, fmt):
373
+ """Show entries with tags.
374
+
375
+ FMT is a jinja template for the output. You have variables of source, text
376
+ and extra.
377
+
378
+ I don't have good logic here, we just show all entries. I could probably get
379
+ some basic and logic with sets, but mostly I assume for now you only want
380
+ one tag, so this works. TODO: add something like boolean logic?
381
+
382
+ """
383
+ template = Template(fmt)
384
+ for tag in tags:
385
+ for row in db.execute(
386
+ """select
387
+ sources.source, sources.text, sources.extra
388
+ from sources
389
+ inner join source_tag on source_tag.source_id = sources.rowid
390
+ inner join tags on source_tag.tag_id = tags.rowid
391
+ where tags.tag = ?""",
392
+ (tag,),
393
+ ).fetchall():
394
+ source, text, extra = row
395
+ extra = json.loads(extra)
396
+ richprint(template.render(**locals()))
397
+
398
+
399
+ @cli.command()
400
+ def list_tags():
401
+ """Print defined tags."""
402
+ print("The following tags are defined.")
403
+ for (tag,) in db.execute("select tag from tags").fetchall():
404
+ print(tag)
405
+
406
+
407
+ ##########
408
+ # Review #
409
+ ##########
410
+
411
+
412
+ @cli.command()
413
+ @click.option("-s", "--since", default="1 week ago")
414
+ @click.option("-f", "--fmt", default=None)
415
+ def review(since, fmt):
416
+ """Review new entries added SINCE.
417
+
418
+ SINCE should be something dateparser can handle.
419
+ FMT is a jinja template for the output. Defaults to an org-mode template.
420
+ """
421
+
422
+ since = dateparser.parse(since).strftime("%Y-%m-%d")
423
+ c = db.execute(
424
+ """select source, text, extra from sources
425
+ where date(date_added) > ?""",
426
+ (since,),
427
+ ).fetchall()
428
+
429
+ template = Template(
430
+ fmt
431
+ or """* {{ extra['display_name'] | replace("\n", " ") }}
432
+ :PROPERTIES:
433
+ :SOURCE: {{ source }}
434
+ :OPENALEX: {{ extra.get('id') }}
435
+ :YEAR: {{ extra.get('publication_year') }}
436
+ :REFERENCE_COUNT: {{ extra.get('referenced_works_count', 0) }}
437
+ :CITED_BY_COUNT: {{ extra.get('cited_by_count', 0) }}
438
+ :END:
439
+
440
+ {{ text }} litdb:{{ source }}
441
+ """
442
+ )
443
+
444
+ for source, text, extra in c:
445
+ extra = json.loads(extra) or {}
446
+ print(template.render(**locals()))
447
+
448
+
449
+ #############
450
+ # Searching #
451
+ #############
452
+
453
+
454
+ @cli.command()
455
+ def screenshot():
456
+ """Do vector search from text in a screenshot.
457
+
458
+ Use OCR to get text from an image on the clipboard (probably from a
459
+ screenshot) and do a vector search on the text.
460
+ """
461
+ from PIL import ImageGrab
462
+ import pytesseract
463
+
464
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
465
+
466
+ # Grab the image from the clipboard
467
+ img = ImageGrab.grabclipboard()
468
+
469
+ if img:
470
+ text = pytesseract.image_to_string(img)
471
+ print(f'Searching for "{text}"')
472
+ vsearch(text)
473
+ else:
474
+ print("No image found in clipboard.")
475
+
476
+
477
+ def record():
478
+ """Make a recording to a tempfile.
479
+
480
+ Returns the audio filename it is recorded in.
481
+ """
482
+ import pyaudio
483
+ import wave
484
+ import threading
485
+
486
+ # Parameters for recording
487
+ FORMAT = pyaudio.paInt16
488
+ CHANNELS = 2
489
+ RATE = 44100
490
+ CHUNK = 1024
491
+
492
+ _, audio_file = tempfile.mkstemp(suffix=".wav")
493
+
494
+ # Initialize PyAudio
495
+ audio = pyaudio.PyAudio()
496
+
497
+ # Function to record audio
498
+ def record_audio():
499
+ # Open stream
500
+ stream = audio.open(
501
+ format=FORMAT,
502
+ channels=CHANNELS,
503
+ rate=RATE,
504
+ input=True,
505
+ frames_per_buffer=CHUNK,
506
+ )
507
+ print("Recording... Press Enter to stop.")
508
+
509
+ frames = []
510
+
511
+ # Record until Enter is pressed
512
+ while not stop_recording.is_set():
513
+ data = stream.read(CHUNK)
514
+ frames.append(data)
515
+
516
+ # Stop and close the stream
517
+ stream.stop_stream()
518
+ stream.close()
519
+
520
+ # Save the recorded data as a WAV file
521
+ wf = wave.open(audio_file, "wb")
522
+ wf.setnchannels(CHANNELS)
523
+ wf.setsampwidth(audio.get_sample_size(FORMAT))
524
+ wf.setframerate(RATE)
525
+ wf.writeframes(b"".join(frames))
526
+ wf.close()
527
+
528
+ # Event to signal when to stop recording
529
+ stop_recording = threading.Event()
530
+
531
+ # Start recording in a separate thread
532
+ recording_thread = threading.Thread(target=record_audio)
533
+ recording_thread.start()
534
+
535
+ # Wait for Enter key press to stop recording
536
+ input()
537
+ stop_recording.set()
538
+
539
+ # Wait for the recording thread to finish
540
+ recording_thread.join()
541
+
542
+ # Terminate PyAudio
543
+ audio.terminate()
544
+
545
+ return audio_file
546
+
547
+
548
+ @cli.command()
549
+ @click.option("-p", "--playback", is_flag=True, help="Play audio back")
550
+ def audio(playback=False):
551
+ """Record audio, convert it to text, and do a vector search on the text.
552
+
553
+ The idea is nice, but the quality of transcription for scientific words is
554
+ not that great. A better transcription library might make this more useful.
555
+ """
556
+ import speech_recognition as sr
557
+
558
+ while True:
559
+ afile = record()
560
+ audio_file = sr.AudioFile(afile)
561
+ r = sr.Recognizer()
562
+ with audio_file as source:
563
+ audio = r.listen(source)
564
+ try:
565
+ text = r.recognize_sphinx(audio)
566
+ print("\n" + text + "\n")
567
+ except sr.UnknownValueError:
568
+ print("Could not understand audio.")
569
+
570
+ if playback:
571
+ import playsound
572
+
573
+ playsound.playsound(afile, block=True)
574
+
575
+ response = input("Is that what you want to search? ([y]/n/q): ")
576
+ if response.lower().startswith("q"):
577
+ return
578
+ elif response.lower().startswith("n"):
579
+ # record a new audio
580
+ continue
581
+ else:
582
+ # move on to searching
583
+ break
584
+
585
+ vsearch(text)
586
+
587
+
588
+ @cli.command()
589
+ @click.argument("query", nargs=-1)
590
+ @click.option("-n", default=3)
591
+ @click.option("-e", "--emacs", is_flag=True, default=False)
592
+ @click.option("-i", "--iterative", is_flag=True, default=False)
593
+ @click.option("-m", "--max-steps", default=None)
594
+ @click.option("-f", "--fmt", default=" {{ i }}. ({{ score|round(3) }}) {{ text }}\n\n")
595
+ @click.option("-x", "--cross-encode", is_flag=True, default=False)
596
+ def vsearch(query, n, emacs, fmt, cross_encode, iterative, max_steps):
597
+ """Do a vector search on QUERY.
598
+
599
+ N is an integer for number of results to return
600
+ EMACS is a flag for changing the output format for emacs
601
+
602
+ FORMAT is a jinja template for the output. The variables you have access to
603
+ are i, source, text, extra, similarity.
604
+
605
+ CROSS_ENCODE is a boolean that resorts the results with a cross-encoder.
606
+
607
+ ITERATIVE is a boolean that expands the search by references, citations and
608
+ related articles from the top matches until you tell it to stop or reach
609
+ MAX_STEPS.
610
+
611
+ MAX_STEPS is the maximum number of iterations to take. If you set this, you
612
+ will not be prompted each time, it will just run those steps until nothing
613
+ better is found, or you reach the number.
614
+
615
+ """
616
+ config = get_config()
617
+ query = " ".join(query)
618
+ model = SentenceTransformer(config["embedding"]["model"])
619
+ emb = model.encode([query]).astype(np.float32).tobytes()
620
+
621
+ if iterative:
622
+ best = None
623
+
624
+ steps = 0
625
+
626
+ while True:
627
+ results = db.execute(
628
+ """select
629
+ sources.source, sources.text,
630
+ sources.extra, vector_distance_cos(?, embedding) as d
631
+ from vector_top_k('embedding_idx', ?, ?)
632
+ join sources on sources.rowid = id
633
+ order by d""",
634
+ (emb, emb, n),
635
+ ).fetchall()
636
+
637
+ for source, text, extra, d in results:
638
+ richprint(f"{d:1.3f}: {source}")
639
+
640
+ steps += 1
641
+ if steps == max_steps:
642
+ break
643
+
644
+ current = [x[0] for x in results] # sources
645
+
646
+ # This means no change
647
+ if current == best:
648
+ print("Nothing new was found")
649
+ break
650
+
651
+ if not max_steps and input(
652
+ "Search for better matches? ([y]/n)"
653
+ ).lower().startswith("n"):
654
+ break
655
+
656
+ # something changed. add references and loop
657
+ best = current
658
+ for source in current:
659
+ add_work(source, True, True, True)
660
+
661
+ else:
662
+ c = db.execute(
663
+ """select sources.source, sources.text,
664
+ sources.extra, vector_distance_cos(?, embedding)
665
+ from vector_top_k('embedding_idx', ?, ?)
666
+ join sources on sources.rowid = id""",
667
+ (emb, emb, n),
668
+ )
669
+
670
+ results = c.fetchall()
671
+
672
+ if cross_encode:
673
+ import torch
674
+ from sentence_transformers.cross_encoder import CrossEncoder
675
+
676
+ # I don't know why I have to set the activation function here, but the
677
+ # score is not 0..1 otherwise
678
+ ce = CrossEncoder(
679
+ config["embedding"]["cross-encoder"],
680
+ default_activation_function=torch.nn.Sigmoid(),
681
+ )
682
+ scores = ce.predict([[query, text] for _, text, _, _ in results])
683
+ # resort based on the scores
684
+ results = [results[i] for i in np.argsort(scores)]
685
+
686
+ if emacs:
687
+ tmpl = (
688
+ "( {% for source, text, extra, score in results %}"
689
+ '("({{ score|round(3) }}) {{ text }}" . "{{ source }}") '
690
+ "{% endfor %})"
691
+ )
692
+ template = Template(tmpl)
693
+ print(template.render(**locals()))
694
+ else:
695
+ for i, row in enumerate(results, 1):
696
+ source, text, extra, score = row
697
+ template = Template(fmt)
698
+ richprint(template.render(**locals()))
699
+
700
+ return results
701
+
702
+
703
+ @cli.command()
704
+ @click.argument("query", nargs=-1)
705
+ @click.option("-n", default=3)
706
+ @click.option(
707
+ "-f", "--fmt", default="{{ source }} ({{ score | round(3) }})\n{{ snippet }}"
708
+ )
709
+ def fulltext(query, n, fmt):
710
+ """Perform a fulltext search on litdb."""
711
+ query = " ".join(query)
712
+
713
+ results = db.execute(
714
+ """select
715
+ sources.source, sources.text, snippet(fulltext, 1, '', '', '', 16), sources.extra, bm25(fulltext)
716
+ from fulltext
717
+ inner join sources on fulltext.source = sources.source
718
+ where fulltext.text match ? order by rank limit ?""",
719
+ (query, n),
720
+ ).fetchall()
721
+
722
+ for source, text, snippet, extra, score in results:
723
+ richprint(Template(fmt).render(**locals()))
724
+
725
+ return results
726
+
727
+
728
+ gpt = cli.command(gpt)
729
+
730
+
731
+ @cli.command()
732
+ @click.argument("source")
733
+ @click.option("-f", "--fmt", default=None)
734
+ @click.option("-e", "--emacs", is_flag=True, default=False)
735
+ @click.option("-n", default=3)
736
+ def similar(source, n, emacs, fmt):
737
+ """Find N sources similar to SOURCE by vector similarity.
738
+
739
+ if EMACS is truthy, the output is lisp for Emacs to read.
740
+ FMT is a jinja template with access to source, text, and extra
741
+ """
742
+ (emb,) = db.execute(
743
+ """select embedding from sources where source = ?""", (source,)
744
+ ).fetchone()
745
+
746
+ allrows = db.execute(
747
+ """select sources.source, sources.text, sources.extra
748
+ from vector_top_k('embedding_idx', ?, ?)
749
+ join sources on sources.rowid = id""",
750
+ # we do n + 1 because the first entry is always the source
751
+ (emb, n + 1),
752
+ ).fetchall()
753
+
754
+ rows = [(source, text, json.loads(extra)) for source, text, extra in allrows[1:]]
755
+
756
+ if emacs:
757
+ template = Template(
758
+ "({% for source, text, extra in rows %}"
759
+ ' ("{{ extra.get("citation") or text }}" . "{{ source }}")'
760
+ " {% endfor %})"
761
+ )
762
+ print(template.render(**locals()))
763
+ else:
764
+ template = Template(fmt or "{{ i }}. {{ source }}\n {{text}}\n\n")
765
+ # print starting at index 1, the first item is always the source.
766
+ for i, row in enumerate(rows, 1):
767
+ source, text, extra = row
768
+ richprint(template.render(**locals()))
769
+
770
+
771
+ @cli.command()
772
+ @click.argument("vector_query")
773
+ @click.argument("text_query")
774
+ @click.option("-n", default=5)
775
+ @click.option(
776
+ "-f", "--fmt", default="{{ source }} ({{ score | round(3) }})\n{{ text }}\n\n"
777
+ )
778
+ def hybrid_search(vector_query, text_query, n, fmt):
779
+ """Perform a hybrid vector and full text search.
780
+
781
+ VECTOR_QUERY: The query to do vector search on
782
+ TEXT_QUERY: The query for full text search
783
+ N is an integer number of documents to return.
784
+ FMT is a jinja template.
785
+ """
786
+ # Get vector results and score
787
+ with click.Context(vsearch) as ctx:
788
+ # source, text, extra, score
789
+ vresults = ctx.invoke(vsearch, query=vector_query.split(" "), n=n, fmt="")
790
+ vscores = [(result[0], result[3]) for result in vresults]
791
+
792
+ # Get text results and score
793
+ with click.Context(fulltext) as ctx:
794
+ tresults = ctx.invoke(fulltext, query=text_query.split(" "), n=n, fmt="")
795
+ # I think sqlite makes scores negative to sort them the way they want. I
796
+ # reverse this here.
797
+ tscores = [(result[0], -result[-1]) for result in tresults]
798
+
799
+ # Normalize scores
800
+ minv, maxv = min([x[1] for x in vscores]), max([x[1] for x in vscores])
801
+ mint, maxt = min([x[1] for x in tscores]), max([x[1] for x in tscores])
802
+
803
+ vscores = {oaid: (score - minv) / (maxv - minv) for oaid, score in vscores}
804
+ tscores = {oaid: (score - minv) / (maxv - minv) for oaid, score in tscores}
805
+
806
+ combined_scores = {}
807
+ for oaid in set(vscores.keys()).union(tscores.keys()):
808
+ vscore = vscores.get(oaid, 0)
809
+ tscore = tscores.get(oaid, 0)
810
+ cscore = 1 / (1 + vscore) + 1 / (1 + tscore)
811
+ combined_scores[oaid] = cscore
812
+
813
+ sorted_results = sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)
814
+ results = []
815
+ for oaid, score in sorted_results:
816
+ c = db.execute(
817
+ "select source, text, extra from sources where source = ?", (oaid,)
818
+ )
819
+ row = c.fetchone()
820
+ results += [[*row, score]]
821
+
822
+ for row in results:
823
+ source, text, extra, score = row
824
+ richprint(Template(fmt).render(**locals()))
825
+
826
+ return results
827
+
828
+
829
+ @cli.command()
830
+ @click.argument("query", nargs=-1)
831
+ @click.option("-g", "--google", is_flag=True)
832
+ @click.option("-gs", "--google-scholar", is_flag=True)
833
+ @click.option("-pm", "--pubmed", is_flag=True)
834
+ @click.option("-ar", "--arxiv", is_flag=True)
835
+ @click.option("-cr", "--chemrxiv", is_flag=True)
836
+ @click.option("-br", "--biorxiv", is_flag=True)
837
+ @click.option("-a", "--all", is_flag=True)
838
+ def web(query, google, google_scholar, pubmed, arxiv, chemrxiv, biorxiv, all):
839
+ """Open a web search for QUERY.
840
+
841
+ We always do an OpenAlex search.
842
+
843
+ If these are true, we also search here
844
+ GOOGLE
845
+ GOOGLE_SCHOLAR
846
+ PUBMED
847
+ ARXIV
848
+ CHEMRXIV
849
+ BIORXIV
850
+ """
851
+ query = " ".join(query)
852
+
853
+ if all:
854
+ google, google_scholar = (
855
+ True,
856
+ True,
857
+ )
858
+ pubmed, arxiv, chemrxiv, biorxiv = True, True, True, True
859
+
860
+ # This is to avoid some huggingface/tokenizer warning. I don't know why we
861
+ # need to do it, but this code forks the process, and triggers that warning.
862
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
863
+
864
+ oa = f"https://openalex.org/works?filter=default.search:{query}"
865
+ webbrowser.open(oa)
866
+
867
+ if google:
868
+ url = f"https://www.google.com/search?q={query}"
869
+ webbrowser.open(url)
870
+
871
+ if google_scholar:
872
+ url = f"https://scholar.google.com/scholar?q={query}"
873
+ webbrowser.open(url)
874
+
875
+ if pubmed:
876
+ url = f"https://pubmed-ncbi-nlm-nih-gov.cmu.idm.oclc.org/?term={query}"
877
+ webbrowser.open(url)
878
+
879
+ if arxiv:
880
+ url = f"https://arxiv.org/search/?query={query}"
881
+ webbrowser.open(url)
882
+
883
+ if chemrxiv:
884
+ url = f"https://chemrxiv.org/engage/chemrxiv/search-dashboard?text={query}"
885
+ webbrowser.open(url)
886
+
887
+ if biorxiv:
888
+ url = f"https://www.biorxiv.org/search/{query}"
889
+ webbrowser.open(url)
890
+
891
+
892
+ ###########
893
+ # Filters #
894
+ ###########
895
+
896
+
897
+ @cli.command()
898
+ @click.argument("_filter")
899
+ @click.option("-d", "--description")
900
+ def add_filter(_filter, description=None):
901
+ """Add an OpenAlex FILTER.
902
+
903
+ This does not run the filter right away. You need
904
+ to manually update the filters later.
905
+ """
906
+ db.execute(
907
+ "insert into queries(filter, description) values (?, ?)", (_filter, description)
908
+ )
909
+ db.commit()
910
+
911
+
912
+ @cli.command()
913
+ @click.argument("_filter")
914
+ def rm_filter(_filter):
915
+ """Remove an OpenAlex FILTER."""
916
+ db.execute("delete from queries where filter = ?", (_filter,))
917
+ db.commit()
918
+
919
+
920
+ update_filter_fmt = """** {{ extra['display_name'] | replace("\n", "") | replace("\r", "") }}
921
+ :PROPERTIES:
922
+ :SOURCE: {{ source }}
923
+ :REFERENCE_COUNT: {{ extra.get('referenced_works_count', 0) }}
924
+ :CITED_BY_COUNT: {{ extra.get('cited_by_count', 0) }}
925
+ :END:
926
+
927
+ litdb:{{ source }}
928
+
929
+ {{ text }}
930
+
931
+ """
932
+
933
+
934
+ @cli.command()
935
+ @click.option("-f", "--fmt", default=update_filter_fmt)
936
+ @click.option("-s", "--silent", is_flag=True, default=False)
937
+ def update_filters(fmt, silent):
938
+ """Update litdb using a filter with works from a created date."""
939
+ filters = db.execute("""select filter, description, last_updated from queries""")
940
+ for f, description, last_updated in filters.fetchall():
941
+ results = update_filter(f, last_updated, silent)
942
+ if results:
943
+ richprint(f"* {description}")
944
+ for result in results:
945
+ source, text, extra = result
946
+ richprint(Template(fmt).render(**locals()))
947
+
948
+
949
+ list_filter_fmt = (
950
+ '{{ "{:3d}".format(rowid) }}.'
951
+ ' {{ "{:30s}".format(description'
952
+ ' or "None") }} {{ f }}'
953
+ " ({{ last_updated }})"
954
+ )
955
+
956
+
957
+ @cli.command()
958
+ @click.option("-f", "--fmt", default=list_filter_fmt)
959
+ def list_filters(fmt):
960
+ """List the filters.
961
+
962
+ FMT is a jinja template with access to the variables rowid, f, description
963
+ and last_updated. f is the filter string.
964
+
965
+ You can dump the filters to stdout like this.
966
+
967
+ > litdb list-filters -f 'litdb add-filter {{ f }}'
968
+
969
+ You could use that to send a list of your filters to someone, or to recreate
970
+ a db somewhere else.
971
+ """
972
+ filters = db.execute(
973
+ """select rowid, filter, description, last_updated
974
+ from queries"""
975
+ )
976
+ for rowid, f, description, last_updated in filters.fetchall():
977
+ richprint(Template(fmt).render(**locals()))
978
+
979
+
980
+ ######################
981
+ # OpenAlex searching #
982
+ ######################
983
+
984
+
985
+ @cli.command()
986
+ @click.argument("query", nargs=-1)
987
+ @click.option("-f", "--filter", "_filter", is_flag=True, default=False)
988
+ @click.option("-e", "--endpoint", default="works")
989
+ @click.option("--sample", default=-1)
990
+ @click.option("--per-page", default=5)
991
+ def openalex(query, _filter, endpoint, sample, per_page):
992
+ """Run an openalex query on FILTER.
993
+
994
+ ENDPOINT should be one of works, authors, or another entity.
995
+
996
+ This does not add anything to your database. It is to help you find starting
997
+ points.
998
+
999
+ To search text:
1000
+ litdb openalex "circular polymer"
1001
+
1002
+ To find a journal id with a specific filter
1003
+ litdb openalex -e sources -f "display_name.search:Digital Discovery"
1004
+
1005
+ """
1006
+ config = get_config()
1007
+ url = f"https://api.openalex.org/{endpoint}"
1008
+
1009
+ if isinstance(query, tuple):
1010
+ query = " ".join(query)
1011
+ if not _filter:
1012
+ query = f"default.search:{query}"
1013
+
1014
+ print(query)
1015
+
1016
+ next_cursor = "*"
1017
+ params = {
1018
+ "email": config["openalex"]["email"],
1019
+ "filter": query,
1020
+ "cursor": next_cursor,
1021
+ "per_page": per_page,
1022
+ }
1023
+
1024
+ if api_key := config["openalex"].get("api_key"):
1025
+ params.update(api_key=api_key)
1026
+
1027
+ if sample > 0:
1028
+ del params["sort"] # incompatible param with sample
1029
+ params.update(sample=sample)
1030
+
1031
+ while next_cursor:
1032
+ resp = requests.get(url, params)
1033
+ if resp.status_code != 200:
1034
+ print(resp.url)
1035
+ print(resp.text)
1036
+ return
1037
+
1038
+ data = resp.json()
1039
+ next_cursor = data["meta"]["next_cursor"]
1040
+ params.update(cursor=next_cursor)
1041
+
1042
+ for result in data["results"]:
1043
+ # Note sometimes there is an exception from bad markup in strings
1044
+ richprint(get_text(result))
1045
+ print()
1046
+
1047
+ if next_cursor:
1048
+ if input("Continue? (y/n)").lower().startswith("n"):
1049
+ return
1050
+
1051
+
1052
+ ########################################
1053
+ # Convenience functions to add filters #
1054
+ ########################################
1055
+
1056
+
1057
+ @cli.command()
1058
+ @click.argument("name", nargs=-1)
1059
+ def author_search(name):
1060
+ """Search OpenAlex for name.
1061
+
1062
+ Uses the autocomplete endpoint to find an author's orcid.
1063
+ """
1064
+ auname = " ".join(name)
1065
+
1066
+ url = "https://api.openalex.org/autocomplete/authors"
1067
+
1068
+ from .openalex import get_data
1069
+
1070
+ data = get_data(url, params={"q": auname})
1071
+
1072
+ for result in data["results"]:
1073
+ richprint(
1074
+ f'- {result["display_name"]}\n {result["hint"]} '
1075
+ f'{result["external_id"]}\n\n'
1076
+ )
1077
+
1078
+
1079
+ @cli.command()
1080
+ @click.argument("orcids", nargs=-1)
1081
+ @click.option("-r", "--remove", is_flag=True, help="remove")
1082
+ def follow(orcids, remove=False):
1083
+ """Add a filter to follow orcid."""
1084
+ for orcid in orcids:
1085
+ if not orcid.startswith("http"):
1086
+ orcid = f"https://orcid.org/{orcid}"
1087
+
1088
+ # Seems like we should get their articles first.
1089
+ add_author(orcid)
1090
+
1091
+ f = f"author.orcid:{orcid}"
1092
+
1093
+ if remove:
1094
+ c = db.execute("""delete from queries where filter = ?""", (f,))
1095
+ db.commit()
1096
+ richprint(f"{c.rowcount} rows removed")
1097
+ return
1098
+
1099
+ url = f"https://api.openalex.org/authors/{orcid}"
1100
+ data = get_data(url)
1101
+ name = data["display_name"]
1102
+
1103
+ today = datetime.date.today().strftime("%Y-%m-%d")
1104
+ db.execute(
1105
+ """insert or ignore into
1106
+ queries(filter, description, last_updated)
1107
+ values (?, ?, ?)""",
1108
+ (f, name, today),
1109
+ )
1110
+
1111
+ richprint(f"Following {name}: {orcid}")
1112
+ db.commit()
1113
+
1114
+
1115
+ @cli.command()
1116
+ @click.argument("query", nargs=-1)
1117
+ @click.option("-r", "--remove", is_flag=True, help="remove")
1118
+ def watch(query, remove=False):
1119
+ """Create a watch on query.
1120
+
1121
+ QUERY: string, a filter for openalex.
1122
+ REMOVE: a flag to remove the query.
1123
+ """
1124
+ # First, we should make sure the filter is valid
1125
+ query = " ".join(query)
1126
+
1127
+ if remove:
1128
+ c = db.execute("""delete from queries where filter = ?""", (query,))
1129
+ db.commit()
1130
+ richprint(f"{c.rowcount} rows removed")
1131
+ return
1132
+
1133
+ url = "https://api.openalex.org/works"
1134
+
1135
+ data = get_data(url, params={"filter": query})
1136
+ if len(data["results"]) == 0:
1137
+ richprint(f"Sorry, {query} does not seem valid.")
1138
+
1139
+ if remove:
1140
+ c = db.execute("""delete from queries where filter = ?""", (query,))
1141
+ richprint(f"Deleted {c.rowcount} rows")
1142
+ db.commit()
1143
+ else:
1144
+ c = db.execute(
1145
+ """insert or ignore into queries(filter, description)
1146
+ values (?, ?)""",
1147
+ (query,),
1148
+ )
1149
+ richprint(f"Added {c.rowcount} rows")
1150
+ db.commit()
1151
+ richprint(f"Watching {query}")
1152
+
1153
+
1154
+ @cli.command()
1155
+ @click.argument("doi")
1156
+ @click.option("-r", "--remove", is_flag=True, help="remove")
1157
+ def citing(doi, remove=False):
1158
+ """Create a watch for articles that cite doi.
1159
+
1160
+ REMOVE is a flag to remove the doi.
1161
+ """
1162
+ url = "https://api.openalex.org/works"
1163
+
1164
+ # We need an OpenAlex id
1165
+ f = f"doi:{doi}"
1166
+
1167
+ data = get_data(url, params={"filter": f})
1168
+ if len(data["results"]) == 0:
1169
+ richprint(f"Sorry, {doi} does not seem valid.")
1170
+
1171
+ wid = data["results"][0]["id"]
1172
+
1173
+ if remove:
1174
+ c = db.execute("""delete from queries where filter = ?""", (f"cites:{wid}",))
1175
+ db.commit()
1176
+ richprint(f"Deleted {c.rowcount} rows")
1177
+ else:
1178
+ c = db.execute(
1179
+ """insert or ignore into queries(filter, description)
1180
+ values (?, ?)""",
1181
+ (f"cites:{wid}", f"Citing papers for {doi}"),
1182
+ )
1183
+
1184
+ db.commit()
1185
+ richprint(f"Added {c.rowcount} rows")
1186
+
1187
+
1188
+ @cli.command()
1189
+ @click.argument("doi")
1190
+ @click.option("-r", "--remove", is_flag=True, help="remove")
1191
+ def related(doi, remove=False):
1192
+ """Create a watch for articles that are related to doi.
1193
+
1194
+ REMOVE is a flag to remove the doi from queries.
1195
+ """
1196
+ url = "https://api.openalex.org/works"
1197
+
1198
+ # We need an OpenAlex id
1199
+ f = f"doi:{doi}"
1200
+
1201
+ data = get_data(url, params={"filter": f})
1202
+ if len(data["results"]) == 0:
1203
+ richprint(f"Sorry, {doi} does not seem valid.")
1204
+
1205
+ wid = data["results"][0]["id"]
1206
+
1207
+ if remove:
1208
+ c = db.execute(
1209
+ """delete from queries where filter = ?""", (f"related_to:{wid}",)
1210
+ )
1211
+ db.commit()
1212
+ richprint(f"Deleted {c.rowcount} rows")
1213
+ else:
1214
+ c = db.execute(
1215
+ """insert or ignore into queries(filter, description)
1216
+ values (?, ?)""",
1217
+ (f"related_to:{wid}", f"Related papers for {doi}"),
1218
+ )
1219
+
1220
+ db.commit()
1221
+ richprint(f"Added {c.rowcount} rows")
1222
+
1223
+
1224
+ #############
1225
+ # Utilities #
1226
+ #############
1227
+
1228
+
1229
+ @cli.command()
1230
+ @click.argument("sources", nargs=-1)
1231
+ def bibtex(sources):
1232
+ """Generate bibtex entries for sources."""
1233
+ if not sources:
1234
+ sources = sys.stdin.read().strip().split()
1235
+
1236
+ for source in sources:
1237
+ (work,) = db.execute(
1238
+ """select extra from sources where source = ?""", (source,)
1239
+ ).fetchone()
1240
+ richprint(dump_bibtex(json.loads(work)))
1241
+
1242
+
1243
+ @cli.command()
1244
+ @click.argument("sources", nargs=-1)
1245
+ def citation(sources):
1246
+ """Generate citation strings for sources."""
1247
+ if not sources:
1248
+ sources = sys.stdin.read().strip().split()
1249
+
1250
+ for i, source in enumerate(sources):
1251
+ (_citation,) = db.execute(
1252
+ """select json_extract(extra, '$.citation')
1253
+ from sources where source = ?""",
1254
+ (source,),
1255
+ ).fetchone()
1256
+ richprint(f"{i + 1:2d}. {_citation}")
1257
+
1258
+
1259
+ @cli.command()
1260
+ @click.argument("doi")
1261
+ def unpaywall(doi):
1262
+ """Use unpaywall to find PDFs for doi."""
1263
+ config = get_config()
1264
+ url = f"https://api.unpaywall.org/v2/{doi}"
1265
+ params = {"email": config["openalex"]["email"]}
1266
+
1267
+ resp = requests.get(url, params)
1268
+ if resp.status_code == 200:
1269
+ data = resp.json()
1270
+ richprint(f'{data["title"]}, {data.get("journal_name") or ""}')
1271
+ richprint(f'Is open access: {data.get("is_oa", False)}')
1272
+
1273
+ for loc in data.get("oa_locations", []):
1274
+ richprint(loc.get("url_for_pdf") or loc.get("url_for_landing_page"))
1275
+ else:
1276
+ richprint(f"{doi} not found in unpaywall")
1277
+
1278
+
1279
+ @cli.command()
1280
+ def about():
1281
+ """Summary statistics of your db."""
1282
+ config = get_config()
1283
+ dbf = os.path.join(config["root"], "litdb.libsql")
1284
+
1285
+ richprint(f"Your database is located at {dbf}")
1286
+ kb = 1024
1287
+ mb = 1024 * kb
1288
+ gb = 1024 * mb
1289
+ richprint(f"Database size: {os.path.getsize(dbf) / gb:1.2f} GB")
1290
+ db = get_db()
1291
+ (nsources,) = db.execute("select count(source) from sources").fetchone()
1292
+ richprint(f"You have {nsources} sources")
1293
+
1294
+
1295
+ @cli.command()
1296
+ @click.argument("sql")
1297
+ def sql(sql):
1298
+ """Run the SQL command on the db."""
1299
+ for row in db.execute(sql).fetchall():
1300
+ richprint(row)
1301
+
1302
+
1303
+ @cli.command()
1304
+ @click.argument("sources", nargs=-1)
1305
+ @click.option("-f", "--fmt", default="{{ source }}\n{{ text }}")
1306
+ def show(sources, fmt):
1307
+ """Show the source.
1308
+
1309
+ FMT is a jinja template with access to source, text, extra for each arg.
1310
+ """
1311
+ for src in sources:
1312
+ result = db.execute(
1313
+ """select source, text, extra from
1314
+ sources where source = ?""",
1315
+ (src,),
1316
+ ).fetchone()
1317
+
1318
+ if result:
1319
+ source, text, extra = result
1320
+ extra = json.loads(extra)
1321
+ richprint(Template(fmt).render(**locals()))
1322
+ else:
1323
+ print(f"Nothing found for {src}")
1324
+
1325
+
1326
+ @cli.command(name="open")
1327
+ @click.argument("source")
1328
+ def visit(source):
1329
+ """Open source."""
1330
+ if source.startswith("http"):
1331
+ webbrowser.open(source, new=2)
1332
+ elif source.endswith(".pdf"):
1333
+ webbrowser.open(f"file://{source}")
1334
+ else:
1335
+ webbrowser.open(f"file://{source}")
1336
+
1337
+
1338
+ @cli.command()
1339
+ def update_embeddings():
1340
+ """Update all the embeddings in your db.
1341
+
1342
+ The only reason you would do this is if you change the embedding model, or
1343
+ the way the chunks are sized in your config.
1344
+
1345
+ """
1346
+ config = get_config()
1347
+ db = get_db()
1348
+ from sentence_transformers import SentenceTransformer
1349
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
1350
+
1351
+ model = SentenceTransformer(config["embedding"]["model"])
1352
+ splitter = RecursiveCharacterTextSplitter(
1353
+ chunk_size=config["embedding"]["chunk_size"],
1354
+ chunk_overlap=config["embedding"]["chunk_overlap"],
1355
+ )
1356
+
1357
+ _, dim = model.encode(["test"]).shape
1358
+
1359
+ # The point of this is to avoid deleting the database.
1360
+ db.execute("drop index if exists embedding_idx")
1361
+ db.execute("alter table sources drop embedding")
1362
+ db.execute(f"alter table sources add column embedding F32_BLOB({dim})")
1363
+ db.commit()
1364
+
1365
+ for rowid, text in db.execute("select rowid, text from sources").fetchall():
1366
+ chunks = splitter.split_text(text)
1367
+ embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
1368
+
1369
+ c = db.execute(
1370
+ "update sources set embedding = ? where rowid = ?", (embedding, rowid)
1371
+ )
1372
+ print(rowid, c.rowcount)
1373
+
1374
+ # I don't know why this has to be here. I had it above, and no updates were
1375
+ # happening.
1376
+ db.execute(
1377
+ """create index if not exists embedding_idx ON sources (libsql_vector_idx(embedding))"""
1378
+ )
1379
+ db.commit()
1380
+
1381
+
1382
+ ######################
1383
+ # Academic functions #
1384
+ ######################
1385
+
1386
+
1387
+ @cli.command()
1388
+ @click.argument("orcid")
1389
+ def coa(orcid):
1390
+ """Generate Table 4 of Collaborators and Other Affiliations for NSF.
1391
+
1392
+ ORCID is an orcid URL for the user to generate the table for.
1393
+ The file is saved in {orcid}-{today}.xlsx.
1394
+ """
1395
+ from .coa import get_coa
1396
+
1397
+ get_coa(orcid)
1398
+
1399
+
1400
+ @cli.command()
1401
+ @click.argument("query", nargs=-1)
1402
+ @click.option("-n", default=5, help="Number of documents to use")
1403
+ def suggest_reviewers(query, n):
1404
+ """Suggest reviewers for QUERY.
1405
+
1406
+ Use up to N similar documents. This is an iterative function, you will be
1407
+ prompted to expand the search.
1408
+ """
1409
+ config = get_config()
1410
+ query = " ".join(query)
1411
+
1412
+ # This is a surprise. You can't just call the functions above! This is
1413
+ # apparently the way to do this.
1414
+ with click.Context(vsearch) as ctx:
1415
+ results = ctx.invoke(
1416
+ vsearch, query=query.split(" "), n=n, fmt="", iterative=True
1417
+ )
1418
+
1419
+ # Now collect the authors from the matching papers
1420
+ authors = []
1421
+
1422
+ for i, row in enumerate(results):
1423
+ source, citation, extra, distance = row
1424
+
1425
+ d = json.loads(extra)
1426
+
1427
+ for authorship in d["authorships"]:
1428
+ authors += [authorship["author"]["id"]]
1429
+
1430
+ # get the unique ones
1431
+ from collections import Counter
1432
+
1433
+ authors = Counter(authors)
1434
+
1435
+ # Get author information
1436
+ data = []
1437
+
1438
+ url = "https://api.openalex.org/authors/"
1439
+ # You can only filter on 50 ids at a time, so we hard code this limit here
1440
+ # and per page.
1441
+ for batch in batched(authors, 50):
1442
+ url = f'https://api.openalex.org/authors?filter=id:{"|".join(batch)}'
1443
+
1444
+ params = {"per-page": 50, "email": config["openalex"]["email"]}
1445
+
1446
+ r = get_data(url, params)
1447
+
1448
+ for d in r["results"]:
1449
+ lki = d.get("last_known_institutions", [])
1450
+ if lki == []:
1451
+ affils = d.get("affiliations", [])
1452
+ if len(affils) >= 0:
1453
+ lki = affils[0]["institution"]["display_name"]
1454
+ else:
1455
+ lki = "unknown"
1456
+
1457
+ else:
1458
+ if len(lki) >= 1:
1459
+ lki = lki[0].get("display_name")
1460
+ else:
1461
+ lki = "unknown"
1462
+
1463
+ row = [
1464
+ d["display_name"],
1465
+ authors[d["id"]],
1466
+ d["summary_stats"]["h_index"],
1467
+ d["id"],
1468
+ lki,
1469
+ ]
1470
+ data += [row]
1471
+
1472
+ # Sort and display the results
1473
+ data.sort(key=lambda row: row[2], reverse=True)
1474
+
1475
+ if get_ipython():
1476
+ display(
1477
+ HTML(
1478
+ tabulate.tabulate(
1479
+ data,
1480
+ headers=["name", "# papers", "h-index", "oaid", "institution"],
1481
+ tablefmt="html",
1482
+ )
1483
+ )
1484
+ )
1485
+ for i, row in enumerate(results):
1486
+ source, citation, extra, distance = row
1487
+ richprint(f"{i + 1:2d}. {citation} (source)\n\n")
1488
+
1489
+ else:
1490
+ s = ["Potential reviewers"]
1491
+ s += [
1492
+ tabulate.tabulate(
1493
+ data,
1494
+ headers=["name", "# papers", "h-index", "oaid", "institution"],
1495
+ tablefmt="orgtbl",
1496
+ )
1497
+ ]
1498
+ s += ["\n" + "From these papers:"]
1499
+ for i, row in enumerate(results):
1500
+ source, citation, extra, distance = row
1501
+ s += [f"{i + 1:2d}. {citation} (source)\n\n"]
1502
+
1503
+ console = Console(color_system="truecolor")
1504
+ with console.pager():
1505
+ for _s in s:
1506
+ console.print(_s)
1507
+
1508
+
1509
+ if __name__ == "__main__":
1510
+ cli()