litdb 2.0.2__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {litdb-2.0.2/src/litdb.egg-info → litdb-2.1.0}/PKG-INFO +73 -27
- litdb-2.0.2/PKG-INFO → litdb-2.1.0/README.org +65 -62
- {litdb-2.0.2 → litdb-2.1.0}/pyproject.toml +13 -3
- litdb-2.1.0/src/litdb/app.py +67 -0
- litdb-2.1.0/src/litdb/audio.py +139 -0
- litdb-2.1.0/src/litdb/chat.py +357 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/cli.py +56 -84
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/db.py +8 -1
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/gpt.py +0 -2
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/utils.py +1 -1
- litdb-2.1.0/src/litdb/youtube.py +48 -0
- litdb-2.0.2/README.org → litdb-2.1.0/src/litdb.egg-info/PKG-INFO +108 -25
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb.egg-info/SOURCES.txt +4 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb.egg-info/requires.txt +6 -0
- {litdb-2.0.2 → litdb-2.1.0}/setup.cfg +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/__init__.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/bibtex.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/coa.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/lab.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/openalex.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb/pdf.py +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb.egg-info/entry_points.txt +0 -0
- {litdb-2.0.2 → litdb-2.1.0}/src/litdb.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,10 +11,12 @@ Requires-Dist: toml
|
|
|
11
11
|
Requires-Dist: tomlkit
|
|
12
12
|
Requires-Dist: requests
|
|
13
13
|
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: docling
|
|
14
15
|
Requires-Dist: langchain
|
|
15
16
|
Requires-Dist: tqdm
|
|
16
17
|
Requires-Dist: libsql-experimental
|
|
17
18
|
Requires-Dist: sentence-transformers
|
|
19
|
+
Requires-Dist: transformers
|
|
18
20
|
Requires-Dist: click
|
|
19
21
|
Requires-Dist: rich
|
|
20
22
|
Requires-Dist: beautifulsoup4
|
|
@@ -24,6 +26,7 @@ Requires-Dist: python-pptx
|
|
|
24
26
|
Requires-Dist: nbconvert
|
|
25
27
|
Requires-Dist: nbformat
|
|
26
28
|
Requires-Dist: ollama
|
|
29
|
+
Requires-Dist: litellm
|
|
27
30
|
Requires-Dist: dateparser
|
|
28
31
|
Requires-Dist: jinja2
|
|
29
32
|
Requires-Dist: pytesseract
|
|
@@ -34,6 +37,9 @@ Requires-Dist: more-itertools
|
|
|
34
37
|
Requires-Dist: nameparser
|
|
35
38
|
Requires-Dist: tabulate
|
|
36
39
|
Requires-Dist: IPython
|
|
40
|
+
Requires-Dist: yt_dlp
|
|
41
|
+
Requires-Dist: webvtt-py
|
|
42
|
+
Requires-Dist: streamlit
|
|
37
43
|
|
|
38
44
|
#+title: litdb - a literature and document database
|
|
39
45
|
|
|
@@ -74,7 +80,7 @@ pip install git+https://github.com/jkitchin/litdb
|
|
|
74
80
|
|
|
75
81
|
You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
|
|
76
82
|
|
|
77
|
-
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
83
|
+
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
78
84
|
|
|
79
85
|
You will need an OpenAlex premium key if you want to use the update-filters feature.
|
|
80
86
|
|
|
@@ -92,6 +98,9 @@ api_key = "..."
|
|
|
92
98
|
|
|
93
99
|
[gpt]
|
|
94
100
|
model = "llama2"
|
|
101
|
+
|
|
102
|
+
[llm]
|
|
103
|
+
model = "ollama/llama2"
|
|
95
104
|
#+END_EXAMPLE
|
|
96
105
|
|
|
97
106
|
You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
|
|
@@ -102,7 +111,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
|
|
|
102
111
|
|
|
103
112
|
When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
|
|
104
113
|
|
|
105
|
-
* Using litdb
|
|
114
|
+
* Using litdb
|
|
106
115
|
|
|
107
116
|
Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
|
|
108
117
|
|
|
@@ -156,7 +165,7 @@ litdb openalex -e sources -f display_name.search:discovery
|
|
|
156
165
|
|
|
157
166
|
** One-time additions of articles to litdb
|
|
158
167
|
|
|
159
|
-
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
168
|
+
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
160
169
|
|
|
161
170
|
#+BEGIN_SRC sh
|
|
162
171
|
litdb add doi --references --citing --related
|
|
@@ -182,6 +191,8 @@ litdb add doi1 doi2 orcid
|
|
|
182
191
|
|
|
183
192
|
These are all one-time additions.
|
|
184
193
|
|
|
194
|
+
You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
|
|
195
|
+
|
|
185
196
|
** Adding filters
|
|
186
197
|
|
|
187
198
|
litdb provides several convenient ways to add queries to update your litdb in the future.
|
|
@@ -244,7 +255,7 @@ litdb rm-filter "filter-string"
|
|
|
244
255
|
|
|
245
256
|
** Review your litdb
|
|
246
257
|
|
|
247
|
-
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
258
|
+
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
248
259
|
|
|
249
260
|
#+BEGIN_SRC sh
|
|
250
261
|
litdb review -s "1 week ago"
|
|
@@ -254,14 +265,14 @@ This works best when you update your litdb regularly. You might want to redirect
|
|
|
254
265
|
|
|
255
266
|
** Searching litdb
|
|
256
267
|
|
|
257
|
-
There are several search options.
|
|
268
|
+
There are several search options.
|
|
258
269
|
|
|
259
270
|
*** vector search
|
|
260
271
|
|
|
261
272
|
The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
|
|
262
273
|
|
|
263
274
|
#+BEGIN_SRC sh
|
|
264
|
-
litdb vsearch "natural language query"
|
|
275
|
+
litdb vsearch "natural language query"
|
|
265
276
|
#+END_SRC
|
|
266
277
|
|
|
267
278
|
The default number of entries returned is 3. You can change that with an optional argument
|
|
@@ -312,6 +323,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
|
|
|
312
323
|
litdb gpt "what is the state of the art in automated laboratories for soft materials"
|
|
313
324
|
#+END_SRC
|
|
314
325
|
|
|
326
|
+
*** Integration with litellm
|
|
327
|
+
|
|
328
|
+
litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
|
|
329
|
+
|
|
330
|
+
The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
|
|
331
|
+
|
|
332
|
+
It uses a different command than the ollama gpt command.
|
|
333
|
+
|
|
334
|
+
#+BEGIN_SRC sh
|
|
335
|
+
litdb chat "what is the state of the art in automated laboratories for soft materials"
|
|
336
|
+
#+END_SRC
|
|
337
|
+
|
|
338
|
+
There are some fancy things you can do with the prompt:
|
|
339
|
+
|
|
340
|
+
1. Avoid using RAG if --norag is in your prompt.
|
|
341
|
+
2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
|
|
342
|
+
3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
|
|
343
|
+
4. A prompt of !save will save the current chat to a file.
|
|
344
|
+
5. You can use this syntax to expand a file or url in the prompt for context:
|
|
345
|
+
#+BEGIN_EXAMPLE
|
|
346
|
+
[[file/url]]
|
|
347
|
+
#+END_EXAMPLE
|
|
348
|
+
|
|
349
|
+
Your prompt history is saved in your litdb, so you can go back to them if you want.
|
|
350
|
+
|
|
351
|
+
*** Web app for litdb
|
|
352
|
+
|
|
353
|
+
If you prefer a browser, you can now launch a streamlit app for litdb:
|
|
354
|
+
|
|
355
|
+
#+BEGIN_SRC sh
|
|
356
|
+
litdb app
|
|
357
|
+
#+END_SRC
|
|
358
|
+
|
|
359
|
+
This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
|
|
360
|
+
|
|
315
361
|
*** search with audio
|
|
316
362
|
|
|
317
363
|
This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
|
|
@@ -441,7 +487,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
|
|
|
441
487
|
|
|
442
488
|
* Adding local files
|
|
443
489
|
|
|
444
|
-
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
490
|
+
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
445
491
|
|
|
446
492
|
It is possible to add any file that can be turned into text to litdb. That includes:
|
|
447
493
|
|
|
@@ -456,11 +502,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
|
|
|
456
502
|
|
|
457
503
|
This limits portability because you need a path if you want to be able to open that file.
|
|
458
504
|
|
|
459
|
-
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
505
|
+
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
460
506
|
|
|
461
|
-
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
507
|
+
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
462
508
|
|
|
463
|
-
You can combine this with the OpenAlex entries in a single database.
|
|
509
|
+
You can combine this with the OpenAlex entries in a single database.
|
|
464
510
|
|
|
465
511
|
You can walk a directory and add files from it with this command.
|
|
466
512
|
|
|
@@ -484,19 +530,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
|
|
|
484
530
|
|
|
485
531
|
[[litdb:https://doi.org/10.1021/jp047349j]]
|
|
486
532
|
|
|
487
|
-
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
533
|
+
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
488
534
|
|
|
489
535
|
You can easily insert a link like this:
|
|
490
536
|
|
|
491
537
|
M-x litdb
|
|
492
538
|
|
|
493
|
-
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
539
|
+
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
494
540
|
|
|
495
|
-
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
541
|
+
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
496
542
|
|
|
497
543
|
* Database design
|
|
498
544
|
|
|
499
|
-
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
545
|
+
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
500
546
|
|
|
501
547
|
The main table in litdb is called sources.
|
|
502
548
|
- sources
|
|
@@ -551,7 +597,7 @@ Similar limitations exist for local files. There are additionally the following
|
|
|
551
597
|
1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
|
|
552
598
|
2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
|
|
553
599
|
|
|
554
|
-
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
600
|
+
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
555
601
|
|
|
556
602
|
There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
|
|
557
603
|
|
|
@@ -569,7 +615,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
|
|
|
569
615
|
|
|
570
616
|
sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
|
|
571
617
|
|
|
572
|
-
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
618
|
+
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
573
619
|
|
|
574
620
|
* Roadmap
|
|
575
621
|
|
|
@@ -583,7 +629,7 @@ Notes on the other hand, might be small enough that they could be stored in the
|
|
|
583
629
|
|
|
584
630
|
** Jupyter lab integration
|
|
585
631
|
|
|
586
|
-
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
632
|
+
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
587
633
|
|
|
588
634
|
** graph visualization
|
|
589
635
|
|
|
@@ -655,7 +701,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
|
|
|
655
701
|
|
|
656
702
|
** image and text models
|
|
657
703
|
|
|
658
|
-
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
704
|
+
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
659
705
|
|
|
660
706
|
** DONE combine full text and vector search
|
|
661
707
|
CLOSED: [2024-12-09 Mon 13:59]
|
|
@@ -737,11 +783,11 @@ print(text)
|
|
|
737
783
|
#+RESULTS:
|
|
738
784
|
: ++RESULTS:
|
|
739
785
|
: ; Selected region: (26, 332, 473, 69)
|
|
740
|
-
:
|
|
786
|
+
:
|
|
741
787
|
|
|
742
788
|
This might be nice later when we have image embeddings.
|
|
743
789
|
|
|
744
|
-
** DONE review process
|
|
790
|
+
** DONE review process
|
|
745
791
|
|
|
746
792
|
#+BEGIN_SRC sh
|
|
747
793
|
litdb review --since '1 week ago'
|
|
@@ -761,7 +807,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
|
|
|
761
807
|
| https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
|
|
762
808
|
| https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
|
|
763
809
|
|
|
764
|
-
** DONE semantic similarity
|
|
810
|
+
** DONE semantic similarity
|
|
765
811
|
CLOSED: [2024-12-04 Wed 13:12]
|
|
766
812
|
|
|
767
813
|
litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
|
|
@@ -772,12 +818,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
|
|
|
772
818
|
|
|
773
819
|
- LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
|
|
774
820
|
- Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
|
|
775
|
-
|
|
821
|
+
|
|
776
822
|
- paper-qa :: https://github.com/Future-House/paper-qa
|
|
777
823
|
- This project by Andrew White uses LLM+RAG to explore a paper.
|
|
778
824
|
|
|
779
825
|
- ColBERT :: https://github.com/stanford-futuredata/ColBERT
|
|
780
|
-
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
826
|
+
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
781
827
|
|
|
782
828
|
Many of these projects require you to make an account. There are freemium levels in each one.
|
|
783
829
|
|
|
@@ -789,7 +835,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
789
835
|
|
|
790
836
|
- Keenious :: https://keenious.com/explore
|
|
791
837
|
- Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
|
|
792
|
-
|
|
838
|
+
|
|
793
839
|
- scite.ai :: https://scite.ai/
|
|
794
840
|
- Browser tool that integrates GPT with the scientific literature, integration with Zotero
|
|
795
841
|
|
|
@@ -803,7 +849,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
803
849
|
- This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
|
|
804
850
|
|
|
805
851
|
- AnythingLLM :: https://anythingllm.com/
|
|
806
|
-
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
852
|
+
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
807
853
|
|
|
808
854
|
- gpt4all :: https://www.nomic.ai/gpt4all
|
|
809
855
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
@@ -1,40 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: litdb
|
|
3
|
-
Version: 2.0.2
|
|
4
|
-
Summary: A literature database tool with GPT integration.
|
|
5
|
-
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
|
-
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
7
|
-
License: MIT
|
|
8
|
-
Project-URL: Homepage, https://github.com/jkitchin/litdb
|
|
9
|
-
Description-Content-Type: text/plain
|
|
10
|
-
Requires-Dist: toml
|
|
11
|
-
Requires-Dist: tomlkit
|
|
12
|
-
Requires-Dist: requests
|
|
13
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
-
Requires-Dist: langchain
|
|
15
|
-
Requires-Dist: tqdm
|
|
16
|
-
Requires-Dist: libsql-experimental
|
|
17
|
-
Requires-Dist: sentence-transformers
|
|
18
|
-
Requires-Dist: click
|
|
19
|
-
Requires-Dist: rich
|
|
20
|
-
Requires-Dist: beautifulsoup4
|
|
21
|
-
Requires-Dist: ratelimit
|
|
22
|
-
Requires-Dist: python-docx
|
|
23
|
-
Requires-Dist: python-pptx
|
|
24
|
-
Requires-Dist: nbconvert
|
|
25
|
-
Requires-Dist: nbformat
|
|
26
|
-
Requires-Dist: ollama
|
|
27
|
-
Requires-Dist: dateparser
|
|
28
|
-
Requires-Dist: jinja2
|
|
29
|
-
Requires-Dist: pytesseract
|
|
30
|
-
Requires-Dist: pillow
|
|
31
|
-
Requires-Dist: bibtexparser
|
|
32
|
-
Requires-Dist: pandas
|
|
33
|
-
Requires-Dist: more-itertools
|
|
34
|
-
Requires-Dist: nameparser
|
|
35
|
-
Requires-Dist: tabulate
|
|
36
|
-
Requires-Dist: IPython
|
|
37
|
-
|
|
38
1
|
#+title: litdb - a literature and document database
|
|
39
2
|
|
|
40
3
|
#+attr_org: :width 600
|
|
@@ -74,7 +37,7 @@ pip install git+https://github.com/jkitchin/litdb
|
|
|
74
37
|
|
|
75
38
|
You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
|
|
76
39
|
|
|
77
|
-
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
40
|
+
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
78
41
|
|
|
79
42
|
You will need an OpenAlex premium key if you want to use the update-filters feature.
|
|
80
43
|
|
|
@@ -92,6 +55,9 @@ api_key = "..."
|
|
|
92
55
|
|
|
93
56
|
[gpt]
|
|
94
57
|
model = "llama2"
|
|
58
|
+
|
|
59
|
+
[llm]
|
|
60
|
+
model = "ollama/llama2"
|
|
95
61
|
#+END_EXAMPLE
|
|
96
62
|
|
|
97
63
|
You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
|
|
@@ -102,7 +68,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
|
|
|
102
68
|
|
|
103
69
|
When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
|
|
104
70
|
|
|
105
|
-
* Using litdb
|
|
71
|
+
* Using litdb
|
|
106
72
|
|
|
107
73
|
Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
|
|
108
74
|
|
|
@@ -156,7 +122,7 @@ litdb openalex -e sources -f display_name.search:discovery
|
|
|
156
122
|
|
|
157
123
|
** One-time additions of articles to litdb
|
|
158
124
|
|
|
159
|
-
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
125
|
+
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
160
126
|
|
|
161
127
|
#+BEGIN_SRC sh
|
|
162
128
|
litdb add doi --references --citing --related
|
|
@@ -182,6 +148,8 @@ litdb add doi1 doi2 orcid
|
|
|
182
148
|
|
|
183
149
|
These are all one-time additions.
|
|
184
150
|
|
|
151
|
+
You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
|
|
152
|
+
|
|
185
153
|
** Adding filters
|
|
186
154
|
|
|
187
155
|
litdb provides several convenient ways to add queries to update your litdb in the future.
|
|
@@ -244,7 +212,7 @@ litdb rm-filter "filter-string"
|
|
|
244
212
|
|
|
245
213
|
** Review your litdb
|
|
246
214
|
|
|
247
|
-
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
215
|
+
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
248
216
|
|
|
249
217
|
#+BEGIN_SRC sh
|
|
250
218
|
litdb review -s "1 week ago"
|
|
@@ -254,14 +222,14 @@ This works best when you update your litdb regularly. You might want to redirect
|
|
|
254
222
|
|
|
255
223
|
** Searching litdb
|
|
256
224
|
|
|
257
|
-
There are several search options.
|
|
225
|
+
There are several search options.
|
|
258
226
|
|
|
259
227
|
*** vector search
|
|
260
228
|
|
|
261
229
|
The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
|
|
262
230
|
|
|
263
231
|
#+BEGIN_SRC sh
|
|
264
|
-
litdb vsearch "natural language query"
|
|
232
|
+
litdb vsearch "natural language query"
|
|
265
233
|
#+END_SRC
|
|
266
234
|
|
|
267
235
|
The default number of entries returned is 3. You can change that with an optional argument
|
|
@@ -312,6 +280,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
|
|
|
312
280
|
litdb gpt "what is the state of the art in automated laboratories for soft materials"
|
|
313
281
|
#+END_SRC
|
|
314
282
|
|
|
283
|
+
*** Integration with litellm
|
|
284
|
+
|
|
285
|
+
litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
|
|
286
|
+
|
|
287
|
+
The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
|
|
288
|
+
|
|
289
|
+
It uses a different command than the ollama gpt command.
|
|
290
|
+
|
|
291
|
+
#+BEGIN_SRC sh
|
|
292
|
+
litdb chat "what is the state of the art in automated laboratories for soft materials"
|
|
293
|
+
#+END_SRC
|
|
294
|
+
|
|
295
|
+
There are some fancy things you can do with the prompt:
|
|
296
|
+
|
|
297
|
+
1. Avoid using RAG if --norag is in your prompt.
|
|
298
|
+
2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
|
|
299
|
+
3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
|
|
300
|
+
4. A prompt of !save will save the current chat to a file.
|
|
301
|
+
5. You can use this syntax to expand a file or url in the prompt for context:
|
|
302
|
+
#+BEGIN_EXAMPLE
|
|
303
|
+
[[file/url]]
|
|
304
|
+
#+END_EXAMPLE
|
|
305
|
+
|
|
306
|
+
Your prompt history is saved in your litdb, so you can go back to them if you want.
|
|
307
|
+
|
|
308
|
+
*** Web app for litdb
|
|
309
|
+
|
|
310
|
+
If you prefer a browser, you can now launch a streamlit app for litdb:
|
|
311
|
+
|
|
312
|
+
#+BEGIN_SRC sh
|
|
313
|
+
litdb app
|
|
314
|
+
#+END_SRC
|
|
315
|
+
|
|
316
|
+
This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
|
|
317
|
+
|
|
315
318
|
*** search with audio
|
|
316
319
|
|
|
317
320
|
This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
|
|
@@ -441,7 +444,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
|
|
|
441
444
|
|
|
442
445
|
* Adding local files
|
|
443
446
|
|
|
444
|
-
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
447
|
+
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
445
448
|
|
|
446
449
|
It is possible to add any file that can be turned into text to litdb. That includes:
|
|
447
450
|
|
|
@@ -456,11 +459,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
|
|
|
456
459
|
|
|
457
460
|
This limits portability because you need a path if you want to be able to open that file.
|
|
458
461
|
|
|
459
|
-
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
462
|
+
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
460
463
|
|
|
461
|
-
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
464
|
+
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
462
465
|
|
|
463
|
-
You can combine this with the OpenAlex entries in a single database.
|
|
466
|
+
You can combine this with the OpenAlex entries in a single database.
|
|
464
467
|
|
|
465
468
|
You can walk a directory and add files from it with this command.
|
|
466
469
|
|
|
@@ -484,19 +487,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
|
|
|
484
487
|
|
|
485
488
|
[[litdb:https://doi.org/10.1021/jp047349j]]
|
|
486
489
|
|
|
487
|
-
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
490
|
+
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
488
491
|
|
|
489
492
|
You can easily insert a link like this:
|
|
490
493
|
|
|
491
494
|
M-x litdb
|
|
492
495
|
|
|
493
|
-
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
496
|
+
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
494
497
|
|
|
495
|
-
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
498
|
+
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
496
499
|
|
|
497
500
|
* Database design
|
|
498
501
|
|
|
499
|
-
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
502
|
+
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
500
503
|
|
|
501
504
|
The main table in litdb is called sources.
|
|
502
505
|
- sources
|
|
@@ -551,7 +554,7 @@ Similar limitations exist for local files. There are additionally the following
|
|
|
551
554
|
1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
|
|
552
555
|
2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
|
|
553
556
|
|
|
554
|
-
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
557
|
+
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
555
558
|
|
|
556
559
|
There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
|
|
557
560
|
|
|
@@ -569,7 +572,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
|
|
|
569
572
|
|
|
570
573
|
sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
|
|
571
574
|
|
|
572
|
-
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
575
|
+
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
573
576
|
|
|
574
577
|
* Roadmap
|
|
575
578
|
|
|
@@ -583,7 +586,7 @@ Notes on the other hand, might be small enough that they could be stored in the
|
|
|
583
586
|
|
|
584
587
|
** Jupyter lab integration
|
|
585
588
|
|
|
586
|
-
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
589
|
+
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
587
590
|
|
|
588
591
|
** graph visualization
|
|
589
592
|
|
|
@@ -655,7 +658,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
|
|
|
655
658
|
|
|
656
659
|
** image and text models
|
|
657
660
|
|
|
658
|
-
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
661
|
+
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
659
662
|
|
|
660
663
|
** DONE combine full text and vector search
|
|
661
664
|
CLOSED: [2024-12-09 Mon 13:59]
|
|
@@ -737,11 +740,11 @@ print(text)
|
|
|
737
740
|
#+RESULTS:
|
|
738
741
|
: ++RESULTS:
|
|
739
742
|
: ; Selected region: (26, 332, 473, 69)
|
|
740
|
-
:
|
|
743
|
+
:
|
|
741
744
|
|
|
742
745
|
This might be nice later when we have image embeddings.
|
|
743
746
|
|
|
744
|
-
** DONE review process
|
|
747
|
+
** DONE review process
|
|
745
748
|
|
|
746
749
|
#+BEGIN_SRC sh
|
|
747
750
|
litdb review --since '1 week ago'
|
|
@@ -761,7 +764,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
|
|
|
761
764
|
| https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
|
|
762
765
|
| https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
|
|
763
766
|
|
|
764
|
-
** DONE semantic similarity
|
|
767
|
+
** DONE semantic similarity
|
|
765
768
|
CLOSED: [2024-12-04 Wed 13:12]
|
|
766
769
|
|
|
767
770
|
litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
|
|
@@ -772,12 +775,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
|
|
|
772
775
|
|
|
773
776
|
- LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
|
|
774
777
|
- Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
|
|
775
|
-
|
|
778
|
+
|
|
776
779
|
- paper-qa :: https://github.com/Future-House/paper-qa
|
|
777
780
|
- This project by Andrew White uses LLM+RAG to explore a paper.
|
|
778
781
|
|
|
779
782
|
- ColBERT :: https://github.com/stanford-futuredata/ColBERT
|
|
780
|
-
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
783
|
+
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
781
784
|
|
|
782
785
|
Many of these projects require you to make an account. There are freemium levels in each one.
|
|
783
786
|
|
|
@@ -789,7 +792,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
789
792
|
|
|
790
793
|
- Keenious :: https://keenious.com/explore
|
|
791
794
|
- Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
|
|
792
|
-
|
|
795
|
+
|
|
793
796
|
- scite.ai :: https://scite.ai/
|
|
794
797
|
- Browser tool that integrates GPT with the scientific literature, integration with Zotero
|
|
795
798
|
|
|
@@ -803,7 +806,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
803
806
|
- This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
|
|
804
807
|
|
|
805
808
|
- AnythingLLM :: https://anythingllm.com/
|
|
806
|
-
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
809
|
+
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
807
810
|
|
|
808
811
|
- gpt4all :: https://www.nomic.ai/gpt4all
|
|
809
812
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
@@ -7,7 +7,7 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "litdb"
|
|
10
|
-
version = "2.0
|
|
10
|
+
version = "2.1.0"
|
|
11
11
|
authors = [
|
|
12
12
|
{name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
|
|
13
13
|
]
|
|
@@ -22,10 +22,12 @@ dependencies = [
|
|
|
22
22
|
"tomlkit",
|
|
23
23
|
"requests",
|
|
24
24
|
"pymupdf4llm",
|
|
25
|
+
"docling",
|
|
25
26
|
"langchain",
|
|
26
27
|
"tqdm",
|
|
27
28
|
"libsql-experimental",
|
|
28
29
|
"sentence-transformers",
|
|
30
|
+
"transformers",
|
|
29
31
|
"click",
|
|
30
32
|
"rich",
|
|
31
33
|
"beautifulsoup4",
|
|
@@ -35,12 +37,13 @@ dependencies = [
|
|
|
35
37
|
"nbconvert",
|
|
36
38
|
"nbformat",
|
|
37
39
|
"ollama",
|
|
40
|
+
"litellm",
|
|
38
41
|
"dateparser",
|
|
39
42
|
"jinja2",
|
|
40
43
|
"pytesseract",
|
|
41
44
|
"pillow",
|
|
42
45
|
"bibtexparser",
|
|
43
|
-
|
|
46
|
+
|
|
44
47
|
# "pyaudio",
|
|
45
48
|
# "playsound",
|
|
46
49
|
# "SpeechRecognition",
|
|
@@ -50,7 +53,14 @@ dependencies = [
|
|
|
50
53
|
"more-itertools",
|
|
51
54
|
"nameparser",
|
|
52
55
|
"tabulate",
|
|
53
|
-
"IPython"
|
|
56
|
+
"IPython",
|
|
57
|
+
|
|
58
|
+
# youtube
|
|
59
|
+
"yt_dlp",
|
|
60
|
+
"webvtt-py",
|
|
61
|
+
|
|
62
|
+
# the streamlit app
|
|
63
|
+
"streamlit"
|
|
54
64
|
]
|
|
55
65
|
|
|
56
66
|
[project.scripts]
|