litdb 2.0.2__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: litdb
3
- Version: 2.0.2
3
+ Version: 2.1.0
4
4
  Summary: A literature database tool with GPT integration.
5
5
  Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
6
6
  Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
@@ -11,10 +11,12 @@ Requires-Dist: toml
11
11
  Requires-Dist: tomlkit
12
12
  Requires-Dist: requests
13
13
  Requires-Dist: pymupdf4llm
14
+ Requires-Dist: docling
14
15
  Requires-Dist: langchain
15
16
  Requires-Dist: tqdm
16
17
  Requires-Dist: libsql-experimental
17
18
  Requires-Dist: sentence-transformers
19
+ Requires-Dist: transformers
18
20
  Requires-Dist: click
19
21
  Requires-Dist: rich
20
22
  Requires-Dist: beautifulsoup4
@@ -24,6 +26,7 @@ Requires-Dist: python-pptx
24
26
  Requires-Dist: nbconvert
25
27
  Requires-Dist: nbformat
26
28
  Requires-Dist: ollama
29
+ Requires-Dist: litellm
27
30
  Requires-Dist: dateparser
28
31
  Requires-Dist: jinja2
29
32
  Requires-Dist: pytesseract
@@ -34,6 +37,9 @@ Requires-Dist: more-itertools
34
37
  Requires-Dist: nameparser
35
38
  Requires-Dist: tabulate
36
39
  Requires-Dist: IPython
40
+ Requires-Dist: yt_dlp
41
+ Requires-Dist: webvtt-py
42
+ Requires-Dist: streamlit
37
43
 
38
44
  #+title: litdb - a literature and document database
39
45
 
@@ -74,7 +80,7 @@ pip install git+https://github.com/jkitchin/litdb
74
80
 
75
81
  You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
76
82
 
77
- There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
83
+ There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
78
84
 
79
85
  You will need an OpenAlex premium key if you want to use the update-filters feature.
80
86
 
@@ -92,6 +98,9 @@ api_key = "..."
92
98
 
93
99
  [gpt]
94
100
  model = "llama2"
101
+
102
+ [llm]
103
+ model = "ollama/llama2"
95
104
  #+END_EXAMPLE
96
105
 
97
106
  You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
@@ -102,7 +111,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
102
111
 
103
112
  When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
104
113
 
105
- * Using litdb
114
+ * Using litdb
106
115
 
107
116
  Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
108
117
 
@@ -156,7 +165,7 @@ litdb openalex -e sources -f display_name.search:discovery
156
165
 
157
166
  ** One-time additions of articles to litdb
158
167
 
159
- You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
168
+ You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
160
169
 
161
170
  #+BEGIN_SRC sh
162
171
  litdb add doi --references --citing --related
@@ -182,6 +191,8 @@ litdb add doi1 doi2 orcid
182
191
 
183
192
  These are all one-time additions.
184
193
 
194
+ You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
195
+
185
196
  ** Adding filters
186
197
 
187
198
  litdb provides several convenient ways to add queries to update your litdb in the future.
@@ -244,7 +255,7 @@ litdb rm-filter "filter-string"
244
255
 
245
256
  ** Review your litdb
246
257
 
247
- I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
258
+ I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
248
259
 
249
260
  #+BEGIN_SRC sh
250
261
  litdb review -s "1 week ago"
@@ -254,14 +265,14 @@ This works best when you update your litdb regularly. You might want to redirect
254
265
 
255
266
  ** Searching litdb
256
267
 
257
- There are several search options.
268
+ There are several search options.
258
269
 
259
270
  *** vector search
260
271
 
261
272
  The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
262
273
 
263
274
  #+BEGIN_SRC sh
264
- litdb vsearch "natural language query"
275
+ litdb vsearch "natural language query"
265
276
  #+END_SRC
266
277
 
267
278
  The default number of entries returned is 3. You can change that with an optional argument
@@ -312,6 +323,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
312
323
  litdb gpt "what is the state of the art in automated laboratories for soft materials"
313
324
  #+END_SRC
314
325
 
326
+ *** Integration with litellm
327
+
328
+ litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
329
+
330
+ The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
331
+
332
+ It uses a different command than the ollama gpt command.
333
+
334
+ #+BEGIN_SRC sh
335
+ litdb chat "what is the state of the art in automated laboratories for soft materials"
336
+ #+END_SRC
337
+
338
+ There are some fancy things you can do with the prompt:
339
+
340
+ 1. Avoid using RAG if --norag is in your prompt.
341
+ 2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
342
+ 3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
343
+ 4. A prompt of !save will save the current chat to a file.
344
+ 5. You can use this syntax to expand a file or url in the prompt for context:
345
+ #+BEGIN_EXAMPLE
346
+ [[file/url]]
347
+ #+END_EXAMPLE
348
+
349
+ Your prompt history is saved in your litdb, so you can go back to them if you want.
350
+
351
+ *** Web app for litdb
352
+
353
+ If you prefer a browser, you can now launch a streamlit app for litdb:
354
+
355
+ #+BEGIN_SRC sh
356
+ litdb app
357
+ #+END_SRC
358
+
359
+ This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
360
+
315
361
  *** search with audio
316
362
 
317
363
  This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
@@ -441,7 +487,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
441
487
 
442
488
  * Adding local files
443
489
 
444
- The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
490
+ The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
445
491
 
446
492
  It is possible to add any file that can be turned into text to litdb. That includes:
447
493
 
@@ -456,11 +502,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
456
502
 
457
503
  This limits portability because you need a path if you want to be able to open that file.
458
504
 
459
- The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
505
+ The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
460
506
 
461
- An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
507
+ An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
462
508
 
463
- You can combine this with the OpenAlex entries in a single database.
509
+ You can combine this with the OpenAlex entries in a single database.
464
510
 
465
511
  You can walk a directory and add files from it with this command.
466
512
 
@@ -484,19 +530,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
484
530
 
485
531
  [[litdb:https://doi.org/10.1021/jp047349j]]
486
532
 
487
- The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
533
+ The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
488
534
 
489
535
  You can easily insert a link like this:
490
536
 
491
537
  M-x litdb
492
538
 
493
- See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
539
+ See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
494
540
 
495
- litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
541
+ litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
496
542
 
497
543
  * Database design
498
544
 
499
- litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
545
+ litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
500
546
 
501
547
  The main table in litdb is called sources.
502
548
  - sources
@@ -551,7 +597,7 @@ Similar limitations exist for local files. There are additionally the following
551
597
  1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
552
598
  2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
553
599
 
554
- For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
600
+ For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
555
601
 
556
602
  There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
557
603
 
@@ -569,7 +615,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
569
615
 
570
616
  sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
571
617
 
572
- libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
618
+ libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
573
619
 
574
620
  * Roadmap
575
621
 
@@ -583,7 +629,7 @@ Notes on the other hand, might be small enough that they could be stored in the
583
629
 
584
630
  ** Jupyter lab integration
585
631
 
586
- An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
632
+ An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
587
633
 
588
634
  ** graph visualization
589
635
 
@@ -655,7 +701,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
655
701
 
656
702
  ** image and text models
657
703
 
658
- One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
704
+ One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
659
705
 
660
706
  ** DONE combine full text and vector search
661
707
  CLOSED: [2024-12-09 Mon 13:59]
@@ -737,11 +783,11 @@ print(text)
737
783
  #+RESULTS:
738
784
  : ++RESULTS:
739
785
  : ; Selected region: (26, 332, 473, 69)
740
- :
786
+ :
741
787
 
742
788
  This might be nice later when we have image embeddings.
743
789
 
744
- ** DONE review process
790
+ ** DONE review process
745
791
 
746
792
  #+BEGIN_SRC sh
747
793
  litdb review --since '1 week ago'
@@ -761,7 +807,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
761
807
  | https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
762
808
  | https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
763
809
 
764
- ** DONE semantic similarity
810
+ ** DONE semantic similarity
765
811
  CLOSED: [2024-12-04 Wed 13:12]
766
812
 
767
813
  litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
@@ -772,12 +818,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
772
818
 
773
819
  - LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
774
820
  - Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
775
-
821
+
776
822
  - paper-qa :: https://github.com/Future-House/paper-qa
777
823
  - This project by Andrew White uses LLM+RAG to explore a paper.
778
824
 
779
825
  - ColBERT :: https://github.com/stanford-futuredata/ColBERT
780
- - ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
826
+ - ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
781
827
 
782
828
  Many of these projects require you to make an account. There are freemium levels in each one.
783
829
 
@@ -789,7 +835,7 @@ Many of these projects require you to make an account. There are freemium levels
789
835
 
790
836
  - Keenious :: https://keenious.com/explore
791
837
  - Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
792
-
838
+
793
839
  - scite.ai :: https://scite.ai/
794
840
  - Browser tool that integrates GPT with the scientific literature, integration with Zotero
795
841
 
@@ -803,7 +849,7 @@ Many of these projects require you to make an account. There are freemium levels
803
849
  - This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
804
850
 
805
851
  - AnythingLLM :: https://anythingllm.com/
806
- - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
852
+ - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
807
853
 
808
854
  - gpt4all :: https://www.nomic.ai/gpt4all
809
855
  - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
@@ -1,40 +1,3 @@
1
- Metadata-Version: 2.1
2
- Name: litdb
3
- Version: 2.0.2
4
- Summary: A literature database tool with GPT integration.
5
- Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
6
- Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
7
- License: MIT
8
- Project-URL: Homepage, https://github.com/jkitchin/litdb
9
- Description-Content-Type: text/plain
10
- Requires-Dist: toml
11
- Requires-Dist: tomlkit
12
- Requires-Dist: requests
13
- Requires-Dist: pymupdf4llm
14
- Requires-Dist: langchain
15
- Requires-Dist: tqdm
16
- Requires-Dist: libsql-experimental
17
- Requires-Dist: sentence-transformers
18
- Requires-Dist: click
19
- Requires-Dist: rich
20
- Requires-Dist: beautifulsoup4
21
- Requires-Dist: ratelimit
22
- Requires-Dist: python-docx
23
- Requires-Dist: python-pptx
24
- Requires-Dist: nbconvert
25
- Requires-Dist: nbformat
26
- Requires-Dist: ollama
27
- Requires-Dist: dateparser
28
- Requires-Dist: jinja2
29
- Requires-Dist: pytesseract
30
- Requires-Dist: pillow
31
- Requires-Dist: bibtexparser
32
- Requires-Dist: pandas
33
- Requires-Dist: more-itertools
34
- Requires-Dist: nameparser
35
- Requires-Dist: tabulate
36
- Requires-Dist: IPython
37
-
38
1
  #+title: litdb - a literature and document database
39
2
 
40
3
  #+attr_org: :width 600
@@ -74,7 +37,7 @@ pip install git+https://github.com/jkitchin/litdb
74
37
 
75
38
  You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
76
39
 
77
- There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
40
+ There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
78
41
 
79
42
  You will need an OpenAlex premium key if you want to use the update-filters feature.
80
43
 
@@ -92,6 +55,9 @@ api_key = "..."
92
55
 
93
56
  [gpt]
94
57
  model = "llama2"
58
+
59
+ [llm]
60
+ model = "ollama/llama2"
95
61
  #+END_EXAMPLE
96
62
 
97
63
  You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
@@ -102,7 +68,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
102
68
 
103
69
  When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
104
70
 
105
- * Using litdb
71
+ * Using litdb
106
72
 
107
73
  Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
108
74
 
@@ -156,7 +122,7 @@ litdb openalex -e sources -f display_name.search:discovery
156
122
 
157
123
  ** One-time additions of articles to litdb
158
124
 
159
- You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
125
+ You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
160
126
 
161
127
  #+BEGIN_SRC sh
162
128
  litdb add doi --references --citing --related
@@ -182,6 +148,8 @@ litdb add doi1 doi2 orcid
182
148
 
183
149
  These are all one-time additions.
184
150
 
151
+ You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
152
+
185
153
  ** Adding filters
186
154
 
187
155
  litdb provides several convenient ways to add queries to update your litdb in the future.
@@ -244,7 +212,7 @@ litdb rm-filter "filter-string"
244
212
 
245
213
  ** Review your litdb
246
214
 
247
- I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
215
+ I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
248
216
 
249
217
  #+BEGIN_SRC sh
250
218
  litdb review -s "1 week ago"
@@ -254,14 +222,14 @@ This works best when you update your litdb regularly. You might want to redirect
254
222
 
255
223
  ** Searching litdb
256
224
 
257
- There are several search options.
225
+ There are several search options.
258
226
 
259
227
  *** vector search
260
228
 
261
229
  The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
262
230
 
263
231
  #+BEGIN_SRC sh
264
- litdb vsearch "natural language query"
232
+ litdb vsearch "natural language query"
265
233
  #+END_SRC
266
234
 
267
235
  The default number of entries returned is 3. You can change that with an optional argument
@@ -312,6 +280,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
312
280
  litdb gpt "what is the state of the art in automated laboratories for soft materials"
313
281
  #+END_SRC
314
282
 
283
+ *** Integration with litellm
284
+
285
+ litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
286
+
287
+ The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
288
+
289
+ It uses a different command than the ollama gpt command.
290
+
291
+ #+BEGIN_SRC sh
292
+ litdb chat "what is the state of the art in automated laboratories for soft materials"
293
+ #+END_SRC
294
+
295
+ There are some fancy things you can do with the prompt:
296
+
297
+ 1. Avoid using RAG if --norag is in your prompt.
298
+ 2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
299
+ 3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
300
+ 4. A prompt of !save will save the current chat to a file.
301
+ 5. You can use this syntax to expand a file or url in the prompt for context:
302
+ #+BEGIN_EXAMPLE
303
+ [[file/url]]
304
+ #+END_EXAMPLE
305
+
306
+ Your prompt history is saved in your litdb, so you can go back to them if you want.
307
+
308
+ *** Web app for litdb
309
+
310
+ If you prefer a browser, you can now launch a streamlit app for litdb:
311
+
312
+ #+BEGIN_SRC sh
313
+ litdb app
314
+ #+END_SRC
315
+
316
+ This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
317
+
315
318
  *** search with audio
316
319
 
317
320
  This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
@@ -441,7 +444,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
441
444
 
442
445
  * Adding local files
443
446
 
444
- The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
447
+ The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
445
448
 
446
449
  It is possible to add any file that can be turned into text to litdb. That includes:
447
450
 
@@ -456,11 +459,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
456
459
 
457
460
  This limits portability because you need a path if you want to be able to open that file.
458
461
 
459
- The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
462
+ The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
460
463
 
461
- An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
464
+ An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
462
465
 
463
- You can combine this with the OpenAlex entries in a single database.
466
+ You can combine this with the OpenAlex entries in a single database.
464
467
 
465
468
  You can walk a directory and add files from it with this command.
466
469
 
@@ -484,19 +487,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
484
487
 
485
488
  [[litdb:https://doi.org/10.1021/jp047349j]]
486
489
 
487
- The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
490
+ The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
488
491
 
489
492
  You can easily insert a link like this:
490
493
 
491
494
  M-x litdb
492
495
 
493
- See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
496
+ See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
494
497
 
495
- litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
498
+ litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
496
499
 
497
500
  * Database design
498
501
 
499
- litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
502
+ litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
500
503
 
501
504
  The main table in litdb is called sources.
502
505
  - sources
@@ -551,7 +554,7 @@ Similar limitations exist for local files. There are additionally the following
551
554
  1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
552
555
  2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
553
556
 
554
- For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
557
+ For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
555
558
 
556
559
  There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
557
560
 
@@ -569,7 +572,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
569
572
 
570
573
  sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
571
574
 
572
- libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
575
+ libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
573
576
 
574
577
  * Roadmap
575
578
 
@@ -583,7 +586,7 @@ Notes on the other hand, might be small enough that they could be stored in the
583
586
 
584
587
  ** Jupyter lab integration
585
588
 
586
- An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
589
+ An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
587
590
 
588
591
  ** graph visualization
589
592
 
@@ -655,7 +658,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
655
658
 
656
659
  ** image and text models
657
660
 
658
- One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
661
+ One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
659
662
 
660
663
  ** DONE combine full text and vector search
661
664
  CLOSED: [2024-12-09 Mon 13:59]
@@ -737,11 +740,11 @@ print(text)
737
740
  #+RESULTS:
738
741
  : ++RESULTS:
739
742
  : ; Selected region: (26, 332, 473, 69)
740
- :
743
+ :
741
744
 
742
745
  This might be nice later when we have image embeddings.
743
746
 
744
- ** DONE review process
747
+ ** DONE review process
745
748
 
746
749
  #+BEGIN_SRC sh
747
750
  litdb review --since '1 week ago'
@@ -761,7 +764,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
761
764
  | https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
762
765
  | https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
763
766
 
764
- ** DONE semantic similarity
767
+ ** DONE semantic similarity
765
768
  CLOSED: [2024-12-04 Wed 13:12]
766
769
 
767
770
  litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
@@ -772,12 +775,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
772
775
 
773
776
  - LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
774
777
  - Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
775
-
778
+
776
779
  - paper-qa :: https://github.com/Future-House/paper-qa
777
780
  - This project by Andrew White uses LLM+RAG to explore a paper.
778
781
 
779
782
  - ColBERT :: https://github.com/stanford-futuredata/ColBERT
780
- - ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
783
+ - ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
781
784
 
782
785
  Many of these projects require you to make an account. There are freemium levels in each one.
783
786
 
@@ -789,7 +792,7 @@ Many of these projects require you to make an account. There are freemium levels
789
792
 
790
793
  - Keenious :: https://keenious.com/explore
791
794
  - Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
792
-
795
+
793
796
  - scite.ai :: https://scite.ai/
794
797
  - Browser tool that integrates GPT with the scientific literature, integration with Zotero
795
798
 
@@ -803,7 +806,7 @@ Many of these projects require you to make an account. There are freemium levels
803
806
  - This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
804
807
 
805
808
  - AnythingLLM :: https://anythingllm.com/
806
- - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
809
+ - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
807
810
 
808
811
  - gpt4all :: https://www.nomic.ai/gpt4all
809
812
  - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
@@ -7,7 +7,7 @@ where = ["src"]
7
7
 
8
8
  [project]
9
9
  name = "litdb"
10
- version = "2.0.2"
10
+ version = "2.1.0"
11
11
  authors = [
12
12
  {name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
13
13
  ]
@@ -22,10 +22,12 @@ dependencies = [
22
22
  "tomlkit",
23
23
  "requests",
24
24
  "pymupdf4llm",
25
+ "docling",
25
26
  "langchain",
26
27
  "tqdm",
27
28
  "libsql-experimental",
28
29
  "sentence-transformers",
30
+ "transformers",
29
31
  "click",
30
32
  "rich",
31
33
  "beautifulsoup4",
@@ -35,12 +37,13 @@ dependencies = [
35
37
  "nbconvert",
36
38
  "nbformat",
37
39
  "ollama",
40
+ "litellm",
38
41
  "dateparser",
39
42
  "jinja2",
40
43
  "pytesseract",
41
44
  "pillow",
42
45
  "bibtexparser",
43
-
46
+
44
47
  # "pyaudio",
45
48
  # "playsound",
46
49
  # "SpeechRecognition",
@@ -50,7 +53,14 @@ dependencies = [
50
53
  "more-itertools",
51
54
  "nameparser",
52
55
  "tabulate",
53
- "IPython"
56
+ "IPython",
57
+
58
+ # youtube
59
+ "yt_dlp",
60
+ "webvtt-py",
61
+
62
+ # the streamlit app
63
+ "streamlit"
54
64
  ]
55
65
 
56
66
  [project.scripts]