litdb 2.0.2__tar.gz → 2.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {litdb-2.0.2/src/litdb.egg-info → litdb-2.1.1}/PKG-INFO +74 -27
- litdb-2.0.2/PKG-INFO → litdb-2.1.1/README.org +66 -62
- {litdb-2.0.2 → litdb-2.1.1}/pyproject.toml +15 -4
- litdb-2.1.1/src/litdb/app.py +83 -0
- litdb-2.1.1/src/litdb/audio.py +139 -0
- litdb-2.1.1/src/litdb/chat.py +377 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/cli.py +56 -84
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/db.py +8 -1
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/gpt.py +0 -2
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/utils.py +1 -1
- litdb-2.1.1/src/litdb/youtube.py +48 -0
- litdb-2.0.2/README.org → litdb-2.1.1/src/litdb.egg-info/PKG-INFO +109 -25
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb.egg-info/SOURCES.txt +4 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb.egg-info/requires.txt +6 -0
- {litdb-2.0.2 → litdb-2.1.1}/setup.cfg +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/__init__.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/bibtex.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/coa.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/lab.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/openalex.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb/pdf.py +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb.egg-info/entry_points.txt +0 -0
- {litdb-2.0.2 → litdb-2.1.1}/src/litdb.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.1
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,10 +11,12 @@ Requires-Dist: toml
|
|
|
11
11
|
Requires-Dist: tomlkit
|
|
12
12
|
Requires-Dist: requests
|
|
13
13
|
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: docling
|
|
14
15
|
Requires-Dist: langchain
|
|
15
16
|
Requires-Dist: tqdm
|
|
16
17
|
Requires-Dist: libsql-experimental
|
|
17
18
|
Requires-Dist: sentence-transformers
|
|
19
|
+
Requires-Dist: transformers
|
|
18
20
|
Requires-Dist: click
|
|
19
21
|
Requires-Dist: rich
|
|
20
22
|
Requires-Dist: beautifulsoup4
|
|
@@ -24,6 +26,7 @@ Requires-Dist: python-pptx
|
|
|
24
26
|
Requires-Dist: nbconvert
|
|
25
27
|
Requires-Dist: nbformat
|
|
26
28
|
Requires-Dist: ollama
|
|
29
|
+
Requires-Dist: litellm
|
|
27
30
|
Requires-Dist: dateparser
|
|
28
31
|
Requires-Dist: jinja2
|
|
29
32
|
Requires-Dist: pytesseract
|
|
@@ -34,6 +37,9 @@ Requires-Dist: more-itertools
|
|
|
34
37
|
Requires-Dist: nameparser
|
|
35
38
|
Requires-Dist: tabulate
|
|
36
39
|
Requires-Dist: IPython
|
|
40
|
+
Requires-Dist: yt_dlp
|
|
41
|
+
Requires-Dist: webvtt-py
|
|
42
|
+
Requires-Dist: streamlit
|
|
37
43
|
|
|
38
44
|
#+title: litdb - a literature and document database
|
|
39
45
|
|
|
@@ -54,6 +60,7 @@ The idea is you add papers to your database, and then you can search it with nat
|
|
|
54
60
|
4. https://youtube.com/live/ZkKKuvVUWkE litdb and Emacs
|
|
55
61
|
5. https://youtube.com/live/j7rItPwWDaY litdb and Jupyter Lab
|
|
56
62
|
6. https://youtube.com/live/SUtvtc7l6y0 litdb + GPT enhancements
|
|
63
|
+
7. https://youtube.com/live/3FZ1ROnCC6Y litdb + LiteLLM and streamlit
|
|
57
64
|
|
|
58
65
|
** installation
|
|
59
66
|
|
|
@@ -74,7 +81,7 @@ pip install git+https://github.com/jkitchin/litdb
|
|
|
74
81
|
|
|
75
82
|
You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
|
|
76
83
|
|
|
77
|
-
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
84
|
+
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
78
85
|
|
|
79
86
|
You will need an OpenAlex premium key if you want to use the update-filters feature.
|
|
80
87
|
|
|
@@ -92,6 +99,9 @@ api_key = "..."
|
|
|
92
99
|
|
|
93
100
|
[gpt]
|
|
94
101
|
model = "llama2"
|
|
102
|
+
|
|
103
|
+
[llm]
|
|
104
|
+
model = "ollama/llama2"
|
|
95
105
|
#+END_EXAMPLE
|
|
96
106
|
|
|
97
107
|
You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
|
|
@@ -102,7 +112,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
|
|
|
102
112
|
|
|
103
113
|
When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
|
|
104
114
|
|
|
105
|
-
* Using litdb
|
|
115
|
+
* Using litdb
|
|
106
116
|
|
|
107
117
|
Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
|
|
108
118
|
|
|
@@ -156,7 +166,7 @@ litdb openalex -e sources -f display_name.search:discovery
|
|
|
156
166
|
|
|
157
167
|
** One-time additions of articles to litdb
|
|
158
168
|
|
|
159
|
-
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
169
|
+
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
160
170
|
|
|
161
171
|
#+BEGIN_SRC sh
|
|
162
172
|
litdb add doi --references --citing --related
|
|
@@ -182,6 +192,8 @@ litdb add doi1 doi2 orcid
|
|
|
182
192
|
|
|
183
193
|
These are all one-time additions.
|
|
184
194
|
|
|
195
|
+
You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
|
|
196
|
+
|
|
185
197
|
** Adding filters
|
|
186
198
|
|
|
187
199
|
litdb provides several convenient ways to add queries to update your litdb in the future.
|
|
@@ -244,7 +256,7 @@ litdb rm-filter "filter-string"
|
|
|
244
256
|
|
|
245
257
|
** Review your litdb
|
|
246
258
|
|
|
247
|
-
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
259
|
+
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
248
260
|
|
|
249
261
|
#+BEGIN_SRC sh
|
|
250
262
|
litdb review -s "1 week ago"
|
|
@@ -254,14 +266,14 @@ This works best when you update your litdb regularly. You might want to redirect
|
|
|
254
266
|
|
|
255
267
|
** Searching litdb
|
|
256
268
|
|
|
257
|
-
There are several search options.
|
|
269
|
+
There are several search options.
|
|
258
270
|
|
|
259
271
|
*** vector search
|
|
260
272
|
|
|
261
273
|
The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
|
|
262
274
|
|
|
263
275
|
#+BEGIN_SRC sh
|
|
264
|
-
litdb vsearch "natural language query"
|
|
276
|
+
litdb vsearch "natural language query"
|
|
265
277
|
#+END_SRC
|
|
266
278
|
|
|
267
279
|
The default number of entries returned is 3. You can change that with an optional argument
|
|
@@ -312,6 +324,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
|
|
|
312
324
|
litdb gpt "what is the state of the art in automated laboratories for soft materials"
|
|
313
325
|
#+END_SRC
|
|
314
326
|
|
|
327
|
+
*** Integration with litellm
|
|
328
|
+
|
|
329
|
+
litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
|
|
330
|
+
|
|
331
|
+
The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
|
|
332
|
+
|
|
333
|
+
It uses a different command than the ollama gpt command.
|
|
334
|
+
|
|
335
|
+
#+BEGIN_SRC sh
|
|
336
|
+
litdb chat "what is the state of the art in automated laboratories for soft materials"
|
|
337
|
+
#+END_SRC
|
|
338
|
+
|
|
339
|
+
There are some fancy things you can do with the prompt:
|
|
340
|
+
|
|
341
|
+
1. Avoid using RAG if --norag is in your prompt.
|
|
342
|
+
2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
|
|
343
|
+
3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
|
|
344
|
+
4. A prompt of !save will save the current chat to a file.
|
|
345
|
+
5. You can use this syntax to expand a file or url in the prompt for context:
|
|
346
|
+
#+BEGIN_EXAMPLE
|
|
347
|
+
[[file/url]]
|
|
348
|
+
#+END_EXAMPLE
|
|
349
|
+
|
|
350
|
+
Your prompt history is saved in your litdb, so you can go back to them if you want.
|
|
351
|
+
|
|
352
|
+
*** Web app for litdb
|
|
353
|
+
|
|
354
|
+
If you prefer a browser, you can now launch a streamlit app for litdb:
|
|
355
|
+
|
|
356
|
+
#+BEGIN_SRC sh
|
|
357
|
+
litdb app
|
|
358
|
+
#+END_SRC
|
|
359
|
+
|
|
360
|
+
This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
|
|
361
|
+
|
|
315
362
|
*** search with audio
|
|
316
363
|
|
|
317
364
|
This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
|
|
@@ -441,7 +488,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
|
|
|
441
488
|
|
|
442
489
|
* Adding local files
|
|
443
490
|
|
|
444
|
-
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
491
|
+
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
445
492
|
|
|
446
493
|
It is possible to add any file that can be turned into text to litdb. That includes:
|
|
447
494
|
|
|
@@ -456,11 +503,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
|
|
|
456
503
|
|
|
457
504
|
This limits portability because you need a path if you want to be able to open that file.
|
|
458
505
|
|
|
459
|
-
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
506
|
+
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
460
507
|
|
|
461
|
-
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
508
|
+
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
462
509
|
|
|
463
|
-
You can combine this with the OpenAlex entries in a single database.
|
|
510
|
+
You can combine this with the OpenAlex entries in a single database.
|
|
464
511
|
|
|
465
512
|
You can walk a directory and add files from it with this command.
|
|
466
513
|
|
|
@@ -484,19 +531,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
|
|
|
484
531
|
|
|
485
532
|
[[litdb:https://doi.org/10.1021/jp047349j]]
|
|
486
533
|
|
|
487
|
-
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
534
|
+
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
488
535
|
|
|
489
536
|
You can easily insert a link like this:
|
|
490
537
|
|
|
491
538
|
M-x litdb
|
|
492
539
|
|
|
493
|
-
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
540
|
+
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
494
541
|
|
|
495
|
-
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
542
|
+
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
496
543
|
|
|
497
544
|
* Database design
|
|
498
545
|
|
|
499
|
-
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
546
|
+
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
500
547
|
|
|
501
548
|
The main table in litdb is called sources.
|
|
502
549
|
- sources
|
|
@@ -551,7 +598,7 @@ Similar limitations exist for local files. There are additionally the following
|
|
|
551
598
|
1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
|
|
552
599
|
2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
|
|
553
600
|
|
|
554
|
-
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
601
|
+
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
555
602
|
|
|
556
603
|
There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
|
|
557
604
|
|
|
@@ -569,7 +616,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
|
|
|
569
616
|
|
|
570
617
|
sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
|
|
571
618
|
|
|
572
|
-
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
619
|
+
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
573
620
|
|
|
574
621
|
* Roadmap
|
|
575
622
|
|
|
@@ -583,7 +630,7 @@ Notes on the other hand, might be small enough that they could be stored in the
|
|
|
583
630
|
|
|
584
631
|
** Jupyter lab integration
|
|
585
632
|
|
|
586
|
-
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
633
|
+
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
587
634
|
|
|
588
635
|
** graph visualization
|
|
589
636
|
|
|
@@ -655,7 +702,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
|
|
|
655
702
|
|
|
656
703
|
** image and text models
|
|
657
704
|
|
|
658
|
-
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
705
|
+
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
659
706
|
|
|
660
707
|
** DONE combine full text and vector search
|
|
661
708
|
CLOSED: [2024-12-09 Mon 13:59]
|
|
@@ -737,11 +784,11 @@ print(text)
|
|
|
737
784
|
#+RESULTS:
|
|
738
785
|
: ++RESULTS:
|
|
739
786
|
: ; Selected region: (26, 332, 473, 69)
|
|
740
|
-
:
|
|
787
|
+
:
|
|
741
788
|
|
|
742
789
|
This might be nice later when we have image embeddings.
|
|
743
790
|
|
|
744
|
-
** DONE review process
|
|
791
|
+
** DONE review process
|
|
745
792
|
|
|
746
793
|
#+BEGIN_SRC sh
|
|
747
794
|
litdb review --since '1 week ago'
|
|
@@ -761,7 +808,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
|
|
|
761
808
|
| https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
|
|
762
809
|
| https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
|
|
763
810
|
|
|
764
|
-
** DONE semantic similarity
|
|
811
|
+
** DONE semantic similarity
|
|
765
812
|
CLOSED: [2024-12-04 Wed 13:12]
|
|
766
813
|
|
|
767
814
|
litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
|
|
@@ -772,12 +819,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
|
|
|
772
819
|
|
|
773
820
|
- LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
|
|
774
821
|
- Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
|
|
775
|
-
|
|
822
|
+
|
|
776
823
|
- paper-qa :: https://github.com/Future-House/paper-qa
|
|
777
824
|
- This project by Andrew White uses LLM+RAG to explore a paper.
|
|
778
825
|
|
|
779
826
|
- ColBERT :: https://github.com/stanford-futuredata/ColBERT
|
|
780
|
-
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
827
|
+
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
781
828
|
|
|
782
829
|
Many of these projects require you to make an account. There are freemium levels in each one.
|
|
783
830
|
|
|
@@ -789,7 +836,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
789
836
|
|
|
790
837
|
- Keenious :: https://keenious.com/explore
|
|
791
838
|
- Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
|
|
792
|
-
|
|
839
|
+
|
|
793
840
|
- scite.ai :: https://scite.ai/
|
|
794
841
|
- Browser tool that integrates GPT with the scientific literature, integration with Zotero
|
|
795
842
|
|
|
@@ -803,7 +850,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
803
850
|
- This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
|
|
804
851
|
|
|
805
852
|
- AnythingLLM :: https://anythingllm.com/
|
|
806
|
-
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
853
|
+
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
807
854
|
|
|
808
855
|
- gpt4all :: https://www.nomic.ai/gpt4all
|
|
809
856
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
@@ -1,40 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: litdb
|
|
3
|
-
Version: 2.0.2
|
|
4
|
-
Summary: A literature database tool with GPT integration.
|
|
5
|
-
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
|
-
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
7
|
-
License: MIT
|
|
8
|
-
Project-URL: Homepage, https://github.com/jkitchin/litdb
|
|
9
|
-
Description-Content-Type: text/plain
|
|
10
|
-
Requires-Dist: toml
|
|
11
|
-
Requires-Dist: tomlkit
|
|
12
|
-
Requires-Dist: requests
|
|
13
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
-
Requires-Dist: langchain
|
|
15
|
-
Requires-Dist: tqdm
|
|
16
|
-
Requires-Dist: libsql-experimental
|
|
17
|
-
Requires-Dist: sentence-transformers
|
|
18
|
-
Requires-Dist: click
|
|
19
|
-
Requires-Dist: rich
|
|
20
|
-
Requires-Dist: beautifulsoup4
|
|
21
|
-
Requires-Dist: ratelimit
|
|
22
|
-
Requires-Dist: python-docx
|
|
23
|
-
Requires-Dist: python-pptx
|
|
24
|
-
Requires-Dist: nbconvert
|
|
25
|
-
Requires-Dist: nbformat
|
|
26
|
-
Requires-Dist: ollama
|
|
27
|
-
Requires-Dist: dateparser
|
|
28
|
-
Requires-Dist: jinja2
|
|
29
|
-
Requires-Dist: pytesseract
|
|
30
|
-
Requires-Dist: pillow
|
|
31
|
-
Requires-Dist: bibtexparser
|
|
32
|
-
Requires-Dist: pandas
|
|
33
|
-
Requires-Dist: more-itertools
|
|
34
|
-
Requires-Dist: nameparser
|
|
35
|
-
Requires-Dist: tabulate
|
|
36
|
-
Requires-Dist: IPython
|
|
37
|
-
|
|
38
1
|
#+title: litdb - a literature and document database
|
|
39
2
|
|
|
40
3
|
#+attr_org: :width 600
|
|
@@ -54,6 +17,7 @@ The idea is you add papers to your database, and then you can search it with nat
|
|
|
54
17
|
4. https://youtube.com/live/ZkKKuvVUWkE litdb and Emacs
|
|
55
18
|
5. https://youtube.com/live/j7rItPwWDaY litdb and Jupyter Lab
|
|
56
19
|
6. https://youtube.com/live/SUtvtc7l6y0 litdb + GPT enhancements
|
|
20
|
+
7. https://youtube.com/live/3FZ1ROnCC6Y litdb + LiteLLM and streamlit
|
|
57
21
|
|
|
58
22
|
** installation
|
|
59
23
|
|
|
@@ -74,7 +38,7 @@ pip install git+https://github.com/jkitchin/litdb
|
|
|
74
38
|
|
|
75
39
|
You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
|
|
76
40
|
|
|
77
|
-
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
41
|
+
There are a few choices you have to make. You have to choose a SentenceTransformer model, and specify the size of the vectors it makes. You also have to specify the chunk_size and chunk_overlap settings that are used to break documents up to compute document level embedding vectors.
|
|
78
42
|
|
|
79
43
|
You will need an OpenAlex premium key if you want to use the update-filters feature.
|
|
80
44
|
|
|
@@ -92,6 +56,9 @@ api_key = "..."
|
|
|
92
56
|
|
|
93
57
|
[gpt]
|
|
94
58
|
model = "llama2"
|
|
59
|
+
|
|
60
|
+
[llm]
|
|
61
|
+
model = "ollama/llama2"
|
|
95
62
|
#+END_EXAMPLE
|
|
96
63
|
|
|
97
64
|
You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
|
|
@@ -102,7 +69,7 @@ export LITDB_ROOT="/path/to/your/default/litdb"
|
|
|
102
69
|
|
|
103
70
|
When you run a litdb command, it will look for a dominating litdb.toml file, which means you are running the command in a litdb project. If one is not found, it will check for the LITDB_ROOT environment variable and use that if it is found. Finally, if that does not exist, it will prompt you to make a new project in the current directory.
|
|
104
71
|
|
|
105
|
-
* Using litdb
|
|
72
|
+
* Using litdb
|
|
106
73
|
|
|
107
74
|
Your litdb starts out empty. You have to add articles that are relevant to you. It is an open question of the best way to build a litdb. The answer surely depends on what your aim is. You have to compromise on breadth and depth with the database size. The CLI makes it pretty easy to do this
|
|
108
75
|
|
|
@@ -156,7 +123,7 @@ litdb openalex -e sources -f display_name.search:discovery
|
|
|
156
123
|
|
|
157
124
|
** One-time additions of articles to litdb
|
|
158
125
|
|
|
159
|
-
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
126
|
+
You add an article by its DOI. There are optional arguments to also add references, citing and related articles.
|
|
160
127
|
|
|
161
128
|
#+BEGIN_SRC sh
|
|
162
129
|
litdb add doi --references --citing --related
|
|
@@ -182,6 +149,8 @@ litdb add doi1 doi2 orcid
|
|
|
182
149
|
|
|
183
150
|
These are all one-time additions.
|
|
184
151
|
|
|
152
|
+
You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
|
|
153
|
+
|
|
185
154
|
** Adding filters
|
|
186
155
|
|
|
187
156
|
litdb provides several convenient ways to add queries to update your litdb in the future.
|
|
@@ -244,7 +213,7 @@ litdb rm-filter "filter-string"
|
|
|
244
213
|
|
|
245
214
|
** Review your litdb
|
|
246
215
|
|
|
247
|
-
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
216
|
+
I find it helpful to review your litdb. To get a list of articles added in the last week, you can run this command.
|
|
248
217
|
|
|
249
218
|
#+BEGIN_SRC sh
|
|
250
219
|
litdb review -s "1 week ago"
|
|
@@ -254,14 +223,14 @@ This works best when you update your litdb regularly. You might want to redirect
|
|
|
254
223
|
|
|
255
224
|
** Searching litdb
|
|
256
225
|
|
|
257
|
-
There are several search options.
|
|
226
|
+
There are several search options.
|
|
258
227
|
|
|
259
228
|
*** vector search
|
|
260
229
|
|
|
261
230
|
The main way litdb was designed to be searched is with by natural language queries. The way this works is your query is converted to a vector using SentenceTransformers, and then a vector search identifies entries in the database that are similar to your query.
|
|
262
231
|
|
|
263
232
|
#+BEGIN_SRC sh
|
|
264
|
-
litdb vsearch "natural language query"
|
|
233
|
+
litdb vsearch "natural language query"
|
|
265
234
|
#+END_SRC
|
|
266
235
|
|
|
267
236
|
The default number of entries returned is 3. You can change that with an optional argument
|
|
@@ -312,6 +281,41 @@ You can use litdb as a RAG source for ollama. This looks up the three most relat
|
|
|
312
281
|
litdb gpt "what is the state of the art in automated laboratories for soft materials"
|
|
313
282
|
#+END_SRC
|
|
314
283
|
|
|
284
|
+
*** Integration with litellm
|
|
285
|
+
|
|
286
|
+
litdb supports litellm so you can use almost any LLM provider you want: OpenAI, Anthropic, Gemini, whatever you have an API key for.
|
|
287
|
+
|
|
288
|
+
The free tier of the API includes 1,500 requests per day with Gemini 1.5 Flash.
|
|
289
|
+
|
|
290
|
+
It uses a different command than the ollama gpt command.
|
|
291
|
+
|
|
292
|
+
#+BEGIN_SRC sh
|
|
293
|
+
litdb chat "what is the state of the art in automated laboratories for soft materials"
|
|
294
|
+
#+END_SRC
|
|
295
|
+
|
|
296
|
+
There are some fancy things you can do with the prompt:
|
|
297
|
+
|
|
298
|
+
1. Avoid using RAG if --norag is in your prompt.
|
|
299
|
+
2. If you surround python objects with backticks, it will try expanding that to the documentation from Python.
|
|
300
|
+
3. A line that starts with < indicates a shell command to run and the output will be expanded into the prompt.
|
|
301
|
+
4. A prompt of !save will save the current chat to a file.
|
|
302
|
+
5. You can use this syntax to expand a file or url in the prompt for context:
|
|
303
|
+
#+BEGIN_EXAMPLE
|
|
304
|
+
[[file/url]]
|
|
305
|
+
#+END_EXAMPLE
|
|
306
|
+
|
|
307
|
+
Your prompt history is saved in your litdb, so you can go back to them if you want.
|
|
308
|
+
|
|
309
|
+
*** Web app for litdb
|
|
310
|
+
|
|
311
|
+
If you prefer a browser, you can now launch a streamlit app for litdb:
|
|
312
|
+
|
|
313
|
+
#+BEGIN_SRC sh
|
|
314
|
+
litdb app
|
|
315
|
+
#+END_SRC
|
|
316
|
+
|
|
317
|
+
This should launch the app in your browser and you can search litdb from it. The terminal application is more advanced in terms of prompt expansion.
|
|
318
|
+
|
|
315
319
|
*** search with audio
|
|
316
320
|
|
|
317
321
|
This command will record audio, transcribe that audio to text, and then do a vector search on that text. You will be prompted when the recording starts, and you press return to stop it. litdb will show you what it heard, and ask if you want to do a vector search on it.
|
|
@@ -441,7 +445,7 @@ litdb sql "select source, json_extract(extra, '$.citation') as citation from sou
|
|
|
441
445
|
|
|
442
446
|
* Adding local files
|
|
443
447
|
|
|
444
|
-
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
448
|
+
The idea of using local files is that it is likely you have collected information in the form of files on your hard drive, and you want to be able to find information in those files.
|
|
445
449
|
|
|
446
450
|
It is possible to add any file that can be turned into text to litdb. That includes:
|
|
447
451
|
|
|
@@ -456,11 +460,11 @@ It is possible to add any file that can be turned into text to litdb. That inclu
|
|
|
456
460
|
|
|
457
461
|
This limits portability because you need a path if you want to be able to open that file.
|
|
458
462
|
|
|
459
|
-
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
463
|
+
The same vector, fulltext and gpt search commands are available for local file entries. These tend to be longer documents than the OpenAlex entries, and I am not sure how well the search works at the document level embeddings. Search at a chunk level is very precise; odds are you want paragraph level similarity to your query.
|
|
460
464
|
|
|
461
|
-
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
465
|
+
An early version of litdb stored each chunk. This is possible, but I used another table for it. You could munge the source to be something like f.pdf::chunk-1 so each one is unique, but that seems more complicated and you would need to do some experiments to see if it is warranted.
|
|
462
466
|
|
|
463
|
-
You can combine this with the OpenAlex entries in a single database.
|
|
467
|
+
You can combine this with the OpenAlex entries in a single database.
|
|
464
468
|
|
|
465
469
|
You can walk a directory and add files from it with this command.
|
|
466
470
|
|
|
@@ -484,19 +488,19 @@ Of course there is some Emacs integration. I made a new link for litdb.
|
|
|
484
488
|
|
|
485
489
|
[[litdb:https://doi.org/10.1021/jp047349j]]
|
|
486
490
|
|
|
487
|
-
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
491
|
+
The links export as \cite{source}, and there is a function ~litdb-generate-bibtex~ to export bibtex entries for all links in the buffer. These entries are not certain to be valid, most likely from the keys (some DOIs are probably invalid keys).
|
|
488
492
|
|
|
489
493
|
You can easily insert a link like this:
|
|
490
494
|
|
|
491
495
|
M-x litdb
|
|
492
496
|
|
|
493
|
-
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
497
|
+
See [[./litdb.el]] for details. This is not a package on MELPA yet. You should just load the .el file in your config. You can also use ~litdb-fulltext~, ~litdb-vsearch~, and ~litdb-gpt~ from Emacs to interact with your litdb.
|
|
494
498
|
|
|
495
|
-
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
499
|
+
litdb.el is under active development, and will be an alternative UI to the terminal eventually. It is too early to tell if it will replace org-ref. It has potential, but that would be a very large undertaking.
|
|
496
500
|
|
|
497
501
|
* Database design
|
|
498
502
|
|
|
499
|
-
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
503
|
+
litdb uses a sqlite database with libsql. libsql is a sqlite fork with additional capabilities, most notably integrated vector search.
|
|
500
504
|
|
|
501
505
|
The main table in litdb is called sources.
|
|
502
506
|
- sources
|
|
@@ -551,7 +555,7 @@ Similar limitations exist for local files. There are additionally the following
|
|
|
551
555
|
1. The quality of document to text influences the ultimate embedding. This varies by type of document, and the library used to convert it.
|
|
552
556
|
2. Local files tend to be longer documents and this can lead to hundreds of text chunks per document. These chunk embeddings are averaged into one embedding. It is not obvious this is as effective as vector search on each chunk, but it is more memory efficient.
|
|
553
557
|
|
|
554
|
-
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
558
|
+
For PDF to text we use [[https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/][pymupd4fll]] which works for this proof of concept. There is a Pro version of that package which supports more file formats. It is not obvious what it would cost to use that. I used [[https://ds4sd.github.io/docling/][docling]] in an early prototype. It also worked pretty well, but it was a little slower I think, and would occasionally segfault so I stopped using it. Spacy is integrating PDF to structured data using docling (https://explosion.ai/blog/pdfs-nlp-structured-data). There is plenty of room for improvement in this dimension, with trade offs in performance and accuracy.
|
|
555
559
|
|
|
556
560
|
There is a new package from Microsoft to convert Office files to Markdown (https://github.com/microsoft/markitdown) that they specifically mention using in the context of LLMs.
|
|
557
561
|
|
|
@@ -569,7 +573,7 @@ vectorlite aims to be faster than sqlite-vec, but it relies on hnsw for vector s
|
|
|
569
573
|
|
|
570
574
|
sqlite-vec is nice, and early versions of litdb used it and its precursor. This approach requires a virtual table for the embeddings. This is installed as an extension, and is still considered in early stages of adoption.
|
|
571
575
|
|
|
572
|
-
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
576
|
+
libsql is a fork of sqlite with integrated vector search, and potential for using it as a cloud database. It is supported by a company, with freemium cloud services. In libsql you store the vectors in a regular table, and search on an embedding index. The code is on GitHUB, and can also be used locally.
|
|
573
577
|
|
|
574
578
|
* Roadmap
|
|
575
579
|
|
|
@@ -583,7 +587,7 @@ Notes on the other hand, might be small enough that they could be stored in the
|
|
|
583
587
|
|
|
584
588
|
** Jupyter lab integration
|
|
585
589
|
|
|
586
|
-
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
590
|
+
An alternative to the CLI and Emacs would be to run this in Jupyter Lab with magic commands and rich output.
|
|
587
591
|
|
|
588
592
|
** graph visualization
|
|
589
593
|
|
|
@@ -655,7 +659,7 @@ The first version of litdb with libsql used a fully remote db on their cloud. Th
|
|
|
655
659
|
|
|
656
660
|
** image and text models
|
|
657
661
|
|
|
658
|
-
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
662
|
+
One day it might be possible to include images in this (https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models). At the moment, OpenAlex entries do not have any images, but other web resources and local files could. I have an image database in org-db, but I don't use it a lot.
|
|
659
663
|
|
|
660
664
|
** DONE combine full text and vector search
|
|
661
665
|
CLOSED: [2024-12-09 Mon 13:59]
|
|
@@ -737,11 +741,11 @@ print(text)
|
|
|
737
741
|
#+RESULTS:
|
|
738
742
|
: ++RESULTS:
|
|
739
743
|
: ; Selected region: (26, 332, 473, 69)
|
|
740
|
-
:
|
|
744
|
+
:
|
|
741
745
|
|
|
742
746
|
This might be nice later when we have image embeddings.
|
|
743
747
|
|
|
744
|
-
** DONE review process
|
|
748
|
+
** DONE review process
|
|
745
749
|
|
|
746
750
|
#+BEGIN_SRC sh
|
|
747
751
|
litdb review --since '1 week ago'
|
|
@@ -761,7 +765,7 @@ select source, date_added from sources where date(date_added) > '2024-11-28' lim
|
|
|
761
765
|
| https://doi.org/10.1088/1361-648x/aa680e | 2024-11-29 17:21:53 |
|
|
762
766
|
| https://doi.org/10.1103/physrevlett.93.156801 | 2024-11-29 17:21:54 |
|
|
763
767
|
|
|
764
|
-
** DONE semantic similarity
|
|
768
|
+
** DONE semantic similarity
|
|
765
769
|
CLOSED: [2024-12-04 Wed 13:12]
|
|
766
770
|
|
|
767
771
|
litdb uses cosine similarity as the distance metric for the nearest neighbors. It might be useful to re-rank these with cross-encoding.
|
|
@@ -772,12 +776,12 @@ https://www.sbert.net/examples/applications/cross-encoder/README.html
|
|
|
772
776
|
|
|
773
777
|
- LitSuggest :: https://www.ncbi.nlm.nih.gov/research/litsuggest/
|
|
774
778
|
- Browser tool that suggests literature for you based on positive and negative PMIDs. Hosted by NIH.
|
|
775
|
-
|
|
779
|
+
|
|
776
780
|
- paper-qa :: https://github.com/Future-House/paper-qa
|
|
777
781
|
- This project by Andrew White uses LLM+RAG to explore a paper.
|
|
778
782
|
|
|
779
783
|
- ColBERT :: https://github.com/stanford-futuredata/ColBERT
|
|
780
|
-
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
784
|
+
- ColBERT is a fast retrieval model for large text collections. In theory it can probably be integrated with litdb. litdb is so simple, and works well enough so far without it.
|
|
781
785
|
|
|
782
786
|
Many of these projects require you to make an account. There are freemium levels in each one.
|
|
783
787
|
|
|
@@ -789,7 +793,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
789
793
|
|
|
790
794
|
- Keenious :: https://keenious.com/explore
|
|
791
795
|
- Browser / Google Docs and Word plugin. Finds related articles to the text in your document. I like Keenious when in Google Docs.
|
|
792
|
-
|
|
796
|
+
|
|
793
797
|
- scite.ai :: https://scite.ai/
|
|
794
798
|
- Browser tool that integrates GPT with the scientific literature, integration with Zotero
|
|
795
799
|
|
|
@@ -803,7 +807,7 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
803
807
|
- This is a desktop app that can be totally local, or in the cloud. It can index your files, and then you can chat with them. There is a freemium level.
|
|
804
808
|
|
|
805
809
|
- AnythingLLM :: https://anythingllm.com/
|
|
806
|
-
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
810
|
+
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
807
811
|
|
|
808
812
|
- gpt4all :: https://www.nomic.ai/gpt4all
|
|
809
813
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
@@ -7,7 +7,7 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "litdb"
|
|
10
|
-
version = "2.
|
|
10
|
+
version = "2.1.1"
|
|
11
11
|
authors = [
|
|
12
12
|
{name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
|
|
13
13
|
]
|
|
@@ -22,10 +22,12 @@ dependencies = [
|
|
|
22
22
|
"tomlkit",
|
|
23
23
|
"requests",
|
|
24
24
|
"pymupdf4llm",
|
|
25
|
+
"docling",
|
|
25
26
|
"langchain",
|
|
26
27
|
"tqdm",
|
|
27
28
|
"libsql-experimental",
|
|
28
29
|
"sentence-transformers",
|
|
30
|
+
"transformers",
|
|
29
31
|
"click",
|
|
30
32
|
"rich",
|
|
31
33
|
"beautifulsoup4",
|
|
@@ -35,22 +37,31 @@ dependencies = [
|
|
|
35
37
|
"nbconvert",
|
|
36
38
|
"nbformat",
|
|
37
39
|
"ollama",
|
|
40
|
+
"litellm",
|
|
38
41
|
"dateparser",
|
|
39
42
|
"jinja2",
|
|
40
43
|
"pytesseract",
|
|
41
44
|
"pillow",
|
|
42
45
|
"bibtexparser",
|
|
43
|
-
|
|
46
|
+
|
|
47
|
+
# these are sort of hard to install, and rely on other libraries
|
|
48
|
+
# I document them here, but consider it optional
|
|
44
49
|
# "pyaudio",
|
|
45
50
|
# "playsound",
|
|
46
|
-
# "SpeechRecognition",
|
|
47
51
|
|
|
48
52
|
# for coa
|
|
49
53
|
"pandas",
|
|
50
54
|
"more-itertools",
|
|
51
55
|
"nameparser",
|
|
52
56
|
"tabulate",
|
|
53
|
-
"IPython"
|
|
57
|
+
"IPython",
|
|
58
|
+
|
|
59
|
+
# youtube
|
|
60
|
+
"yt_dlp",
|
|
61
|
+
"webvtt-py",
|
|
62
|
+
|
|
63
|
+
# the streamlit app
|
|
64
|
+
"streamlit"
|
|
54
65
|
]
|
|
55
66
|
|
|
56
67
|
[project.scripts]
|