litdb 2.1.8__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litdb-2.2.0/LICENSE +21 -0
- {litdb-2.1.8/src/litdb.egg-info → litdb-2.2.0}/PKG-INFO +249 -24
- litdb-2.1.8/PKG-INFO → litdb-2.2.0/README.org +184 -61
- litdb-2.2.0/pyproject.toml +201 -0
- litdb-2.2.0/src/litdb/app.py +3399 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/bibtex.py +5 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/chat.py +46 -18
- litdb-2.2.0/src/litdb/cli.py +131 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/coa.py +27 -12
- litdb-2.2.0/src/litdb/commands/__init__.py +14 -0
- litdb-2.2.0/src/litdb/commands/data_processing.py +323 -0
- litdb-2.2.0/src/litdb/commands/export.py +141 -0
- litdb-2.2.0/src/litdb/commands/filters.py +119 -0
- litdb-2.2.0/src/litdb/commands/install.py +237 -0
- litdb-2.2.0/src/litdb/commands/manage.py +298 -0
- litdb-2.2.0/src/litdb/commands/openalex_commands.py +282 -0
- litdb-2.2.0/src/litdb/commands/research_commands.py +291 -0
- litdb-2.2.0/src/litdb/commands/review.py +96 -0
- litdb-2.2.0/src/litdb/commands/search.py +338 -0
- litdb-2.2.0/src/litdb/commands/tags.py +162 -0
- litdb-2.2.0/src/litdb/commands/utilities.py +152 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/crawl.py +10 -7
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/db.py +80 -23
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/doe_collaborators.py +4 -3
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/extract.py +26 -25
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/gpt.py +2 -2
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/images.py +13 -5
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/lsearch.py +15 -3
- litdb-2.2.0/src/litdb/mcp_server.py +689 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/research.py +24 -12
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/server.py +1 -5
- litdb-2.2.0/src/litdb/summary.py +584 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/utils.py +1 -1
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/youtube.py +3 -2
- litdb-2.1.8/README.org → litdb-2.2.0/src/litdb.egg-info/PKG-INFO +286 -2
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb.egg-info/SOURCES.txt +16 -2
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb.egg-info/entry_points.txt +1 -1
- litdb-2.2.0/src/litdb.egg-info/requires.txt +100 -0
- litdb-2.1.8/pyproject.toml +0 -100
- litdb-2.1.8/src/litdb/app.py +0 -83
- litdb-2.1.8/src/litdb/cli.py +0 -1687
- litdb-2.1.8/src/litdb/mcp.py +0 -168
- litdb-2.1.8/src/litdb.egg-info/requires.txt +0 -49
- {litdb-2.1.8 → litdb-2.2.0}/setup.cfg +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/__init__.py +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/audio.py +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/lab.py +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/openalex.py +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb/pdf.py +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.1.8 → litdb-2.2.0}/src/litdb.egg-info/top_level.txt +0 -0
litdb-2.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 John Kitchin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
7
7
|
License-Expression: MIT
|
|
8
8
|
Project-URL: Homepage, https://github.com/jkitchin/litdb
|
|
9
9
|
Description-Content-Type: text/plain
|
|
10
|
+
License-File: LICENSE
|
|
10
11
|
Requires-Dist: toml
|
|
11
12
|
Requires-Dist: tomlkit
|
|
12
13
|
Requires-Dist: requests
|
|
13
14
|
Requires-Dist: pymupdf4llm
|
|
14
15
|
Requires-Dist: docling
|
|
15
|
-
Requires-Dist: langchain
|
|
16
|
+
Requires-Dist: langchain-core
|
|
17
|
+
Requires-Dist: langchain-text-splitters
|
|
16
18
|
Requires-Dist: tqdm
|
|
17
|
-
Requires-Dist: libsql
|
|
19
|
+
Requires-Dist: libsql
|
|
18
20
|
Requires-Dist: sentence-transformers
|
|
19
21
|
Requires-Dist: transformers
|
|
20
22
|
Requires-Dist: click
|
|
@@ -37,25 +39,66 @@ Requires-Dist: more-itertools
|
|
|
37
39
|
Requires-Dist: nameparser
|
|
38
40
|
Requires-Dist: tabulate
|
|
39
41
|
Requires-Dist: IPython
|
|
40
|
-
Requires-Dist:
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
|
|
43
|
-
Requires-Dist:
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist:
|
|
46
|
-
Requires-Dist:
|
|
47
|
-
Requires-Dist:
|
|
48
|
-
|
|
49
|
-
Requires-Dist:
|
|
50
|
-
|
|
51
|
-
Requires-Dist:
|
|
52
|
-
Requires-Dist:
|
|
53
|
-
Requires-Dist:
|
|
54
|
-
Requires-Dist:
|
|
55
|
-
Requires-Dist:
|
|
56
|
-
Requires-Dist:
|
|
57
|
-
Requires-Dist:
|
|
58
|
-
Requires-Dist:
|
|
42
|
+
Requires-Dist: gmft
|
|
43
|
+
Requires-Dist: ruff>=0.13.3
|
|
44
|
+
Provides-Extra: test
|
|
45
|
+
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
46
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "test"
|
|
47
|
+
Requires-Dist: pytest-mock>=3.12.0; extra == "test"
|
|
48
|
+
Requires-Dist: responses>=0.24.0; extra == "test"
|
|
49
|
+
Requires-Dist: freezegun>=1.4.0; extra == "test"
|
|
50
|
+
Provides-Extra: futurehouse
|
|
51
|
+
Requires-Dist: futurehouse_client; extra == "futurehouse"
|
|
52
|
+
Provides-Extra: research
|
|
53
|
+
Requires-Dist: gpt_researcher; extra == "research"
|
|
54
|
+
Requires-Dist: duckduckgo-search; extra == "research"
|
|
55
|
+
Requires-Dist: langchain_ollama; extra == "research"
|
|
56
|
+
Requires-Dist: langchain_google_genai; extra == "research"
|
|
57
|
+
Requires-Dist: langchain_anthropic; extra == "research"
|
|
58
|
+
Requires-Dist: mistune; extra == "research"
|
|
59
|
+
Requires-Dist: htmldocx; extra == "research"
|
|
60
|
+
Requires-Dist: docx; extra == "research"
|
|
61
|
+
Requires-Dist: md2pdf; extra == "research"
|
|
62
|
+
Requires-Dist: pypandoc; extra == "research"
|
|
63
|
+
Provides-Extra: app
|
|
64
|
+
Requires-Dist: streamlit; extra == "app"
|
|
65
|
+
Provides-Extra: youtube
|
|
66
|
+
Requires-Dist: yt_dlp; extra == "youtube"
|
|
67
|
+
Requires-Dist: webvtt-py; extra == "youtube"
|
|
68
|
+
Provides-Extra: crawl
|
|
69
|
+
Requires-Dist: scrapy; extra == "crawl"
|
|
70
|
+
Requires-Dist: backoff; extra == "crawl"
|
|
71
|
+
Provides-Extra: image
|
|
72
|
+
Requires-Dist: pyperclip; extra == "image"
|
|
73
|
+
Requires-Dist: pillow_heif; extra == "image"
|
|
74
|
+
Provides-Extra: mcp
|
|
75
|
+
Requires-Dist: mcp[cli]; extra == "mcp"
|
|
76
|
+
Provides-Extra: audio
|
|
77
|
+
Requires-Dist: pyaudio; extra == "audio"
|
|
78
|
+
Requires-Dist: SpeechRecognition; extra == "audio"
|
|
79
|
+
Provides-Extra: all
|
|
80
|
+
Requires-Dist: futurehouse_client; extra == "all"
|
|
81
|
+
Requires-Dist: gpt_researcher; extra == "all"
|
|
82
|
+
Requires-Dist: duckduckgo-search; extra == "all"
|
|
83
|
+
Requires-Dist: langchain_ollama; extra == "all"
|
|
84
|
+
Requires-Dist: langchain_google_genai; extra == "all"
|
|
85
|
+
Requires-Dist: langchain_anthropic; extra == "all"
|
|
86
|
+
Requires-Dist: mistune; extra == "all"
|
|
87
|
+
Requires-Dist: htmldocx; extra == "all"
|
|
88
|
+
Requires-Dist: docx; extra == "all"
|
|
89
|
+
Requires-Dist: md2pdf; extra == "all"
|
|
90
|
+
Requires-Dist: pypandoc; extra == "all"
|
|
91
|
+
Requires-Dist: streamlit; extra == "all"
|
|
92
|
+
Requires-Dist: yt_dlp; extra == "all"
|
|
93
|
+
Requires-Dist: webvtt-py; extra == "all"
|
|
94
|
+
Requires-Dist: scrapy; extra == "all"
|
|
95
|
+
Requires-Dist: backoff; extra == "all"
|
|
96
|
+
Requires-Dist: pyperclip; extra == "all"
|
|
97
|
+
Requires-Dist: pillow_heif; extra == "all"
|
|
98
|
+
Requires-Dist: mcp[cli]; extra == "all"
|
|
99
|
+
Requires-Dist: pyaudio; extra == "all"
|
|
100
|
+
Requires-Dist: SpeechRecognition; extra == "all"
|
|
101
|
+
Dynamic: license-file
|
|
59
102
|
|
|
60
103
|
#+title: litdb - a literature and document database
|
|
61
104
|
|
|
@@ -63,8 +106,9 @@ Requires-Dist: mcp[cli]
|
|
|
63
106
|
[[./litdb.png]]
|
|
64
107
|
#+BEGIN_html
|
|
65
108
|
<a href="https://github.com/jkitchin/litdb/actions/workflows/build.yml">
|
|
66
|
-
<img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="
|
|
109
|
+
<img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="Build Status">
|
|
67
110
|
</a>
|
|
111
|
+
<img src="https://img.shields.io/badge/tests-133_passed,_66_skipped-brightgreen" alt="Tests">
|
|
68
112
|
#+END_HTML
|
|
69
113
|
|
|
70
114
|
* litdb concept
|
|
@@ -113,6 +157,17 @@ uv pip install litdb
|
|
|
113
157
|
|
|
114
158
|
You have to activate that virtual env to use litdb, which may be annoying.
|
|
115
159
|
|
|
160
|
+
** Recent additions
|
|
161
|
+
|
|
162
|
+
*** Version 2.1.8
|
|
163
|
+
|
|
164
|
+
- *fromtext command*: Extract and add references from pasted text using LLM parsing and CrossRef matching. See [[./FROMTEXT_USAGE.md][detailed documentation]].
|
|
165
|
+
- *summary command*: Generate newsletter-style summaries of recent articles with automatic topic extraction and classification.
|
|
166
|
+
- *extract command*: Extract tables from PDF files.
|
|
167
|
+
- *schema command*: Extract structured data from documents using a flexible schema DSL or JSON format.
|
|
168
|
+
- *Bug fix*: Corrected OpenAlex API parameter from ~email~ to ~mailto~ for proper API compliance.
|
|
169
|
+
- *Security fix*: Replaced unsafe ~eval()~ with ~ast.literal_eval()~ in schema parsing.
|
|
170
|
+
|
|
116
171
|
** configuration
|
|
117
172
|
|
|
118
173
|
You have to create a toml configuration file. This file is called litdb.toml. The directory this file is in is considered the root directory. All commands will start in the current working directory and look up to find this file. You can put this file in your home directory, or you can have sub-directories, e.g. a per project litdb.
|
|
@@ -123,23 +178,34 @@ You will need an OpenAlex premium key if you want to use the update-filters feat
|
|
|
123
178
|
|
|
124
179
|
#+BEGIN_EXAMPLE
|
|
125
180
|
[embedding]
|
|
181
|
+
# SentenceTransformer model for vector search
|
|
126
182
|
model = 'all-MiniLM-L6-v2'
|
|
127
183
|
cross-encoder = 'cross-encoder/ms-marco-MiniLM-L-6-v2'
|
|
128
184
|
chunk_size = 1000
|
|
129
185
|
chunk_overlap = 200
|
|
130
186
|
|
|
131
|
-
|
|
132
187
|
[openalex]
|
|
188
|
+
# Email for OpenAlex API (polite pool access)
|
|
133
189
|
email = "you@example.com"
|
|
190
|
+
# Optional: Premium API key for update-filters feature
|
|
134
191
|
api_key = "..."
|
|
135
192
|
|
|
136
193
|
[gpt]
|
|
194
|
+
# Model for the 'litdb gpt' command (uses ollama)
|
|
137
195
|
model = "llama2"
|
|
138
196
|
|
|
139
197
|
[llm]
|
|
198
|
+
# Model for LiteLLM-based commands (chat, fromtext, summary, schema, research)
|
|
199
|
+
# Supports any provider: ollama/*, openai/*, anthropic/*, google_genai/*, etc.
|
|
140
200
|
model = "ollama/llama2"
|
|
141
201
|
#+END_EXAMPLE
|
|
142
202
|
|
|
203
|
+
Configuration sections and their usage:
|
|
204
|
+
- *[embedding]*: Used by all vector search operations (~vsearch~, ~hybrid-search~, etc.)
|
|
205
|
+
- *[openalex]*: Required for OpenAlex searches and updates
|
|
206
|
+
- *[gpt]*: Used by the ~litdb gpt~ command (ollama-only)
|
|
207
|
+
- *[llm]*: Used by ~chat~, ~fromtext~, ~summary~, ~schema~, and ~research~ commands (supports all LiteLLM providers)
|
|
208
|
+
|
|
143
209
|
You can define an environment variable to the root of your default litdb project. This should be a directory with a litdb.toml file in it.
|
|
144
210
|
|
|
145
211
|
#+BEGIN_SRC sh
|
|
@@ -158,6 +224,21 @@ litdb has a cli with an entry command of litdb and subcommands (like git) for in
|
|
|
158
224
|
litdb --help
|
|
159
225
|
#+END_SRC
|
|
160
226
|
|
|
227
|
+
** Command Organization
|
|
228
|
+
|
|
229
|
+
litdb's 43 commands are organized into logical groups:
|
|
230
|
+
|
|
231
|
+
- *Management*: init, add, remove, index, reindex, update_embeddings
|
|
232
|
+
- *Search*: vsearch, fulltext, hybrid_search, lsearch, similar, image_search, screenshot
|
|
233
|
+
- *Export*: bibtex, citation, show, visit, about, sql
|
|
234
|
+
- *Tags*: add_tag, rm_tag, delete_tag, show_tag, list_tags
|
|
235
|
+
- *Review*: review, summary
|
|
236
|
+
- *Filters*: add_filter, rm_filter, update_filters, list_filters
|
|
237
|
+
- *OpenAlex*: openalex, author_search, follow, watch, citing, related, unpaywall
|
|
238
|
+
- *Research*: fhresearch, research, suggest_reviewers
|
|
239
|
+
- *Data Processing*: crossref, fromtext, extract, schema, crawl
|
|
240
|
+
- *Utilities*: web, audio, chat, app, version, coa
|
|
241
|
+
|
|
161
242
|
** Searching the web
|
|
162
243
|
|
|
163
244
|
You have to start somewhere. You can use this to open a search in OpenAlex.
|
|
@@ -244,6 +325,37 @@ These are all one-time additions.
|
|
|
244
325
|
|
|
245
326
|
You can also add things like YouTube videos and podcasts. We use ML to extract the audio from these to text so they become searchable!
|
|
246
327
|
|
|
328
|
+
*** Adding references from text
|
|
329
|
+
|
|
330
|
+
You can paste text containing academic references and litdb will use an LLM to extract and add them automatically. This is useful when copying reference lists from PDFs, papers, or websites.
|
|
331
|
+
|
|
332
|
+
#+BEGIN_SRC sh
|
|
333
|
+
litdb fromtext "Kitchin, Examples of Effective Data Sharing in Scientific Publishing, ACS Catalysis, 2015, DOI: 10.1021/acscatal.5b00538"
|
|
334
|
+
#+END_SRC
|
|
335
|
+
|
|
336
|
+
You can also add multiple references at once:
|
|
337
|
+
|
|
338
|
+
#+BEGIN_SRC sh
|
|
339
|
+
litdb fromtext "Recent work by Smith et al. (Nature, 2020) showed...
|
|
340
|
+
Another study by Jones (Science, 2019) demonstrated..."
|
|
341
|
+
#+END_SRC
|
|
342
|
+
|
|
343
|
+
The command supports the same options as ~litdb add~ for adding related works:
|
|
344
|
+
|
|
345
|
+
#+BEGIN_SRC sh
|
|
346
|
+
litdb fromtext "TEXT" --references --related --citing
|
|
347
|
+
#+END_SRC
|
|
348
|
+
|
|
349
|
+
You can also specify which LLM model to use:
|
|
350
|
+
|
|
351
|
+
#+BEGIN_SRC sh
|
|
352
|
+
litdb fromtext "TEXT" --model "gpt-4o"
|
|
353
|
+
#+END_SRC
|
|
354
|
+
|
|
355
|
+
The LLM extracts structured reference information, then litdb uses CrossRef to find DOIs and match references. High-confidence matches are added automatically, while lower-confidence matches prompt for confirmation.
|
|
356
|
+
|
|
357
|
+
See [[./FROMTEXT_USAGE.md][detailed documentation]] for more information on supported citation formats and how the matching works.
|
|
358
|
+
|
|
247
359
|
** Adding filters
|
|
248
360
|
|
|
249
361
|
litdb provides several convenient ways to add queries to update your litdb in the future.
|
|
@@ -314,6 +426,47 @@ litdb review -s "1 week ago"
|
|
|
314
426
|
|
|
315
427
|
This works best when you update your litdb regularly. You might want to redirect that into a file so you can review it in an editor of your choice.
|
|
316
428
|
|
|
429
|
+
** Generating newsletter summaries
|
|
430
|
+
|
|
431
|
+
You can generate a newsletter-style summary of recently added articles. This command uses an LLM to analyze articles, extract topics, aggregate them into themes, and create narrative summaries organized by topic.
|
|
432
|
+
|
|
433
|
+
#+BEGIN_SRC sh
|
|
434
|
+
litdb summary -s "1 week" -o newsletter.org
|
|
435
|
+
#+END_SRC
|
|
436
|
+
|
|
437
|
+
The default time period is "1 week", but you can specify different periods using natural language:
|
|
438
|
+
|
|
439
|
+
#+BEGIN_SRC sh
|
|
440
|
+
litdb summary -s "2 weeks"
|
|
441
|
+
litdb summary -s "1 month"
|
|
442
|
+
litdb summary -s "3 days"
|
|
443
|
+
#+END_SRC
|
|
444
|
+
|
|
445
|
+
You can specify which LLM model to use (defaults to the model in your ~litdb.toml~ config):
|
|
446
|
+
|
|
447
|
+
#+BEGIN_SRC sh
|
|
448
|
+
litdb summary -s "1 week" --model "gpt-4o"
|
|
449
|
+
litdb summary -s "2 weeks" --model "anthropic/claude-3-5-sonnet-20241022"
|
|
450
|
+
#+END_SRC
|
|
451
|
+
|
|
452
|
+
If you omit the ~-o~ option, the summary is written to a file with an auto-generated name based on the date range.
|
|
453
|
+
|
|
454
|
+
The output is an org-mode formatted newsletter with:
|
|
455
|
+
1. Main topic sections (5-10 automatically identified themes)
|
|
456
|
+
2. Subtopic subsections with narrative summaries
|
|
457
|
+
3. Numbered references within the narrative text
|
|
458
|
+
4. Bibliography of articles for each subtopic
|
|
459
|
+
|
|
460
|
+
This command works best with articles that have abstracts. It only processes entries classified as articles (journal articles, reviews, preprints, etc.), skipping other entry types like books or datasets.
|
|
461
|
+
|
|
462
|
+
The process:
|
|
463
|
+
1. Fetches articles added since the specified date
|
|
464
|
+
2. Extracts 3-5 topics from each article using an LLM
|
|
465
|
+
3. Aggregates all topics into 5-10 main themes with subtopics
|
|
466
|
+
4. Classifies each article into the most relevant topic/subtopic
|
|
467
|
+
5. Generates narrative summaries for each subtopic, referencing the articles
|
|
468
|
+
6. Formats everything as an org-mode document
|
|
469
|
+
|
|
317
470
|
** Searching litdb
|
|
318
471
|
|
|
319
472
|
There are several search options.
|
|
@@ -590,6 +743,70 @@ brew list pango gobject-introspection
|
|
|
590
743
|
export DYLD_LIBRARY_PATH=$(brew --prefix)/lib:$DYLD_LIBRARY_PATH
|
|
591
744
|
#+END_SRC
|
|
592
745
|
|
|
746
|
+
** Extracting tables from PDFs
|
|
747
|
+
|
|
748
|
+
You can extract tables from PDF files using the ~extract~ command. This uses computer vision and machine learning to identify and extract tables.
|
|
749
|
+
|
|
750
|
+
#+BEGIN_SRC sh
|
|
751
|
+
litdb extract paper.pdf
|
|
752
|
+
#+END_SRC
|
|
753
|
+
|
|
754
|
+
This will extract all tables from the PDF. To extract specific tables (using 1-based indexing):
|
|
755
|
+
|
|
756
|
+
#+BEGIN_SRC sh
|
|
757
|
+
litdb extract paper.pdf -t 1 -t 3
|
|
758
|
+
#+END_SRC
|
|
759
|
+
|
|
760
|
+
This extracts only tables 1 and 3. You can specify an output format with the ~-f~ option:
|
|
761
|
+
|
|
762
|
+
#+BEGIN_SRC sh
|
|
763
|
+
litdb extract paper.pdf -t 1 -f markdown
|
|
764
|
+
#+END_SRC
|
|
765
|
+
|
|
766
|
+
The extracted tables are returned as pandas DataFrames, which can be formatted in various ways.
|
|
767
|
+
|
|
768
|
+
** Extracting structured data with schemas
|
|
769
|
+
|
|
770
|
+
You can extract structured information from documents (PDFs, web pages, etc.) using the ~schema~ command. This uses an LLM to parse the document and extract data according to a schema you define.
|
|
771
|
+
|
|
772
|
+
*** Using the schema DSL
|
|
773
|
+
|
|
774
|
+
The schema command supports a simple domain-specific language (DSL) for defining schemas:
|
|
775
|
+
|
|
776
|
+
#+BEGIN_SRC sh
|
|
777
|
+
litdb schema paper.pdf "title:str, authors:list, year:int, doi?:str"
|
|
778
|
+
#+END_SRC
|
|
779
|
+
|
|
780
|
+
Schema DSL syntax:
|
|
781
|
+
- ~fieldName:type~ - Required field with type
|
|
782
|
+
- ~fieldName?:type~ - Optional field (can be None)
|
|
783
|
+
- ~fieldName:type=defaultValue~ - Field with default value
|
|
784
|
+
- ~fieldName?:type=defaultValue~ - Optional field with default
|
|
785
|
+
|
|
786
|
+
Supported types: ~str~, ~int~, ~float~, ~bool~, ~list~, ~dict~
|
|
787
|
+
|
|
788
|
+
Examples:
|
|
789
|
+
|
|
790
|
+
#+BEGIN_SRC sh
|
|
791
|
+
# Extract basic paper metadata
|
|
792
|
+
litdb schema paper.pdf "title:str, authors:list, year:int"
|
|
793
|
+
|
|
794
|
+
# Extract with optional DOI and default value for journal
|
|
795
|
+
litdb schema paper.pdf "title:str, doi?:str, journal:str=Unknown"
|
|
796
|
+
|
|
797
|
+
# Extract experimental data
|
|
798
|
+
litdb schema data.pdf "temperature:float, pressure:float, yield:float, catalyst:str"
|
|
799
|
+
#+END_SRC
|
|
800
|
+
|
|
801
|
+
*** Using JSON schemas
|
|
802
|
+
|
|
803
|
+
You can also provide a schema as a JSON object:
|
|
804
|
+
|
|
805
|
+
#+BEGIN_SRC sh
|
|
806
|
+
litdb schema paper.pdf '{"title": ["str", None], "year": ["int", 2024]}'
|
|
807
|
+
#+END_SRC
|
|
808
|
+
|
|
809
|
+
The schema command first converts the document to markdown using Docling, then uses an LLM with structured output to extract the requested fields. The LLM model used is configured in the ~[llm]~ section of your ~litdb.toml~ file.
|
|
593
810
|
|
|
594
811
|
** Claude Desktop + MCP
|
|
595
812
|
|
|
@@ -972,3 +1189,11 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
972
1189
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
973
1190
|
|
|
974
1191
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1192
|
+
|
|
1193
|
+
* Tasks
|
|
1194
|
+
|
|
1195
|
+
** DONE create a Claude skill for litdb
|
|
1196
|
+
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1197
|
+
|
|
1198
|
+
** DONE Make a nicer streamlit app for the front end.
|
|
1199
|
+
CLOSED: [2025-10-28 Tue 10:40]
|