litdb 2.2.1__tar.gz → 2.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {litdb-2.2.1/src/litdb.egg-info → litdb-2.2.3}/PKG-INFO +98 -17
- {litdb-2.2.1 → litdb-2.2.3}/README.org +92 -14
- {litdb-2.2.1 → litdb-2.2.3}/pyproject.toml +6 -3
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/app.py +10 -8
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/bibtex.py +23 -8
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/chat.py +10 -8
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/coa.py +83 -14
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/data_processing.py +37 -4
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/install.py +8 -2
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/manage.py +56 -1
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/utilities.py +19 -6
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/db.py +177 -28
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/doe_collaborators.py +15 -11
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/mcp_server.py +59 -25
- {litdb-2.2.1 → litdb-2.2.3/src/litdb.egg-info}/PKG-INFO +98 -17
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/requires.txt +5 -2
- {litdb-2.2.1 → litdb-2.2.3}/LICENSE +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/setup.cfg +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/__init__.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/audio.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/cli.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/__init__.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/export.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/filters.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/openalex_commands.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/research_commands.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/review.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/search.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/tags.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/crawl.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/extract.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/gpt.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/images.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/lab.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/lsearch.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/openalex.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/pdf.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/research.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/server.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/summary.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/utils.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb/youtube.py +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/SOURCES.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/entry_points.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.3
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,7 +11,7 @@ License-File: LICENSE
|
|
|
11
11
|
Requires-Dist: toml
|
|
12
12
|
Requires-Dist: tomlkit
|
|
13
13
|
Requires-Dist: requests
|
|
14
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: pymupdf4llm<2
|
|
15
15
|
Requires-Dist: docling
|
|
16
16
|
Requires-Dist: langchain-core
|
|
17
17
|
Requires-Dist: langchain-text-splitters
|
|
@@ -33,7 +33,7 @@ Requires-Dist: dateparser
|
|
|
33
33
|
Requires-Dist: jinja2
|
|
34
34
|
Requires-Dist: pytesseract
|
|
35
35
|
Requires-Dist: pillow
|
|
36
|
-
Requires-Dist: bibtexparser
|
|
36
|
+
Requires-Dist: bibtexparser<2
|
|
37
37
|
Requires-Dist: pandas
|
|
38
38
|
Requires-Dist: more-itertools
|
|
39
39
|
Requires-Dist: nameparser
|
|
@@ -41,6 +41,9 @@ Requires-Dist: tabulate
|
|
|
41
41
|
Requires-Dist: IPython
|
|
42
42
|
Requires-Dist: gmft
|
|
43
43
|
Requires-Dist: ruff>=0.13.3
|
|
44
|
+
Requires-Dist: torch
|
|
45
|
+
Requires-Dist: torchvision
|
|
46
|
+
Requires-Dist: fastmcp>=2.13.3
|
|
44
47
|
Provides-Extra: test
|
|
45
48
|
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
46
49
|
Requires-Dist: pytest-cov>=4.1.0; extra == "test"
|
|
@@ -109,6 +112,9 @@ Dynamic: license-file
|
|
|
109
112
|
<img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="Build Status">
|
|
110
113
|
</a>
|
|
111
114
|
<img src="https://img.shields.io/badge/tests-133_passed,_66_skipped-brightgreen" alt="Tests">
|
|
115
|
+
<a href="https://doi.org/10.5281/zenodo.23514948">
|
|
116
|
+
<img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23514948.svg" alt="DOI">
|
|
117
|
+
</a>
|
|
112
118
|
#+END_HTML
|
|
113
119
|
|
|
114
120
|
* litdb concept
|
|
@@ -155,7 +161,13 @@ source .venv/bin/activate
|
|
|
155
161
|
uv pip install litdb
|
|
156
162
|
#+END_SRC
|
|
157
163
|
|
|
158
|
-
You have to activate that virtual env to use litdb, which may be annoying.
|
|
164
|
+
You have to activate that virtual env to use litdb, which may be annoying. An alternative is to install litdb as a uv tool, which puts ~litdb~ and ~litdb_mcp~ on your PATH in their own environment:
|
|
165
|
+
|
|
166
|
+
#+BEGIN_SRC sh
|
|
167
|
+
uv tool install litdb
|
|
168
|
+
#+END_SRC
|
|
169
|
+
|
|
170
|
+
This is also the easiest way to make the MCP server available to AI agents. See [[*Using litdb with AI agents (skill and MCP server)]].
|
|
159
171
|
|
|
160
172
|
** Recent additions
|
|
161
173
|
|
|
@@ -574,7 +586,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
574
586
|
|
|
575
587
|
Note that you need to install these libraries for this feature to work:
|
|
576
588
|
|
|
577
|
-
pyaudio,
|
|
589
|
+
pyaudio, SpeechRecognition
|
|
578
590
|
|
|
579
591
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
580
592
|
|
|
@@ -808,16 +820,93 @@ litdb schema paper.pdf '{"title": ["str", None], "year": ["int", 2024]}'
|
|
|
808
820
|
|
|
809
821
|
The schema command first converts the document to markdown using Docling, then uses an LLM with structured output to extract the requested fields. The LLM model used is configured in the ~[llm]~ section of your ~litdb.toml~ file.
|
|
810
822
|
|
|
811
|
-
**
|
|
823
|
+
** Using litdb with AI agents (skill and MCP server)
|
|
824
|
+
|
|
825
|
+
litdb comes with two things for AI coding agents such as Claude Code, VS Code (GitHub Copilot), Cursor, Codex, OpenCode and Gemini CLI:
|
|
826
|
+
|
|
827
|
+
- An *agent skill* ([[./skills/litdb/SKILL.md][skills/litdb/SKILL.md]]) that teaches the agent how to use the litdb CLI: adding papers, searching, filters, tags, exporting BibTeX, and so on.
|
|
828
|
+
- An *MCP server* (~litdb_mcp~) that gives the agent direct, read-only tools for your database: ~vsearch~, ~fulltext_search~, ~find_similar~, ~openalex~, ~get_source_details~, ~get_citation~, ~get_bibtex~, ~list_tags~, ~get_tagged_articles~ and ~generate_nsf_coa~.
|
|
829
|
+
|
|
830
|
+
The easiest way to install both is [[https://github.com/jkitchin/knowhow][knowhow]], which needs nothing installed besides [[https://github.com/astral-sh/uv][uv]]. The skill follows the [[https://agentskills.io/specification][Agent Skills specification]], so it also works with ~npx skills~.
|
|
831
|
+
|
|
832
|
+
*** Install the skill
|
|
833
|
+
|
|
834
|
+
#+BEGIN_SRC sh
|
|
835
|
+
# See what litdb offers
|
|
836
|
+
uvx knowhow add jkitchin/litdb --list
|
|
837
|
+
|
|
838
|
+
# Install the skill in the current project
|
|
839
|
+
uvx knowhow add jkitchin/litdb --skill litdb
|
|
840
|
+
|
|
841
|
+
# or for your user, in every project
|
|
842
|
+
uvx knowhow add jkitchin/litdb --skill litdb -g
|
|
843
|
+
#+END_SRC
|
|
844
|
+
|
|
845
|
+
The skill is copied to ~.agents/skills/litdb~ (read by VS Code, Codex, OpenCode, Gemini CLI, Cursor and Copilot) and linked into ~.claude/skills/litdb~ for Claude Code. Then ask your agent things like "add the references of 10.1021/jp047349j to my litdb" or "make a bibtex file of everything tagged thesis".
|
|
846
|
+
|
|
847
|
+
Keep it up to date with ~uvx knowhow check~ and ~uvx knowhow upgrade~, and remove it with ~uvx knowhow remove litdb~.
|
|
848
|
+
|
|
849
|
+
If you work from a clone of this repository, ~litdb install claude-skill~ also copies the skill to ~~/.claude/skills/litdb~.
|
|
850
|
+
|
|
851
|
+
*** Install the MCP server
|
|
852
|
+
|
|
853
|
+
The MCP server runs the ~litdb_mcp~ command, so first make sure it is on your PATH (~uv tool install litdb~ does this). The server is defined in [[./mcp/litdb.json][mcp/litdb.json]], and knowhow translates it into each agent's config format:
|
|
854
|
+
|
|
855
|
+
#+BEGIN_SRC sh
|
|
856
|
+
# Claude Code, in this project (writes .mcp.json)
|
|
857
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code
|
|
858
|
+
|
|
859
|
+
# Claude Code for all your projects
|
|
860
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -g
|
|
861
|
+
|
|
862
|
+
# Several agents at once
|
|
863
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -a vscode -a cursor -a codex
|
|
864
|
+
#+END_SRC
|
|
865
|
+
|
|
866
|
+
knowhow shows you the server config and asks for confirmation before installing it (add ~-y~ to skip the prompt).
|
|
867
|
+
|
|
868
|
+
The server needs to know which database to use. It looks for:
|
|
869
|
+
|
|
870
|
+
1. the ~litdb~ environment variable, the path to a ~litdb.libsql~ file;
|
|
871
|
+
2. a ~litdb.toml~ in the agent's working directory or one of its parents;
|
|
872
|
+
3. the ~LITDB_ROOT~ environment variable, the directory that contains ~litdb.toml~.
|
|
873
|
+
|
|
874
|
+
The installed config passes ~LITDB_ROOT~ through from your environment, so set it in your shell profile before starting the agent:
|
|
812
875
|
|
|
813
876
|
#+BEGIN_SRC sh
|
|
814
|
-
|
|
877
|
+
export LITDB_ROOT=~/litdb # the directory with litdb.toml and litdb.libsql
|
|
815
878
|
#+END_SRC
|
|
816
879
|
|
|
817
|
-
|
|
880
|
+
To check it works in Claude Code, run ~/mcp~ and look for ~litdb~, or ask "what is in my litdb about alloy segregation?". Remove the server with ~uvx knowhow remove --mcp litdb~.
|
|
881
|
+
|
|
882
|
+
If you prefer to configure an agent by hand, this is the config (Claude Code ~.mcp.json~ format):
|
|
818
883
|
|
|
819
|
-
|
|
884
|
+
#+BEGIN_SRC json
|
|
885
|
+
{
|
|
886
|
+
"mcpServers": {
|
|
887
|
+
"litdb": {
|
|
888
|
+
"command": "litdb_mcp",
|
|
889
|
+
"env": {"LITDB_ROOT": "/path/to/your/litdb"}
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
#+END_SRC
|
|
820
894
|
|
|
895
|
+
or with the Claude Code CLI:
|
|
896
|
+
|
|
897
|
+
#+BEGIN_SRC sh
|
|
898
|
+
claude mcp add litdb --scope user -e LITDB_ROOT=/path/to/your/litdb -- litdb_mcp
|
|
899
|
+
#+END_SRC
|
|
900
|
+
|
|
901
|
+
*** Claude Desktop
|
|
902
|
+
|
|
903
|
+
Claude Desktop is not supported by knowhow. Instead, run this from your litdb directory (the one with ~litdb.toml~), or pass ~--db /path/to/litdb.libsql~:
|
|
904
|
+
|
|
905
|
+
#+BEGIN_SRC sh
|
|
906
|
+
litdb install mcp-server
|
|
907
|
+
#+END_SRC
|
|
908
|
+
|
|
909
|
+
Then restart Claude Desktop. Now you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation." Uninstall it with ~litdb install uninstall-mcp~.
|
|
821
910
|
|
|
822
911
|
* Adding local files
|
|
823
912
|
|
|
@@ -1189,11 +1278,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1189
1278
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1190
1279
|
|
|
1191
1280
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1192
|
-
|
|
1193
|
-
* Tasks
|
|
1194
|
-
|
|
1195
|
-
** DONE create a Claude skill for litdb
|
|
1196
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1197
|
-
|
|
1198
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1199
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
<img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="Build Status">
|
|
8
8
|
</a>
|
|
9
9
|
<img src="https://img.shields.io/badge/tests-133_passed,_66_skipped-brightgreen" alt="Tests">
|
|
10
|
+
<a href="https://doi.org/10.5281/zenodo.23514948">
|
|
11
|
+
<img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23514948.svg" alt="DOI">
|
|
12
|
+
</a>
|
|
10
13
|
#+END_HTML
|
|
11
14
|
|
|
12
15
|
* litdb concept
|
|
@@ -53,7 +56,13 @@ source .venv/bin/activate
|
|
|
53
56
|
uv pip install litdb
|
|
54
57
|
#+END_SRC
|
|
55
58
|
|
|
56
|
-
You have to activate that virtual env to use litdb, which may be annoying.
|
|
59
|
+
You have to activate that virtual env to use litdb, which may be annoying. An alternative is to install litdb as a uv tool, which puts ~litdb~ and ~litdb_mcp~ on your PATH in their own environment:
|
|
60
|
+
|
|
61
|
+
#+BEGIN_SRC sh
|
|
62
|
+
uv tool install litdb
|
|
63
|
+
#+END_SRC
|
|
64
|
+
|
|
65
|
+
This is also the easiest way to make the MCP server available to AI agents. See [[*Using litdb with AI agents (skill and MCP server)]].
|
|
57
66
|
|
|
58
67
|
** Recent additions
|
|
59
68
|
|
|
@@ -472,7 +481,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
472
481
|
|
|
473
482
|
Note that you need to install these libraries for this feature to work:
|
|
474
483
|
|
|
475
|
-
pyaudio,
|
|
484
|
+
pyaudio, SpeechRecognition
|
|
476
485
|
|
|
477
486
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
478
487
|
|
|
@@ -706,16 +715,93 @@ litdb schema paper.pdf '{"title": ["str", None], "year": ["int", 2024]}'
|
|
|
706
715
|
|
|
707
716
|
The schema command first converts the document to markdown using Docling, then uses an LLM with structured output to extract the requested fields. The LLM model used is configured in the ~[llm]~ section of your ~litdb.toml~ file.
|
|
708
717
|
|
|
709
|
-
**
|
|
718
|
+
** Using litdb with AI agents (skill and MCP server)
|
|
719
|
+
|
|
720
|
+
litdb comes with two things for AI coding agents such as Claude Code, VS Code (GitHub Copilot), Cursor, Codex, OpenCode and Gemini CLI:
|
|
721
|
+
|
|
722
|
+
- An *agent skill* ([[./skills/litdb/SKILL.md][skills/litdb/SKILL.md]]) that teaches the agent how to use the litdb CLI: adding papers, searching, filters, tags, exporting BibTeX, and so on.
|
|
723
|
+
- An *MCP server* (~litdb_mcp~) that gives the agent direct, read-only tools for your database: ~vsearch~, ~fulltext_search~, ~find_similar~, ~openalex~, ~get_source_details~, ~get_citation~, ~get_bibtex~, ~list_tags~, ~get_tagged_articles~ and ~generate_nsf_coa~.
|
|
724
|
+
|
|
725
|
+
The easiest way to install both is [[https://github.com/jkitchin/knowhow][knowhow]], which needs nothing installed besides [[https://github.com/astral-sh/uv][uv]]. The skill follows the [[https://agentskills.io/specification][Agent Skills specification]], so it also works with ~npx skills~.
|
|
726
|
+
|
|
727
|
+
*** Install the skill
|
|
728
|
+
|
|
729
|
+
#+BEGIN_SRC sh
|
|
730
|
+
# See what litdb offers
|
|
731
|
+
uvx knowhow add jkitchin/litdb --list
|
|
732
|
+
|
|
733
|
+
# Install the skill in the current project
|
|
734
|
+
uvx knowhow add jkitchin/litdb --skill litdb
|
|
735
|
+
|
|
736
|
+
# or for your user, in every project
|
|
737
|
+
uvx knowhow add jkitchin/litdb --skill litdb -g
|
|
738
|
+
#+END_SRC
|
|
739
|
+
|
|
740
|
+
The skill is copied to ~.agents/skills/litdb~ (read by VS Code, Codex, OpenCode, Gemini CLI, Cursor and Copilot) and linked into ~.claude/skills/litdb~ for Claude Code. Then ask your agent things like "add the references of 10.1021/jp047349j to my litdb" or "make a bibtex file of everything tagged thesis".
|
|
741
|
+
|
|
742
|
+
Keep it up to date with ~uvx knowhow check~ and ~uvx knowhow upgrade~, and remove it with ~uvx knowhow remove litdb~.
|
|
743
|
+
|
|
744
|
+
If you work from a clone of this repository, ~litdb install claude-skill~ also copies the skill to ~~/.claude/skills/litdb~.
|
|
745
|
+
|
|
746
|
+
*** Install the MCP server
|
|
747
|
+
|
|
748
|
+
The MCP server runs the ~litdb_mcp~ command, so first make sure it is on your PATH (~uv tool install litdb~ does this). The server is defined in [[./mcp/litdb.json][mcp/litdb.json]], and knowhow translates it into each agent's config format:
|
|
749
|
+
|
|
750
|
+
#+BEGIN_SRC sh
|
|
751
|
+
# Claude Code, in this project (writes .mcp.json)
|
|
752
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code
|
|
753
|
+
|
|
754
|
+
# Claude Code for all your projects
|
|
755
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -g
|
|
756
|
+
|
|
757
|
+
# Several agents at once
|
|
758
|
+
uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -a vscode -a cursor -a codex
|
|
759
|
+
#+END_SRC
|
|
760
|
+
|
|
761
|
+
knowhow shows you the server config and asks for confirmation before installing it (add ~-y~ to skip the prompt).
|
|
762
|
+
|
|
763
|
+
The server needs to know which database to use. It looks for:
|
|
764
|
+
|
|
765
|
+
1. the ~litdb~ environment variable, the path to a ~litdb.libsql~ file;
|
|
766
|
+
2. a ~litdb.toml~ in the agent's working directory or one of its parents;
|
|
767
|
+
3. the ~LITDB_ROOT~ environment variable, the directory that contains ~litdb.toml~.
|
|
768
|
+
|
|
769
|
+
The installed config passes ~LITDB_ROOT~ through from your environment, so set it in your shell profile before starting the agent:
|
|
710
770
|
|
|
711
771
|
#+BEGIN_SRC sh
|
|
712
|
-
|
|
772
|
+
export LITDB_ROOT=~/litdb # the directory with litdb.toml and litdb.libsql
|
|
713
773
|
#+END_SRC
|
|
714
774
|
|
|
715
|
-
|
|
775
|
+
To check it works in Claude Code, run ~/mcp~ and look for ~litdb~, or ask "what is in my litdb about alloy segregation?". Remove the server with ~uvx knowhow remove --mcp litdb~.
|
|
776
|
+
|
|
777
|
+
If you prefer to configure an agent by hand, this is the config (Claude Code ~.mcp.json~ format):
|
|
716
778
|
|
|
717
|
-
|
|
779
|
+
#+BEGIN_SRC json
|
|
780
|
+
{
|
|
781
|
+
"mcpServers": {
|
|
782
|
+
"litdb": {
|
|
783
|
+
"command": "litdb_mcp",
|
|
784
|
+
"env": {"LITDB_ROOT": "/path/to/your/litdb"}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
#+END_SRC
|
|
718
789
|
|
|
790
|
+
or with the Claude Code CLI:
|
|
791
|
+
|
|
792
|
+
#+BEGIN_SRC sh
|
|
793
|
+
claude mcp add litdb --scope user -e LITDB_ROOT=/path/to/your/litdb -- litdb_mcp
|
|
794
|
+
#+END_SRC
|
|
795
|
+
|
|
796
|
+
*** Claude Desktop
|
|
797
|
+
|
|
798
|
+
Claude Desktop is not supported by knowhow. Instead, run this from your litdb directory (the one with ~litdb.toml~), or pass ~--db /path/to/litdb.libsql~:
|
|
799
|
+
|
|
800
|
+
#+BEGIN_SRC sh
|
|
801
|
+
litdb install mcp-server
|
|
802
|
+
#+END_SRC
|
|
803
|
+
|
|
804
|
+
Then restart Claude Desktop. Now you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation." Uninstall it with ~litdb install uninstall-mcp~.
|
|
719
805
|
|
|
720
806
|
* Adding local files
|
|
721
807
|
|
|
@@ -1087,11 +1173,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1087
1173
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1088
1174
|
|
|
1089
1175
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1090
|
-
|
|
1091
|
-
* Tasks
|
|
1092
|
-
|
|
1093
|
-
** DONE create a Claude skill for litdb
|
|
1094
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1095
|
-
|
|
1096
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1097
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -7,7 +7,7 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "litdb"
|
|
10
|
-
version = "2.2.
|
|
10
|
+
version = "2.2.3"
|
|
11
11
|
authors = [
|
|
12
12
|
{name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
|
|
13
13
|
]
|
|
@@ -22,7 +22,7 @@ dependencies = [
|
|
|
22
22
|
"toml",
|
|
23
23
|
"tomlkit",
|
|
24
24
|
"requests",
|
|
25
|
-
"pymupdf4llm",
|
|
25
|
+
"pymupdf4llm<2",
|
|
26
26
|
"docling",
|
|
27
27
|
"langchain-core",
|
|
28
28
|
"langchain-text-splitters",
|
|
@@ -44,7 +44,7 @@ dependencies = [
|
|
|
44
44
|
"jinja2",
|
|
45
45
|
"pytesseract",
|
|
46
46
|
"pillow",
|
|
47
|
-
"bibtexparser",
|
|
47
|
+
"bibtexparser<2",
|
|
48
48
|
# Data processing and COA
|
|
49
49
|
"pandas",
|
|
50
50
|
"more-itertools",
|
|
@@ -55,6 +55,9 @@ dependencies = [
|
|
|
55
55
|
"gmft",
|
|
56
56
|
# Development tools
|
|
57
57
|
"ruff>=0.13.3",
|
|
58
|
+
"torch",
|
|
59
|
+
"torchvision",
|
|
60
|
+
"fastmcp>=2.13.3",
|
|
58
61
|
]
|
|
59
62
|
|
|
60
63
|
[project.scripts]
|
|
@@ -205,10 +205,11 @@ def format_citation(extra):
|
|
|
205
205
|
parts.append(data["title"])
|
|
206
206
|
|
|
207
207
|
# Journal/venue
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
208
|
+
pl = data.get("primary_location") or {}
|
|
209
|
+
src = pl.get("source") or {}
|
|
210
|
+
venue = src.get("display_name", "")
|
|
211
|
+
if venue:
|
|
212
|
+
parts.append(f"*{venue}*")
|
|
212
213
|
|
|
213
214
|
return ". ".join(parts) if parts else None
|
|
214
215
|
except Exception:
|
|
@@ -1061,10 +1062,11 @@ def tab_openalex_search():
|
|
|
1061
1062
|
st.markdown(f"**Citations:** {citation_count}")
|
|
1062
1063
|
|
|
1063
1064
|
# Venue
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
)
|
|
1067
|
-
|
|
1065
|
+
pl = work.get("primary_location") or {}
|
|
1066
|
+
src = pl.get("source") or {}
|
|
1067
|
+
venue_name = src.get("display_name")
|
|
1068
|
+
if venue_name:
|
|
1069
|
+
st.markdown(f"**Venue:** {venue_name}")
|
|
1068
1070
|
|
|
1069
1071
|
# Abstract preview
|
|
1070
1072
|
if work.get("abstract_inverted_index"):
|
|
@@ -15,6 +15,11 @@ entry_type_by_crossref_type = {
|
|
|
15
15
|
"book-part": "inbook",
|
|
16
16
|
"other": "misc",
|
|
17
17
|
"paratext": "misc", # new line added
|
|
18
|
+
"erratum": "misc", # new line added
|
|
19
|
+
"letter": "article", # new line added
|
|
20
|
+
"editorial": "article", # new line added
|
|
21
|
+
"libguides": "misc", # new line added
|
|
22
|
+
"supplementary-materials": "misc", # new line added
|
|
18
23
|
"book": "book",
|
|
19
24
|
"journal-volume": "misc",
|
|
20
25
|
"book-set": "misc",
|
|
@@ -46,7 +51,8 @@ def dump_bibtex(work):
|
|
|
46
51
|
"Unable to generate a bibtex entry. This was probably an entry added by url or other non-doi source."
|
|
47
52
|
)
|
|
48
53
|
return None
|
|
49
|
-
|
|
54
|
+
# Try type_crossref first (for backwards compatibility), then fall back to type
|
|
55
|
+
wtype = work.get("type_crossref") or work.get("type")
|
|
50
56
|
if wtype is None:
|
|
51
57
|
print(f"Unable to generate a bibtex entry for {work.get('id')}")
|
|
52
58
|
return None
|
|
@@ -108,7 +114,7 @@ def _populate_entry(entry, work):
|
|
|
108
114
|
elif entry_type == "misc":
|
|
109
115
|
_set_if(entry, "author", _author(work))
|
|
110
116
|
_set_if(entry, "title", work.get("title"))
|
|
111
|
-
_set_if(entry, "howpublished", (work
|
|
117
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
112
118
|
_set_if(entry, "year", _year(work))
|
|
113
119
|
elif entry_type == "phdthesis":
|
|
114
120
|
_set_if(entry, "author", _author(work))
|
|
@@ -118,7 +124,7 @@ def _populate_entry(entry, work):
|
|
|
118
124
|
elif entry_type == "proceedings":
|
|
119
125
|
_set_if(entry, "editor", _author(work))
|
|
120
126
|
_set_if(entry, "title", work.get("title"))
|
|
121
|
-
_set_if(entry, "series",
|
|
127
|
+
_set_if(entry, "series", _primary_location_display_name(work))
|
|
122
128
|
_set_if(entry, "volume", _volume(work))
|
|
123
129
|
_set_if(entry, "publisher", _publisher(work))
|
|
124
130
|
_set_if(entry, "year", _year(work))
|
|
@@ -126,7 +132,7 @@ def _populate_entry(entry, work):
|
|
|
126
132
|
_set_if(entry, "author", _author(work))
|
|
127
133
|
_set_if(entry, "title", work.get("title"))
|
|
128
134
|
_set_if(entry, "year", _year(work))
|
|
129
|
-
_set_if(entry, "howpublished", (work
|
|
135
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
130
136
|
|
|
131
137
|
|
|
132
138
|
def _set_if(entry, key, value):
|
|
@@ -139,7 +145,7 @@ def _year(work):
|
|
|
139
145
|
|
|
140
146
|
|
|
141
147
|
def _book_title(work):
|
|
142
|
-
return (work
|
|
148
|
+
return _primary_location_display_name(work)
|
|
143
149
|
|
|
144
150
|
|
|
145
151
|
def _school(work):
|
|
@@ -168,11 +174,20 @@ def _journal_name(work):
|
|
|
168
174
|
|
|
169
175
|
|
|
170
176
|
def _publisher(work):
|
|
171
|
-
|
|
177
|
+
pl = work.get("primary_location") or {}
|
|
178
|
+
src = pl.get("source") or {}
|
|
179
|
+
return src.get("host_organization_name")
|
|
172
180
|
|
|
173
181
|
|
|
174
|
-
def
|
|
175
|
-
|
|
182
|
+
def _primary_location_display_name(work):
|
|
183
|
+
pl = work.get("primary_location") or {}
|
|
184
|
+
src = pl.get("source") or {}
|
|
185
|
+
return src.get("display_name")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _primary_location_url(work):
|
|
189
|
+
pl = work.get("primary_location") or {}
|
|
190
|
+
return pl.get("landing_page_url") or pl.get("pdf_url")
|
|
176
191
|
|
|
177
192
|
|
|
178
193
|
def _volume(work):
|
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
"""LitGPT using litellm."""
|
|
2
2
|
|
|
3
|
+
import importlib
|
|
4
|
+
import logging
|
|
3
5
|
import os
|
|
6
|
+
import pydoc
|
|
7
|
+
import re
|
|
4
8
|
import readline
|
|
5
9
|
import subprocess
|
|
6
|
-
|
|
7
10
|
import warnings
|
|
8
11
|
|
|
9
12
|
import numpy as np
|
|
13
|
+
from docling.document_converter import DocumentConverter
|
|
14
|
+
from docling.exceptions import ConversionError
|
|
10
15
|
from litellm import completion
|
|
11
16
|
from rich import print as richprint
|
|
12
17
|
from sentence_transformers import SentenceTransformer
|
|
13
18
|
|
|
14
|
-
import pydoc
|
|
15
|
-
import re
|
|
16
|
-
import importlib
|
|
17
|
-
from docling.document_converter import DocumentConverter
|
|
18
|
-
from docling.exceptions import ConversionError
|
|
19
|
-
import logging
|
|
20
|
-
|
|
21
19
|
# Try to import backoff (optional dependency in crawl extra)
|
|
22
20
|
try:
|
|
23
21
|
import backoff
|
|
@@ -220,6 +218,10 @@ def get_completion(model, messages, max_tokens=None):
|
|
|
220
218
|
|
|
221
219
|
except KeyboardInterrupt:
|
|
222
220
|
pass
|
|
221
|
+
except AttributeError as e:
|
|
222
|
+
# Suppress litellm TranscriptionCreateParams.__annotations__ errors
|
|
223
|
+
if "__annotations__" not in str(e):
|
|
224
|
+
raise
|
|
223
225
|
|
|
224
226
|
return output
|
|
225
227
|
|
|
@@ -16,21 +16,53 @@ def get_coa(orcid, email=None):
|
|
|
16
16
|
ORCID: str the author orcid to retrieve results for.
|
|
17
17
|
email: str optional email for OpenAlex API (for polite pool)
|
|
18
18
|
"""
|
|
19
|
+
print(f"Starting COA generation for ORCID: {orcid}")
|
|
19
20
|
|
|
20
|
-
|
|
21
|
+
# First, find the correct author profile (handle duplicate ORCID profiles)
|
|
22
|
+
orcid = orcid.replace("https://orcid.org/", "")
|
|
23
|
+
orcid_url = f"https://orcid.org/{orcid}"
|
|
21
24
|
|
|
22
|
-
|
|
25
|
+
print(f"Finding author profile for ORCID {orcid}...")
|
|
26
|
+
authors_url = "https://api.openalex.org/authors"
|
|
27
|
+
author_params = {"filter": f"orcid:{orcid_url}"}
|
|
28
|
+
if email:
|
|
29
|
+
author_params["mailto"] = email
|
|
23
30
|
|
|
31
|
+
author_resp = requests.get(authors_url, params=author_params)
|
|
32
|
+
author_resp.raise_for_status()
|
|
33
|
+
author_data = author_resp.json()
|
|
34
|
+
|
|
35
|
+
if not author_data["results"]:
|
|
36
|
+
raise ValueError(f"No author found for ORCID {orcid_url}")
|
|
37
|
+
|
|
38
|
+
# If multiple profiles exist with same ORCID, use the one with most works
|
|
39
|
+
if len(author_data["results"]) > 1:
|
|
40
|
+
print(
|
|
41
|
+
f" Warning: Found {len(author_data['results'])} author profiles with this ORCID"
|
|
42
|
+
)
|
|
43
|
+
for a in author_data["results"]:
|
|
44
|
+
print(
|
|
45
|
+
f" - {a['display_name']}: {a['works_count']} works (ID: {a['id']})"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Select author with most works
|
|
49
|
+
author = max(author_data["results"], key=lambda x: x.get("works_count", 0))
|
|
50
|
+
author_id = author["id"].replace("https://openalex.org/", "")
|
|
51
|
+
print(
|
|
52
|
+
f" Using profile: {author['display_name']} ({author['works_count']} total works)"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
url = "https://api.openalex.org/works"
|
|
56
|
+
next_cursor = "*"
|
|
24
57
|
pubs = []
|
|
25
58
|
|
|
26
59
|
current_year = datetime.datetime.now().year
|
|
27
60
|
four_years_ago = current_year - 4
|
|
28
|
-
|
|
29
|
-
orcid = orcid.replace("https://orcid.org/", "")
|
|
61
|
+
print(f"Fetching publications from {four_years_ago} to {current_year}...")
|
|
30
62
|
|
|
31
63
|
while next_cursor:
|
|
32
64
|
_filter = (
|
|
33
|
-
f"author.
|
|
65
|
+
f"author.id:https://openalex.org/{author_id}"
|
|
34
66
|
f",publication_year:>{four_years_ago - 1}"
|
|
35
67
|
)
|
|
36
68
|
|
|
@@ -56,7 +88,17 @@ def get_coa(orcid, email=None):
|
|
|
56
88
|
pubs += data["results"]
|
|
57
89
|
next_cursor = data["meta"].get("next_cursor", None)
|
|
58
90
|
|
|
91
|
+
print(f"Found {len(pubs)} publications")
|
|
92
|
+
|
|
93
|
+
# Show publication years distribution
|
|
94
|
+
year_counts = {}
|
|
95
|
+
for pub in pubs:
|
|
96
|
+
year = pub.get("publication_year", "Unknown")
|
|
97
|
+
year_counts[year] = year_counts.get(year, 0) + 1
|
|
98
|
+
print("Publications by year:", dict(sorted(year_counts.items(), reverse=True)))
|
|
99
|
+
|
|
59
100
|
# We get all the authors from all the papers first.
|
|
101
|
+
print("Extracting authors from publications...")
|
|
60
102
|
authors = []
|
|
61
103
|
|
|
62
104
|
for pub in pubs:
|
|
@@ -67,20 +109,31 @@ def get_coa(orcid, email=None):
|
|
|
67
109
|
|
|
68
110
|
aus = pub["authorships"]
|
|
69
111
|
for au in aus:
|
|
70
|
-
|
|
112
|
+
# Skip authors with no display name or no OpenAlex ID
|
|
113
|
+
display_name = au["author"].get("display_name")
|
|
114
|
+
if not display_name or not au["author"].get("id"):
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
hn = HumanName(display_name)
|
|
71
118
|
name = f"{hn.last}, {hn.first} {hn.middle or ''}"
|
|
72
119
|
|
|
73
120
|
authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
|
|
74
121
|
|
|
75
122
|
# sort authors alphabetically, then by year descending
|
|
76
123
|
authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
|
|
124
|
+
print(f"Found {len(authors)} total author entries (including duplicates)")
|
|
77
125
|
|
|
78
|
-
# Now, get all the affiliations
|
|
79
|
-
#
|
|
80
|
-
#
|
|
126
|
+
# Now, get all the affiliations using a two-strategy approach:
|
|
127
|
+
# 1. Use last_known_institutions (from most recent publication)
|
|
128
|
+
# 2. Fall back to affiliation with most recent year in affiliations array
|
|
81
129
|
oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
|
|
130
|
+
print(f"Found {len(oaids)} unique authors")
|
|
131
|
+
print("Fetching affiliation information from OpenAlex...")
|
|
82
132
|
affiliations = {}
|
|
133
|
+
batch_num = 0
|
|
83
134
|
for batch in batched(oaids, 50):
|
|
135
|
+
batch_num += 1
|
|
136
|
+
print(f" Fetching batch {batch_num} of authors...")
|
|
84
137
|
url = f"https://api.openalex.org/authors?filter=id:{'|'.join(batch)}"
|
|
85
138
|
|
|
86
139
|
params = {"per-page": 50}
|
|
@@ -92,12 +145,24 @@ def get_coa(orcid, email=None):
|
|
|
92
145
|
d = requests.get(url, params=params)
|
|
93
146
|
|
|
94
147
|
for au in d.json()["results"]:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
148
|
+
affiliation_name = ""
|
|
149
|
+
|
|
150
|
+
# Strategy 1: Use last_known_institutions (most recent publication)
|
|
151
|
+
last_known = au.get("last_known_institutions", [])
|
|
152
|
+
if last_known and len(last_known) > 0:
|
|
153
|
+
affiliation_name = last_known[0].get("display_name", "")
|
|
154
|
+
|
|
155
|
+
# Strategy 2: If no last_known_institutions, use affiliation with most recent year
|
|
156
|
+
if not affiliation_name:
|
|
157
|
+
affils = au.get("affiliations", [])
|
|
158
|
+
if len(affils) > 0:
|
|
159
|
+
# Find affiliation with most recent year
|
|
160
|
+
best_affil = max(affils, key=lambda a: max(a.get("years", [0])))
|
|
161
|
+
affiliation_name = best_affil["institution"]["display_name"]
|
|
100
162
|
|
|
163
|
+
affiliations[au["id"]] = affiliation_name
|
|
164
|
+
|
|
165
|
+
print("Building unique author list...")
|
|
101
166
|
uniq = {}
|
|
102
167
|
uniq_authors = [] # by openalex id
|
|
103
168
|
all_authors = []
|
|
@@ -109,6 +174,8 @@ def get_coa(orcid, email=None):
|
|
|
109
174
|
uniq_authors += [["A:", name, affil, "", last_active]]
|
|
110
175
|
all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
|
|
111
176
|
|
|
177
|
+
print(f"Creating Excel file with {len(uniq_authors)} unique authors...")
|
|
178
|
+
|
|
112
179
|
# unique authors
|
|
113
180
|
df = pd.DataFrame(
|
|
114
181
|
uniq_authors,
|
|
@@ -176,7 +243,9 @@ def get_coa(orcid, email=None):
|
|
|
176
243
|
b64 = base64.b64encode(excel_bytes).decode("utf-8")
|
|
177
244
|
uri = f'<pre>{filename}</pre><br><a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download COA</a>'
|
|
178
245
|
display(HTML(uri))
|
|
246
|
+
print("✓ COA generation complete!")
|
|
179
247
|
return None
|
|
180
248
|
else:
|
|
181
249
|
# Return path for MCP server / CLI
|
|
250
|
+
print(f"✓ COA saved to: {downloads_path}")
|
|
182
251
|
return str(downloads_path)
|