litdb 2.2.1__tar.gz → 2.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {litdb-2.2.1/src/litdb.egg-info → litdb-2.2.3}/PKG-INFO +98 -17
  2. {litdb-2.2.1 → litdb-2.2.3}/README.org +92 -14
  3. {litdb-2.2.1 → litdb-2.2.3}/pyproject.toml +6 -3
  4. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/app.py +10 -8
  5. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/bibtex.py +23 -8
  6. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/chat.py +10 -8
  7. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/coa.py +83 -14
  8. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/data_processing.py +37 -4
  9. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/install.py +8 -2
  10. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/manage.py +56 -1
  11. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/utilities.py +19 -6
  12. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/db.py +177 -28
  13. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/doe_collaborators.py +15 -11
  14. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/mcp_server.py +59 -25
  15. {litdb-2.2.1 → litdb-2.2.3/src/litdb.egg-info}/PKG-INFO +98 -17
  16. {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/requires.txt +5 -2
  17. {litdb-2.2.1 → litdb-2.2.3}/LICENSE +0 -0
  18. {litdb-2.2.1 → litdb-2.2.3}/setup.cfg +0 -0
  19. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/__init__.py +0 -0
  20. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/audio.py +0 -0
  21. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/cli.py +0 -0
  22. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/__init__.py +0 -0
  23. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/export.py +0 -0
  24. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/filters.py +0 -0
  25. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/openalex_commands.py +0 -0
  26. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/research_commands.py +0 -0
  27. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/review.py +0 -0
  28. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/search.py +0 -0
  29. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/commands/tags.py +0 -0
  30. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/crawl.py +0 -0
  31. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/extract.py +0 -0
  32. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/gpt.py +0 -0
  33. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/images.py +0 -0
  34. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/lab.py +0 -0
  35. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/lsearch.py +0 -0
  36. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/openalex.py +0 -0
  37. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/pdf.py +0 -0
  38. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/research.py +0 -0
  39. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/server.py +0 -0
  40. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/summary.py +0 -0
  41. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/utils.py +0 -0
  42. {litdb-2.2.1 → litdb-2.2.3}/src/litdb/youtube.py +0 -0
  43. {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/SOURCES.txt +0 -0
  44. {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/dependency_links.txt +0 -0
  45. {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/entry_points.txt +0 -0
  46. {litdb-2.2.1 → litdb-2.2.3}/src/litdb.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: litdb
3
- Version: 2.2.1
3
+ Version: 2.2.3
4
4
  Summary: A literature database tool with GPT integration.
5
5
  Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
6
6
  Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
@@ -11,7 +11,7 @@ License-File: LICENSE
11
11
  Requires-Dist: toml
12
12
  Requires-Dist: tomlkit
13
13
  Requires-Dist: requests
14
- Requires-Dist: pymupdf4llm
14
+ Requires-Dist: pymupdf4llm<2
15
15
  Requires-Dist: docling
16
16
  Requires-Dist: langchain-core
17
17
  Requires-Dist: langchain-text-splitters
@@ -33,7 +33,7 @@ Requires-Dist: dateparser
33
33
  Requires-Dist: jinja2
34
34
  Requires-Dist: pytesseract
35
35
  Requires-Dist: pillow
36
- Requires-Dist: bibtexparser
36
+ Requires-Dist: bibtexparser<2
37
37
  Requires-Dist: pandas
38
38
  Requires-Dist: more-itertools
39
39
  Requires-Dist: nameparser
@@ -41,6 +41,9 @@ Requires-Dist: tabulate
41
41
  Requires-Dist: IPython
42
42
  Requires-Dist: gmft
43
43
  Requires-Dist: ruff>=0.13.3
44
+ Requires-Dist: torch
45
+ Requires-Dist: torchvision
46
+ Requires-Dist: fastmcp>=2.13.3
44
47
  Provides-Extra: test
45
48
  Requires-Dist: pytest>=8.0.0; extra == "test"
46
49
  Requires-Dist: pytest-cov>=4.1.0; extra == "test"
@@ -109,6 +112,9 @@ Dynamic: license-file
109
112
  <img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="Build Status">
110
113
  </a>
111
114
  <img src="https://img.shields.io/badge/tests-133_passed,_66_skipped-brightgreen" alt="Tests">
115
+ <a href="https://doi.org/10.5281/zenodo.23514948">
116
+ <img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23514948.svg" alt="DOI">
117
+ </a>
112
118
  #+END_HTML
113
119
 
114
120
  * litdb concept
@@ -155,7 +161,13 @@ source .venv/bin/activate
155
161
  uv pip install litdb
156
162
  #+END_SRC
157
163
 
158
- You have to activate that virtual env to use litdb, which may be annoying.
164
+ You have to activate that virtual env to use litdb, which may be annoying. An alternative is to install litdb as a uv tool, which puts ~litdb~ and ~litdb_mcp~ on your PATH in their own environment:
165
+
166
+ #+BEGIN_SRC sh
167
+ uv tool install litdb
168
+ #+END_SRC
169
+
170
+ This is also the easiest way to make the MCP server available to AI agents. See [[*Using litdb with AI agents (skill and MCP server)]].
159
171
 
160
172
  ** Recent additions
161
173
 
@@ -574,7 +586,7 @@ I haven't found the transcription to be that good on technical scientific terms.
574
586
 
575
587
  Note that you need to install these libraries for this feature to work:
576
588
 
577
- pyaudio, playsound, SpeechRecognition
589
+ pyaudio, SpeechRecognition
578
590
 
579
591
  These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
580
592
 
@@ -808,16 +820,93 @@ litdb schema paper.pdf '{"title": ["str", None], "year": ["int", 2024]}'
808
820
 
809
821
  The schema command first converts the document to markdown using Docling, then uses an LLM with structured output to extract the requested fields. The LLM model used is configured in the ~[llm]~ section of your ~litdb.toml~ file.
810
822
 
811
- ** Claude Desktop + MCP
823
+ ** Using litdb with AI agents (skill and MCP server)
824
+
825
+ litdb comes with two things for AI coding agents such as Claude Code, VS Code (GitHub Copilot), Cursor, Codex, OpenCode and Gemini CLI:
826
+
827
+ - An *agent skill* ([[./skills/litdb/SKILL.md][skills/litdb/SKILL.md]]) that teaches the agent how to use the litdb CLI: adding papers, searching, filters, tags, exporting BibTeX, and so on.
828
+ - An *MCP server* (~litdb_mcp~) that gives the agent direct, read-only tools for your database: ~vsearch~, ~fulltext_search~, ~find_similar~, ~openalex~, ~get_source_details~, ~get_citation~, ~get_bibtex~, ~list_tags~, ~get_tagged_articles~ and ~generate_nsf_coa~.
829
+
830
+ The easiest way to install both is [[https://github.com/jkitchin/knowhow][knowhow]], which needs nothing installed besides [[https://github.com/astral-sh/uv][uv]]. The skill follows the [[https://agentskills.io/specification][Agent Skills specification]], so it also works with ~npx skills~.
831
+
832
+ *** Install the skill
833
+
834
+ #+BEGIN_SRC sh
835
+ # See what litdb offers
836
+ uvx knowhow add jkitchin/litdb --list
837
+
838
+ # Install the skill in the current project
839
+ uvx knowhow add jkitchin/litdb --skill litdb
840
+
841
+ # or for your user, in every project
842
+ uvx knowhow add jkitchin/litdb --skill litdb -g
843
+ #+END_SRC
844
+
845
+ The skill is copied to ~.agents/skills/litdb~ (read by VS Code, Codex, OpenCode, Gemini CLI, Cursor and Copilot) and linked into ~.claude/skills/litdb~ for Claude Code. Then ask your agent things like "add the references of 10.1021/jp047349j to my litdb" or "make a bibtex file of everything tagged thesis".
846
+
847
+ Keep it up to date with ~uvx knowhow check~ and ~uvx knowhow upgrade~, and remove it with ~uvx knowhow remove litdb~.
848
+
849
+ If you work from a clone of this repository, ~litdb install claude-skill~ also copies the skill to ~~/.claude/skills/litdb~.
850
+
851
+ *** Install the MCP server
852
+
853
+ The MCP server runs the ~litdb_mcp~ command, so first make sure it is on your PATH (~uv tool install litdb~ does this). The server is defined in [[./mcp/litdb.json][mcp/litdb.json]], and knowhow translates it into each agent's config format:
854
+
855
+ #+BEGIN_SRC sh
856
+ # Claude Code, in this project (writes .mcp.json)
857
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code
858
+
859
+ # Claude Code for all your projects
860
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -g
861
+
862
+ # Several agents at once
863
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -a vscode -a cursor -a codex
864
+ #+END_SRC
865
+
866
+ knowhow shows you the server config and asks for confirmation before installing it (add ~-y~ to skip the prompt).
867
+
868
+ The server needs to know which database to use. It looks for:
869
+
870
+ 1. the ~litdb~ environment variable, the path to a ~litdb.libsql~ file;
871
+ 2. a ~litdb.toml~ in the agent's working directory or one of its parents;
872
+ 3. the ~LITDB_ROOT~ environment variable, the directory that contains ~litdb.toml~.
873
+
874
+ The installed config passes ~LITDB_ROOT~ through from your environment, so set it in your shell profile before starting the agent:
812
875
 
813
876
  #+BEGIN_SRC sh
814
- litdb_mcp install /path/to/litdb.libsql
877
+ export LITDB_ROOT=~/litdb # the directory with litdb.toml and litdb.libsql
815
878
  #+END_SRC
816
879
 
817
- Then, restart Claude Desktop.
880
+ To check it works in Claude Code, run ~/mcp~ and look for ~litdb~, or ask "what is in my litdb about alloy segregation?". Remove the server with ~uvx knowhow remove --mcp litdb~.
881
+
882
+ If you prefer to configure an agent by hand, this is the config (Claude Code ~.mcp.json~ format):
818
883
 
819
- Now in the Claude Desktop app you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation."
884
+ #+BEGIN_SRC json
885
+ {
886
+ "mcpServers": {
887
+ "litdb": {
888
+ "command": "litdb_mcp",
889
+ "env": {"LITDB_ROOT": "/path/to/your/litdb"}
890
+ }
891
+ }
892
+ }
893
+ #+END_SRC
820
894
 
895
+ or with the Claude Code CLI:
896
+
897
+ #+BEGIN_SRC sh
898
+ claude mcp add litdb --scope user -e LITDB_ROOT=/path/to/your/litdb -- litdb_mcp
899
+ #+END_SRC
900
+
901
+ *** Claude Desktop
902
+
903
+ Claude Desktop is not supported by knowhow. Instead, run this from your litdb directory (the one with ~litdb.toml~), or pass ~--db /path/to/litdb.libsql~:
904
+
905
+ #+BEGIN_SRC sh
906
+ litdb install mcp-server
907
+ #+END_SRC
908
+
909
+ Then restart Claude Desktop. Now you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation." Uninstall it with ~litdb install uninstall-mcp~.
821
910
 
822
911
  * Adding local files
823
912
 
@@ -1189,11 +1278,3 @@ Many of these projects require you to make an account. There are freemium levels
1189
1278
  - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
1190
1279
 
1191
1280
  With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
1192
-
1193
- * Tasks
1194
-
1195
- ** DONE create a Claude skill for litdb
1196
- CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
1197
-
1198
- ** DONE Make a nicer streamlit app for the front end.
1199
- CLOSED: [2025-10-28 Tue 10:40]
@@ -7,6 +7,9 @@
7
7
  <img src="https://github.com/jkitchin/litdb/actions/workflows/build.yml/badge.svg" alt="Build Status">
8
8
  </a>
9
9
  <img src="https://img.shields.io/badge/tests-133_passed,_66_skipped-brightgreen" alt="Tests">
10
+ <a href="https://doi.org/10.5281/zenodo.23514948">
11
+ <img src="https://zenodo.org/badge/DOI/10.5281/zenodo.23514948.svg" alt="DOI">
12
+ </a>
10
13
  #+END_HTML
11
14
 
12
15
  * litdb concept
@@ -53,7 +56,13 @@ source .venv/bin/activate
53
56
  uv pip install litdb
54
57
  #+END_SRC
55
58
 
56
- You have to activate that virtual env to use litdb, which may be annoying.
59
+ You have to activate that virtual env to use litdb, which may be annoying. An alternative is to install litdb as a uv tool, which puts ~litdb~ and ~litdb_mcp~ on your PATH in their own environment:
60
+
61
+ #+BEGIN_SRC sh
62
+ uv tool install litdb
63
+ #+END_SRC
64
+
65
+ This is also the easiest way to make the MCP server available to AI agents. See [[*Using litdb with AI agents (skill and MCP server)]].
57
66
 
58
67
  ** Recent additions
59
68
 
@@ -472,7 +481,7 @@ I haven't found the transcription to be that good on technical scientific terms.
472
481
 
473
482
  Note that you need to install these libraries for this feature to work:
474
483
 
475
- pyaudio, playsound, SpeechRecognition
484
+ pyaudio, SpeechRecognition
476
485
 
477
486
  These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
478
487
 
@@ -706,16 +715,93 @@ litdb schema paper.pdf '{"title": ["str", None], "year": ["int", 2024]}'
706
715
 
707
716
  The schema command first converts the document to markdown using Docling, then uses an LLM with structured output to extract the requested fields. The LLM model used is configured in the ~[llm]~ section of your ~litdb.toml~ file.
708
717
 
709
- ** Claude Desktop + MCP
718
+ ** Using litdb with AI agents (skill and MCP server)
719
+
720
+ litdb comes with two things for AI coding agents such as Claude Code, VS Code (GitHub Copilot), Cursor, Codex, OpenCode and Gemini CLI:
721
+
722
+ - An *agent skill* ([[./skills/litdb/SKILL.md][skills/litdb/SKILL.md]]) that teaches the agent how to use the litdb CLI: adding papers, searching, filters, tags, exporting BibTeX, and so on.
723
+ - An *MCP server* (~litdb_mcp~) that gives the agent direct, read-only tools for your database: ~vsearch~, ~fulltext_search~, ~find_similar~, ~openalex~, ~get_source_details~, ~get_citation~, ~get_bibtex~, ~list_tags~, ~get_tagged_articles~ and ~generate_nsf_coa~.
724
+
725
+ The easiest way to install both is [[https://github.com/jkitchin/knowhow][knowhow]], which needs nothing installed besides [[https://github.com/astral-sh/uv][uv]]. The skill follows the [[https://agentskills.io/specification][Agent Skills specification]], so it also works with ~npx skills~.
726
+
727
+ *** Install the skill
728
+
729
+ #+BEGIN_SRC sh
730
+ # See what litdb offers
731
+ uvx knowhow add jkitchin/litdb --list
732
+
733
+ # Install the skill in the current project
734
+ uvx knowhow add jkitchin/litdb --skill litdb
735
+
736
+ # or for your user, in every project
737
+ uvx knowhow add jkitchin/litdb --skill litdb -g
738
+ #+END_SRC
739
+
740
+ The skill is copied to ~.agents/skills/litdb~ (read by VS Code, Codex, OpenCode, Gemini CLI, Cursor and Copilot) and linked into ~.claude/skills/litdb~ for Claude Code. Then ask your agent things like "add the references of 10.1021/jp047349j to my litdb" or "make a bibtex file of everything tagged thesis".
741
+
742
+ Keep it up to date with ~uvx knowhow check~ and ~uvx knowhow upgrade~, and remove it with ~uvx knowhow remove litdb~.
743
+
744
+ If you work from a clone of this repository, ~litdb install claude-skill~ also copies the skill to ~~/.claude/skills/litdb~.
745
+
746
+ *** Install the MCP server
747
+
748
+ The MCP server runs the ~litdb_mcp~ command, so first make sure it is on your PATH (~uv tool install litdb~ does this). The server is defined in [[./mcp/litdb.json][mcp/litdb.json]], and knowhow translates it into each agent's config format:
749
+
750
+ #+BEGIN_SRC sh
751
+ # Claude Code, in this project (writes .mcp.json)
752
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code
753
+
754
+ # Claude Code for all your projects
755
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -g
756
+
757
+ # Several agents at once
758
+ uvx knowhow add jkitchin/litdb --mcp litdb -a claude-code -a vscode -a cursor -a codex
759
+ #+END_SRC
760
+
761
+ knowhow shows you the server config and asks for confirmation before installing it (add ~-y~ to skip the prompt).
762
+
763
+ The server needs to know which database to use. It looks for:
764
+
765
+ 1. the ~litdb~ environment variable, the path to a ~litdb.libsql~ file;
766
+ 2. a ~litdb.toml~ in the agent's working directory or one of its parents;
767
+ 3. the ~LITDB_ROOT~ environment variable, the directory that contains ~litdb.toml~.
768
+
769
+ The installed config passes ~LITDB_ROOT~ through from your environment, so set it in your shell profile before starting the agent:
710
770
 
711
771
  #+BEGIN_SRC sh
712
- litdb_mcp install /path/to/litdb.libsql
772
+ export LITDB_ROOT=~/litdb # the directory with litdb.toml and litdb.libsql
713
773
  #+END_SRC
714
774
 
715
- Then, restart Claude Desktop.
775
+ To check it works in Claude Code, run ~/mcp~ and look for ~litdb~, or ask "what is in my litdb about alloy segregation?". Remove the server with ~uvx knowhow remove --mcp litdb~.
776
+
777
+ If you prefer to configure an agent by hand, this is the config (Claude Code ~.mcp.json~ format):
716
778
 
717
- Now in the Claude Desktop app you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation."
779
+ #+BEGIN_SRC json
780
+ {
781
+ "mcpServers": {
782
+ "litdb": {
783
+ "command": "litdb_mcp",
784
+ "env": {"LITDB_ROOT": "/path/to/your/litdb"}
785
+ }
786
+ }
787
+ }
788
+ #+END_SRC
718
789
 
790
+ or with the Claude Code CLI:
791
+
792
+ #+BEGIN_SRC sh
793
+ claude mcp add litdb --scope user -e LITDB_ROOT=/path/to/your/litdb -- litdb_mcp
794
+ #+END_SRC
795
+
796
+ *** Claude Desktop
797
+
798
+ Claude Desktop is not supported by knowhow. Instead, run this from your litdb directory (the one with ~litdb.toml~), or pass ~--db /path/to/litdb.libsql~:
799
+
800
+ #+BEGIN_SRC sh
801
+ litdb install mcp-server
802
+ #+END_SRC
803
+
804
+ Then restart Claude Desktop. Now you can use prompts like "Search my litdb for references on alloy segregation" or "Search OpenAlex for articles on alloy segregation." Uninstall it with ~litdb install uninstall-mcp~.
719
805
 
720
806
  * Adding local files
721
807
 
@@ -1087,11 +1173,3 @@ Many of these projects require you to make an account. There are freemium levels
1087
1173
  - Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
1088
1174
 
1089
1175
  With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
1090
-
1091
- * Tasks
1092
-
1093
- ** DONE create a Claude skill for litdb
1094
- CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
1095
-
1096
- ** DONE Make a nicer streamlit app for the front end.
1097
- CLOSED: [2025-10-28 Tue 10:40]
@@ -7,7 +7,7 @@ where = ["src"]
7
7
 
8
8
  [project]
9
9
  name = "litdb"
10
- version = "2.2.1"
10
+ version = "2.2.3"
11
11
  authors = [
12
12
  {name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
13
13
  ]
@@ -22,7 +22,7 @@ dependencies = [
22
22
  "toml",
23
23
  "tomlkit",
24
24
  "requests",
25
- "pymupdf4llm",
25
+ "pymupdf4llm<2",
26
26
  "docling",
27
27
  "langchain-core",
28
28
  "langchain-text-splitters",
@@ -44,7 +44,7 @@ dependencies = [
44
44
  "jinja2",
45
45
  "pytesseract",
46
46
  "pillow",
47
- "bibtexparser",
47
+ "bibtexparser<2",
48
48
  # Data processing and COA
49
49
  "pandas",
50
50
  "more-itertools",
@@ -55,6 +55,9 @@ dependencies = [
55
55
  "gmft",
56
56
  # Development tools
57
57
  "ruff>=0.13.3",
58
+ "torch",
59
+ "torchvision",
60
+ "fastmcp>=2.13.3",
58
61
  ]
59
62
 
60
63
  [project.scripts]
@@ -205,10 +205,11 @@ def format_citation(extra):
205
205
  parts.append(data["title"])
206
206
 
207
207
  # Journal/venue
208
- if "host_venue" in data and data["host_venue"]:
209
- venue = data["host_venue"].get("display_name", "")
210
- if venue:
211
- parts.append(f"*{venue}*")
208
+ pl = data.get("primary_location") or {}
209
+ src = pl.get("source") or {}
210
+ venue = src.get("display_name", "")
211
+ if venue:
212
+ parts.append(f"*{venue}*")
212
213
 
213
214
  return ". ".join(parts) if parts else None
214
215
  except Exception:
@@ -1061,10 +1062,11 @@ def tab_openalex_search():
1061
1062
  st.markdown(f"**Citations:** {citation_count}")
1062
1063
 
1063
1064
  # Venue
1064
- if work.get("host_venue") and work["host_venue"].get(
1065
- "display_name"
1066
- ):
1067
- st.markdown(f"**Venue:** {work['host_venue']['display_name']}")
1065
+ pl = work.get("primary_location") or {}
1066
+ src = pl.get("source") or {}
1067
+ venue_name = src.get("display_name")
1068
+ if venue_name:
1069
+ st.markdown(f"**Venue:** {venue_name}")
1068
1070
 
1069
1071
  # Abstract preview
1070
1072
  if work.get("abstract_inverted_index"):
@@ -15,6 +15,11 @@ entry_type_by_crossref_type = {
15
15
  "book-part": "inbook",
16
16
  "other": "misc",
17
17
  "paratext": "misc", # new line added
18
+ "erratum": "misc", # new line added
19
+ "letter": "article", # new line added
20
+ "editorial": "article", # new line added
21
+ "libguides": "misc", # new line added
22
+ "supplementary-materials": "misc", # new line added
18
23
  "book": "book",
19
24
  "journal-volume": "misc",
20
25
  "book-set": "misc",
@@ -46,7 +51,8 @@ def dump_bibtex(work):
46
51
  "Unable to generate a bibtex entry. This was probably an entry added by url or other non-doi source."
47
52
  )
48
53
  return None
49
- wtype = work.get("type_crossref")
54
+ # Try type_crossref first (for backwards compatibility), then fall back to type
55
+ wtype = work.get("type_crossref") or work.get("type")
50
56
  if wtype is None:
51
57
  print(f"Unable to generate a bibtex entry for {work.get('id')}")
52
58
  return None
@@ -108,7 +114,7 @@ def _populate_entry(entry, work):
108
114
  elif entry_type == "misc":
109
115
  _set_if(entry, "author", _author(work))
110
116
  _set_if(entry, "title", work.get("title"))
111
- _set_if(entry, "howpublished", (work.get("host_venue") or {}).get("url"))
117
+ _set_if(entry, "howpublished", _primary_location_url(work))
112
118
  _set_if(entry, "year", _year(work))
113
119
  elif entry_type == "phdthesis":
114
120
  _set_if(entry, "author", _author(work))
@@ -118,7 +124,7 @@ def _populate_entry(entry, work):
118
124
  elif entry_type == "proceedings":
119
125
  _set_if(entry, "editor", _author(work))
120
126
  _set_if(entry, "title", work.get("title"))
121
- _set_if(entry, "series", _host_venue_display_name(work))
127
+ _set_if(entry, "series", _primary_location_display_name(work))
122
128
  _set_if(entry, "volume", _volume(work))
123
129
  _set_if(entry, "publisher", _publisher(work))
124
130
  _set_if(entry, "year", _year(work))
@@ -126,7 +132,7 @@ def _populate_entry(entry, work):
126
132
  _set_if(entry, "author", _author(work))
127
133
  _set_if(entry, "title", work.get("title"))
128
134
  _set_if(entry, "year", _year(work))
129
- _set_if(entry, "howpublished", (work.get("host_venue") or {}).get("url"))
135
+ _set_if(entry, "howpublished", _primary_location_url(work))
130
136
 
131
137
 
132
138
  def _set_if(entry, key, value):
@@ -139,7 +145,7 @@ def _year(work):
139
145
 
140
146
 
141
147
  def _book_title(work):
142
- return (work.get("host_venue") or {}).get("display_name")
148
+ return _primary_location_display_name(work)
143
149
 
144
150
 
145
151
  def _school(work):
@@ -168,11 +174,20 @@ def _journal_name(work):
168
174
 
169
175
 
170
176
  def _publisher(work):
171
- return (work.get("host_venue") or {}).get("publisher")
177
+ pl = work.get("primary_location") or {}
178
+ src = pl.get("source") or {}
179
+ return src.get("host_organization_name")
172
180
 
173
181
 
174
- def _host_venue_display_name(work):
175
- return (work.get("host_venue") or {}).get("display_name")
182
+ def _primary_location_display_name(work):
183
+ pl = work.get("primary_location") or {}
184
+ src = pl.get("source") or {}
185
+ return src.get("display_name")
186
+
187
+
188
+ def _primary_location_url(work):
189
+ pl = work.get("primary_location") or {}
190
+ return pl.get("landing_page_url") or pl.get("pdf_url")
176
191
 
177
192
 
178
193
  def _volume(work):
@@ -1,23 +1,21 @@
1
1
  """LitGPT using litellm."""
2
2
 
3
+ import importlib
4
+ import logging
3
5
  import os
6
+ import pydoc
7
+ import re
4
8
  import readline
5
9
  import subprocess
6
-
7
10
  import warnings
8
11
 
9
12
  import numpy as np
13
+ from docling.document_converter import DocumentConverter
14
+ from docling.exceptions import ConversionError
10
15
  from litellm import completion
11
16
  from rich import print as richprint
12
17
  from sentence_transformers import SentenceTransformer
13
18
 
14
- import pydoc
15
- import re
16
- import importlib
17
- from docling.document_converter import DocumentConverter
18
- from docling.exceptions import ConversionError
19
- import logging
20
-
21
19
  # Try to import backoff (optional dependency in crawl extra)
22
20
  try:
23
21
  import backoff
@@ -220,6 +218,10 @@ def get_completion(model, messages, max_tokens=None):
220
218
 
221
219
  except KeyboardInterrupt:
222
220
  pass
221
+ except AttributeError as e:
222
+ # Suppress litellm TranscriptionCreateParams.__annotations__ errors
223
+ if "__annotations__" not in str(e):
224
+ raise
223
225
 
224
226
  return output
225
227
 
@@ -16,21 +16,53 @@ def get_coa(orcid, email=None):
16
16
  ORCID: str the author orcid to retrieve results for.
17
17
  email: str optional email for OpenAlex API (for polite pool)
18
18
  """
19
+ print(f"Starting COA generation for ORCID: {orcid}")
19
20
 
20
- url = "https://api.openalex.org/works"
21
+ # First, find the correct author profile (handle duplicate ORCID profiles)
22
+ orcid = orcid.replace("https://orcid.org/", "")
23
+ orcid_url = f"https://orcid.org/{orcid}"
21
24
 
22
- next_cursor = "*"
25
+ print(f"Finding author profile for ORCID {orcid}...")
26
+ authors_url = "https://api.openalex.org/authors"
27
+ author_params = {"filter": f"orcid:{orcid_url}"}
28
+ if email:
29
+ author_params["mailto"] = email
23
30
 
31
+ author_resp = requests.get(authors_url, params=author_params)
32
+ author_resp.raise_for_status()
33
+ author_data = author_resp.json()
34
+
35
+ if not author_data["results"]:
36
+ raise ValueError(f"No author found for ORCID {orcid_url}")
37
+
38
+ # If multiple profiles exist with same ORCID, use the one with most works
39
+ if len(author_data["results"]) > 1:
40
+ print(
41
+ f" Warning: Found {len(author_data['results'])} author profiles with this ORCID"
42
+ )
43
+ for a in author_data["results"]:
44
+ print(
45
+ f" - {a['display_name']}: {a['works_count']} works (ID: {a['id']})"
46
+ )
47
+
48
+ # Select author with most works
49
+ author = max(author_data["results"], key=lambda x: x.get("works_count", 0))
50
+ author_id = author["id"].replace("https://openalex.org/", "")
51
+ print(
52
+ f" Using profile: {author['display_name']} ({author['works_count']} total works)"
53
+ )
54
+
55
+ url = "https://api.openalex.org/works"
56
+ next_cursor = "*"
24
57
  pubs = []
25
58
 
26
59
  current_year = datetime.datetime.now().year
27
60
  four_years_ago = current_year - 4
28
-
29
- orcid = orcid.replace("https://orcid.org/", "")
61
+ print(f"Fetching publications from {four_years_ago} to {current_year}...")
30
62
 
31
63
  while next_cursor:
32
64
  _filter = (
33
- f"author.orcid:https://orcid.org/{orcid}"
65
+ f"author.id:https://openalex.org/{author_id}"
34
66
  f",publication_year:>{four_years_ago - 1}"
35
67
  )
36
68
 
@@ -56,7 +88,17 @@ def get_coa(orcid, email=None):
56
88
  pubs += data["results"]
57
89
  next_cursor = data["meta"].get("next_cursor", None)
58
90
 
91
+ print(f"Found {len(pubs)} publications")
92
+
93
+ # Show publication years distribution
94
+ year_counts = {}
95
+ for pub in pubs:
96
+ year = pub.get("publication_year", "Unknown")
97
+ year_counts[year] = year_counts.get(year, 0) + 1
98
+ print("Publications by year:", dict(sorted(year_counts.items(), reverse=True)))
99
+
59
100
  # We get all the authors from all the papers first.
101
+ print("Extracting authors from publications...")
60
102
  authors = []
61
103
 
62
104
  for pub in pubs:
@@ -67,20 +109,31 @@ def get_coa(orcid, email=None):
67
109
 
68
110
  aus = pub["authorships"]
69
111
  for au in aus:
70
- hn = HumanName(au["author"]["display_name"])
112
+ # Skip authors with no display name or no OpenAlex ID
113
+ display_name = au["author"].get("display_name")
114
+ if not display_name or not au["author"].get("id"):
115
+ continue
116
+
117
+ hn = HumanName(display_name)
71
118
  name = f"{hn.last}, {hn.first} {hn.middle or ''}"
72
119
 
73
120
  authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
74
121
 
75
122
  # sort authors alphabetically, then by year descending
76
123
  authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
124
+ print(f"Found {len(authors)} total author entries (including duplicates)")
77
125
 
78
- # Now, get all the affiliations. This assumes the first one is most recent.
79
- # I could also use the last known institution, but this is sometimes empty
80
- # too.
126
+ # Now, get all the affiliations using a two-strategy approach:
127
+ # 1. Use last_known_institutions (from most recent publication)
128
+ # 2. Fall back to affiliation with most recent year in affiliations array
81
129
  oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
130
+ print(f"Found {len(oaids)} unique authors")
131
+ print("Fetching affiliation information from OpenAlex...")
82
132
  affiliations = {}
133
+ batch_num = 0
83
134
  for batch in batched(oaids, 50):
135
+ batch_num += 1
136
+ print(f" Fetching batch {batch_num} of authors...")
84
137
  url = f"https://api.openalex.org/authors?filter=id:{'|'.join(batch)}"
85
138
 
86
139
  params = {"per-page": 50}
@@ -92,12 +145,24 @@ def get_coa(orcid, email=None):
92
145
  d = requests.get(url, params=params)
93
146
 
94
147
  for au in d.json()["results"]:
95
- affils = au["affiliations"]
96
- if len(affils) > 0:
97
- affiliations[au["id"]] = affils[0]["institution"]["display_name"]
98
- else:
99
- affiliations[au["id"]] = ""
148
+ affiliation_name = ""
149
+
150
+ # Strategy 1: Use last_known_institutions (most recent publication)
151
+ last_known = au.get("last_known_institutions", [])
152
+ if last_known and len(last_known) > 0:
153
+ affiliation_name = last_known[0].get("display_name", "")
154
+
155
+ # Strategy 2: If no last_known_institutions, use affiliation with most recent year
156
+ if not affiliation_name:
157
+ affils = au.get("affiliations", [])
158
+ if len(affils) > 0:
159
+ # Find affiliation with most recent year
160
+ best_affil = max(affils, key=lambda a: max(a.get("years", [0])))
161
+ affiliation_name = best_affil["institution"]["display_name"]
100
162
 
163
+ affiliations[au["id"]] = affiliation_name
164
+
165
+ print("Building unique author list...")
101
166
  uniq = {}
102
167
  uniq_authors = [] # by openalex id
103
168
  all_authors = []
@@ -109,6 +174,8 @@ def get_coa(orcid, email=None):
109
174
  uniq_authors += [["A:", name, affil, "", last_active]]
110
175
  all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
111
176
 
177
+ print(f"Creating Excel file with {len(uniq_authors)} unique authors...")
178
+
112
179
  # unique authors
113
180
  df = pd.DataFrame(
114
181
  uniq_authors,
@@ -176,7 +243,9 @@ def get_coa(orcid, email=None):
176
243
  b64 = base64.b64encode(excel_bytes).decode("utf-8")
177
244
  uri = f'<pre>{filename}</pre><br><a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download COA</a>'
178
245
  display(HTML(uri))
246
+ print("✓ COA generation complete!")
179
247
  return None
180
248
  else:
181
249
  # Return path for MCP server / CLI
250
+ print(f"✓ COA saved to: {downloads_path}")
182
251
  return str(downloads_path)