litdb 2.2.1__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {litdb-2.2.1/src/litdb.egg-info → litdb-2.2.2}/PKG-INFO +7 -12
- {litdb-2.2.1 → litdb-2.2.2}/README.org +1 -9
- {litdb-2.2.1 → litdb-2.2.2}/pyproject.toml +6 -3
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/app.py +10 -8
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/bibtex.py +23 -8
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/chat.py +10 -8
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/coa.py +83 -14
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/data_processing.py +37 -4
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/manage.py +56 -1
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/utilities.py +19 -6
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/db.py +177 -28
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/doe_collaborators.py +15 -11
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/mcp_server.py +9 -7
- {litdb-2.2.1 → litdb-2.2.2/src/litdb.egg-info}/PKG-INFO +7 -12
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb.egg-info/requires.txt +5 -2
- {litdb-2.2.1 → litdb-2.2.2}/LICENSE +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/setup.cfg +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/__init__.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/audio.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/cli.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/__init__.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/export.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/filters.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/install.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/openalex_commands.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/research_commands.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/review.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/search.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/commands/tags.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/crawl.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/extract.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/gpt.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/images.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/lab.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/lsearch.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/openalex.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/pdf.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/research.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/server.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/summary.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/utils.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb/youtube.py +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb.egg-info/SOURCES.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb.egg-info/entry_points.txt +0 -0
- {litdb-2.2.1 → litdb-2.2.2}/src/litdb.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,7 +11,7 @@ License-File: LICENSE
|
|
|
11
11
|
Requires-Dist: toml
|
|
12
12
|
Requires-Dist: tomlkit
|
|
13
13
|
Requires-Dist: requests
|
|
14
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: pymupdf4llm<2
|
|
15
15
|
Requires-Dist: docling
|
|
16
16
|
Requires-Dist: langchain-core
|
|
17
17
|
Requires-Dist: langchain-text-splitters
|
|
@@ -33,7 +33,7 @@ Requires-Dist: dateparser
|
|
|
33
33
|
Requires-Dist: jinja2
|
|
34
34
|
Requires-Dist: pytesseract
|
|
35
35
|
Requires-Dist: pillow
|
|
36
|
-
Requires-Dist: bibtexparser
|
|
36
|
+
Requires-Dist: bibtexparser<2
|
|
37
37
|
Requires-Dist: pandas
|
|
38
38
|
Requires-Dist: more-itertools
|
|
39
39
|
Requires-Dist: nameparser
|
|
@@ -41,6 +41,9 @@ Requires-Dist: tabulate
|
|
|
41
41
|
Requires-Dist: IPython
|
|
42
42
|
Requires-Dist: gmft
|
|
43
43
|
Requires-Dist: ruff>=0.13.3
|
|
44
|
+
Requires-Dist: torch
|
|
45
|
+
Requires-Dist: torchvision
|
|
46
|
+
Requires-Dist: fastmcp>=2.13.3
|
|
44
47
|
Provides-Extra: test
|
|
45
48
|
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
46
49
|
Requires-Dist: pytest-cov>=4.1.0; extra == "test"
|
|
@@ -574,7 +577,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
574
577
|
|
|
575
578
|
Note that you need to install these libraries for this feature to work:
|
|
576
579
|
|
|
577
|
-
pyaudio,
|
|
580
|
+
pyaudio, SpeechRecognition
|
|
578
581
|
|
|
579
582
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
580
583
|
|
|
@@ -1189,11 +1192,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1189
1192
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1190
1193
|
|
|
1191
1194
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1192
|
-
|
|
1193
|
-
* Tasks
|
|
1194
|
-
|
|
1195
|
-
** DONE create a Claude skill for litdb
|
|
1196
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1197
|
-
|
|
1198
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1199
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -472,7 +472,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
472
472
|
|
|
473
473
|
Note that you need to install these libraries for this feature to work:
|
|
474
474
|
|
|
475
|
-
pyaudio,
|
|
475
|
+
pyaudio, SpeechRecognition
|
|
476
476
|
|
|
477
477
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
478
478
|
|
|
@@ -1087,11 +1087,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1087
1087
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1088
1088
|
|
|
1089
1089
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1090
|
-
|
|
1091
|
-
* Tasks
|
|
1092
|
-
|
|
1093
|
-
** DONE create a Claude skill for litdb
|
|
1094
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1095
|
-
|
|
1096
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1097
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -7,7 +7,7 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "litdb"
|
|
10
|
-
version = "2.2.
|
|
10
|
+
version = "2.2.2"
|
|
11
11
|
authors = [
|
|
12
12
|
{name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
|
|
13
13
|
]
|
|
@@ -22,7 +22,7 @@ dependencies = [
|
|
|
22
22
|
"toml",
|
|
23
23
|
"tomlkit",
|
|
24
24
|
"requests",
|
|
25
|
-
"pymupdf4llm",
|
|
25
|
+
"pymupdf4llm<2",
|
|
26
26
|
"docling",
|
|
27
27
|
"langchain-core",
|
|
28
28
|
"langchain-text-splitters",
|
|
@@ -44,7 +44,7 @@ dependencies = [
|
|
|
44
44
|
"jinja2",
|
|
45
45
|
"pytesseract",
|
|
46
46
|
"pillow",
|
|
47
|
-
"bibtexparser",
|
|
47
|
+
"bibtexparser<2",
|
|
48
48
|
# Data processing and COA
|
|
49
49
|
"pandas",
|
|
50
50
|
"more-itertools",
|
|
@@ -55,6 +55,9 @@ dependencies = [
|
|
|
55
55
|
"gmft",
|
|
56
56
|
# Development tools
|
|
57
57
|
"ruff>=0.13.3",
|
|
58
|
+
"torch",
|
|
59
|
+
"torchvision",
|
|
60
|
+
"fastmcp>=2.13.3",
|
|
58
61
|
]
|
|
59
62
|
|
|
60
63
|
[project.scripts]
|
|
@@ -205,10 +205,11 @@ def format_citation(extra):
|
|
|
205
205
|
parts.append(data["title"])
|
|
206
206
|
|
|
207
207
|
# Journal/venue
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
208
|
+
pl = data.get("primary_location") or {}
|
|
209
|
+
src = pl.get("source") or {}
|
|
210
|
+
venue = src.get("display_name", "")
|
|
211
|
+
if venue:
|
|
212
|
+
parts.append(f"*{venue}*")
|
|
212
213
|
|
|
213
214
|
return ". ".join(parts) if parts else None
|
|
214
215
|
except Exception:
|
|
@@ -1061,10 +1062,11 @@ def tab_openalex_search():
|
|
|
1061
1062
|
st.markdown(f"**Citations:** {citation_count}")
|
|
1062
1063
|
|
|
1063
1064
|
# Venue
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
)
|
|
1067
|
-
|
|
1065
|
+
pl = work.get("primary_location") or {}
|
|
1066
|
+
src = pl.get("source") or {}
|
|
1067
|
+
venue_name = src.get("display_name")
|
|
1068
|
+
if venue_name:
|
|
1069
|
+
st.markdown(f"**Venue:** {venue_name}")
|
|
1068
1070
|
|
|
1069
1071
|
# Abstract preview
|
|
1070
1072
|
if work.get("abstract_inverted_index"):
|
|
@@ -15,6 +15,11 @@ entry_type_by_crossref_type = {
|
|
|
15
15
|
"book-part": "inbook",
|
|
16
16
|
"other": "misc",
|
|
17
17
|
"paratext": "misc", # new line added
|
|
18
|
+
"erratum": "misc", # new line added
|
|
19
|
+
"letter": "article", # new line added
|
|
20
|
+
"editorial": "article", # new line added
|
|
21
|
+
"libguides": "misc", # new line added
|
|
22
|
+
"supplementary-materials": "misc", # new line added
|
|
18
23
|
"book": "book",
|
|
19
24
|
"journal-volume": "misc",
|
|
20
25
|
"book-set": "misc",
|
|
@@ -46,7 +51,8 @@ def dump_bibtex(work):
|
|
|
46
51
|
"Unable to generate a bibtex entry. This was probably an entry added by url or other non-doi source."
|
|
47
52
|
)
|
|
48
53
|
return None
|
|
49
|
-
|
|
54
|
+
# Try type_crossref first (for backwards compatibility), then fall back to type
|
|
55
|
+
wtype = work.get("type_crossref") or work.get("type")
|
|
50
56
|
if wtype is None:
|
|
51
57
|
print(f"Unable to generate a bibtex entry for {work.get('id')}")
|
|
52
58
|
return None
|
|
@@ -108,7 +114,7 @@ def _populate_entry(entry, work):
|
|
|
108
114
|
elif entry_type == "misc":
|
|
109
115
|
_set_if(entry, "author", _author(work))
|
|
110
116
|
_set_if(entry, "title", work.get("title"))
|
|
111
|
-
_set_if(entry, "howpublished", (work
|
|
117
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
112
118
|
_set_if(entry, "year", _year(work))
|
|
113
119
|
elif entry_type == "phdthesis":
|
|
114
120
|
_set_if(entry, "author", _author(work))
|
|
@@ -118,7 +124,7 @@ def _populate_entry(entry, work):
|
|
|
118
124
|
elif entry_type == "proceedings":
|
|
119
125
|
_set_if(entry, "editor", _author(work))
|
|
120
126
|
_set_if(entry, "title", work.get("title"))
|
|
121
|
-
_set_if(entry, "series",
|
|
127
|
+
_set_if(entry, "series", _primary_location_display_name(work))
|
|
122
128
|
_set_if(entry, "volume", _volume(work))
|
|
123
129
|
_set_if(entry, "publisher", _publisher(work))
|
|
124
130
|
_set_if(entry, "year", _year(work))
|
|
@@ -126,7 +132,7 @@ def _populate_entry(entry, work):
|
|
|
126
132
|
_set_if(entry, "author", _author(work))
|
|
127
133
|
_set_if(entry, "title", work.get("title"))
|
|
128
134
|
_set_if(entry, "year", _year(work))
|
|
129
|
-
_set_if(entry, "howpublished", (work
|
|
135
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
130
136
|
|
|
131
137
|
|
|
132
138
|
def _set_if(entry, key, value):
|
|
@@ -139,7 +145,7 @@ def _year(work):
|
|
|
139
145
|
|
|
140
146
|
|
|
141
147
|
def _book_title(work):
|
|
142
|
-
return (work
|
|
148
|
+
return _primary_location_display_name(work)
|
|
143
149
|
|
|
144
150
|
|
|
145
151
|
def _school(work):
|
|
@@ -168,11 +174,20 @@ def _journal_name(work):
|
|
|
168
174
|
|
|
169
175
|
|
|
170
176
|
def _publisher(work):
|
|
171
|
-
|
|
177
|
+
pl = work.get("primary_location") or {}
|
|
178
|
+
src = pl.get("source") or {}
|
|
179
|
+
return src.get("host_organization_name")
|
|
172
180
|
|
|
173
181
|
|
|
174
|
-
def
|
|
175
|
-
|
|
182
|
+
def _primary_location_display_name(work):
|
|
183
|
+
pl = work.get("primary_location") or {}
|
|
184
|
+
src = pl.get("source") or {}
|
|
185
|
+
return src.get("display_name")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _primary_location_url(work):
|
|
189
|
+
pl = work.get("primary_location") or {}
|
|
190
|
+
return pl.get("landing_page_url") or pl.get("pdf_url")
|
|
176
191
|
|
|
177
192
|
|
|
178
193
|
def _volume(work):
|
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
"""LitGPT using litellm."""
|
|
2
2
|
|
|
3
|
+
import importlib
|
|
4
|
+
import logging
|
|
3
5
|
import os
|
|
6
|
+
import pydoc
|
|
7
|
+
import re
|
|
4
8
|
import readline
|
|
5
9
|
import subprocess
|
|
6
|
-
|
|
7
10
|
import warnings
|
|
8
11
|
|
|
9
12
|
import numpy as np
|
|
13
|
+
from docling.document_converter import DocumentConverter
|
|
14
|
+
from docling.exceptions import ConversionError
|
|
10
15
|
from litellm import completion
|
|
11
16
|
from rich import print as richprint
|
|
12
17
|
from sentence_transformers import SentenceTransformer
|
|
13
18
|
|
|
14
|
-
import pydoc
|
|
15
|
-
import re
|
|
16
|
-
import importlib
|
|
17
|
-
from docling.document_converter import DocumentConverter
|
|
18
|
-
from docling.exceptions import ConversionError
|
|
19
|
-
import logging
|
|
20
|
-
|
|
21
19
|
# Try to import backoff (optional dependency in crawl extra)
|
|
22
20
|
try:
|
|
23
21
|
import backoff
|
|
@@ -220,6 +218,10 @@ def get_completion(model, messages, max_tokens=None):
|
|
|
220
218
|
|
|
221
219
|
except KeyboardInterrupt:
|
|
222
220
|
pass
|
|
221
|
+
except AttributeError as e:
|
|
222
|
+
# Suppress litellm TranscriptionCreateParams.__annotations__ errors
|
|
223
|
+
if "__annotations__" not in str(e):
|
|
224
|
+
raise
|
|
223
225
|
|
|
224
226
|
return output
|
|
225
227
|
|
|
@@ -16,21 +16,53 @@ def get_coa(orcid, email=None):
|
|
|
16
16
|
ORCID: str the author orcid to retrieve results for.
|
|
17
17
|
email: str optional email for OpenAlex API (for polite pool)
|
|
18
18
|
"""
|
|
19
|
+
print(f"Starting COA generation for ORCID: {orcid}")
|
|
19
20
|
|
|
20
|
-
|
|
21
|
+
# First, find the correct author profile (handle duplicate ORCID profiles)
|
|
22
|
+
orcid = orcid.replace("https://orcid.org/", "")
|
|
23
|
+
orcid_url = f"https://orcid.org/{orcid}"
|
|
21
24
|
|
|
22
|
-
|
|
25
|
+
print(f"Finding author profile for ORCID {orcid}...")
|
|
26
|
+
authors_url = "https://api.openalex.org/authors"
|
|
27
|
+
author_params = {"filter": f"orcid:{orcid_url}"}
|
|
28
|
+
if email:
|
|
29
|
+
author_params["mailto"] = email
|
|
23
30
|
|
|
31
|
+
author_resp = requests.get(authors_url, params=author_params)
|
|
32
|
+
author_resp.raise_for_status()
|
|
33
|
+
author_data = author_resp.json()
|
|
34
|
+
|
|
35
|
+
if not author_data["results"]:
|
|
36
|
+
raise ValueError(f"No author found for ORCID {orcid_url}")
|
|
37
|
+
|
|
38
|
+
# If multiple profiles exist with same ORCID, use the one with most works
|
|
39
|
+
if len(author_data["results"]) > 1:
|
|
40
|
+
print(
|
|
41
|
+
f" Warning: Found {len(author_data['results'])} author profiles with this ORCID"
|
|
42
|
+
)
|
|
43
|
+
for a in author_data["results"]:
|
|
44
|
+
print(
|
|
45
|
+
f" - {a['display_name']}: {a['works_count']} works (ID: {a['id']})"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Select author with most works
|
|
49
|
+
author = max(author_data["results"], key=lambda x: x.get("works_count", 0))
|
|
50
|
+
author_id = author["id"].replace("https://openalex.org/", "")
|
|
51
|
+
print(
|
|
52
|
+
f" Using profile: {author['display_name']} ({author['works_count']} total works)"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
url = "https://api.openalex.org/works"
|
|
56
|
+
next_cursor = "*"
|
|
24
57
|
pubs = []
|
|
25
58
|
|
|
26
59
|
current_year = datetime.datetime.now().year
|
|
27
60
|
four_years_ago = current_year - 4
|
|
28
|
-
|
|
29
|
-
orcid = orcid.replace("https://orcid.org/", "")
|
|
61
|
+
print(f"Fetching publications from {four_years_ago} to {current_year}...")
|
|
30
62
|
|
|
31
63
|
while next_cursor:
|
|
32
64
|
_filter = (
|
|
33
|
-
f"author.
|
|
65
|
+
f"author.id:https://openalex.org/{author_id}"
|
|
34
66
|
f",publication_year:>{four_years_ago - 1}"
|
|
35
67
|
)
|
|
36
68
|
|
|
@@ -56,7 +88,17 @@ def get_coa(orcid, email=None):
|
|
|
56
88
|
pubs += data["results"]
|
|
57
89
|
next_cursor = data["meta"].get("next_cursor", None)
|
|
58
90
|
|
|
91
|
+
print(f"Found {len(pubs)} publications")
|
|
92
|
+
|
|
93
|
+
# Show publication years distribution
|
|
94
|
+
year_counts = {}
|
|
95
|
+
for pub in pubs:
|
|
96
|
+
year = pub.get("publication_year", "Unknown")
|
|
97
|
+
year_counts[year] = year_counts.get(year, 0) + 1
|
|
98
|
+
print("Publications by year:", dict(sorted(year_counts.items(), reverse=True)))
|
|
99
|
+
|
|
59
100
|
# We get all the authors from all the papers first.
|
|
101
|
+
print("Extracting authors from publications...")
|
|
60
102
|
authors = []
|
|
61
103
|
|
|
62
104
|
for pub in pubs:
|
|
@@ -67,20 +109,31 @@ def get_coa(orcid, email=None):
|
|
|
67
109
|
|
|
68
110
|
aus = pub["authorships"]
|
|
69
111
|
for au in aus:
|
|
70
|
-
|
|
112
|
+
# Skip authors with no display name or no OpenAlex ID
|
|
113
|
+
display_name = au["author"].get("display_name")
|
|
114
|
+
if not display_name or not au["author"].get("id"):
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
hn = HumanName(display_name)
|
|
71
118
|
name = f"{hn.last}, {hn.first} {hn.middle or ''}"
|
|
72
119
|
|
|
73
120
|
authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
|
|
74
121
|
|
|
75
122
|
# sort authors alphabetically, then by year descending
|
|
76
123
|
authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
|
|
124
|
+
print(f"Found {len(authors)} total author entries (including duplicates)")
|
|
77
125
|
|
|
78
|
-
# Now, get all the affiliations
|
|
79
|
-
#
|
|
80
|
-
#
|
|
126
|
+
# Now, get all the affiliations using a two-strategy approach:
|
|
127
|
+
# 1. Use last_known_institutions (from most recent publication)
|
|
128
|
+
# 2. Fall back to affiliation with most recent year in affiliations array
|
|
81
129
|
oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
|
|
130
|
+
print(f"Found {len(oaids)} unique authors")
|
|
131
|
+
print("Fetching affiliation information from OpenAlex...")
|
|
82
132
|
affiliations = {}
|
|
133
|
+
batch_num = 0
|
|
83
134
|
for batch in batched(oaids, 50):
|
|
135
|
+
batch_num += 1
|
|
136
|
+
print(f" Fetching batch {batch_num} of authors...")
|
|
84
137
|
url = f"https://api.openalex.org/authors?filter=id:{'|'.join(batch)}"
|
|
85
138
|
|
|
86
139
|
params = {"per-page": 50}
|
|
@@ -92,12 +145,24 @@ def get_coa(orcid, email=None):
|
|
|
92
145
|
d = requests.get(url, params=params)
|
|
93
146
|
|
|
94
147
|
for au in d.json()["results"]:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
148
|
+
affiliation_name = ""
|
|
149
|
+
|
|
150
|
+
# Strategy 1: Use last_known_institutions (most recent publication)
|
|
151
|
+
last_known = au.get("last_known_institutions", [])
|
|
152
|
+
if last_known and len(last_known) > 0:
|
|
153
|
+
affiliation_name = last_known[0].get("display_name", "")
|
|
154
|
+
|
|
155
|
+
# Strategy 2: If no last_known_institutions, use affiliation with most recent year
|
|
156
|
+
if not affiliation_name:
|
|
157
|
+
affils = au.get("affiliations", [])
|
|
158
|
+
if len(affils) > 0:
|
|
159
|
+
# Find affiliation with most recent year
|
|
160
|
+
best_affil = max(affils, key=lambda a: max(a.get("years", [0])))
|
|
161
|
+
affiliation_name = best_affil["institution"]["display_name"]
|
|
100
162
|
|
|
163
|
+
affiliations[au["id"]] = affiliation_name
|
|
164
|
+
|
|
165
|
+
print("Building unique author list...")
|
|
101
166
|
uniq = {}
|
|
102
167
|
uniq_authors = [] # by openalex id
|
|
103
168
|
all_authors = []
|
|
@@ -109,6 +174,8 @@ def get_coa(orcid, email=None):
|
|
|
109
174
|
uniq_authors += [["A:", name, affil, "", last_active]]
|
|
110
175
|
all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
|
|
111
176
|
|
|
177
|
+
print(f"Creating Excel file with {len(uniq_authors)} unique authors...")
|
|
178
|
+
|
|
112
179
|
# unique authors
|
|
113
180
|
df = pd.DataFrame(
|
|
114
181
|
uniq_authors,
|
|
@@ -176,7 +243,9 @@ def get_coa(orcid, email=None):
|
|
|
176
243
|
b64 = base64.b64encode(excel_bytes).decode("utf-8")
|
|
177
244
|
uri = f'<pre>{filename}</pre><br><a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download COA</a>'
|
|
178
245
|
display(HTML(uri))
|
|
246
|
+
print("✓ COA generation complete!")
|
|
179
247
|
return None
|
|
180
248
|
else:
|
|
181
249
|
# Return path for MCP server / CLI
|
|
250
|
+
print(f"✓ COA saved to: {downloads_path}")
|
|
182
251
|
return str(downloads_path)
|
|
@@ -9,6 +9,7 @@ Commands:
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import sys
|
|
12
13
|
from difflib import SequenceMatcher
|
|
13
14
|
|
|
14
15
|
import click
|
|
@@ -24,7 +25,10 @@ from . import manage
|
|
|
24
25
|
@click.option("--references", is_flag=True)
|
|
25
26
|
@click.option("--related", is_flag=True)
|
|
26
27
|
@click.option("--citing", is_flag=True)
|
|
27
|
-
|
|
28
|
+
@click.option(
|
|
29
|
+
"-y", "--yes", is_flag=True, help="Auto-add all results (bypass selection prompt)."
|
|
30
|
+
)
|
|
31
|
+
def crossref(query, references, related, citing, yes):
|
|
28
32
|
"""Add entries to litdb from a crossref query."""
|
|
29
33
|
query = " ".join(query)
|
|
30
34
|
resp = requests.get("https://api.crossref.org/works", params={"query": query})
|
|
@@ -43,10 +47,24 @@ def crossref(query, references, related, citing):
|
|
|
43
47
|
f"{i}. {' '.join(title)}, {authors}, {source} ({year}), https://doi.org/{item['DOI']}."
|
|
44
48
|
)
|
|
45
49
|
|
|
46
|
-
|
|
50
|
+
# Handle selection based on yes flag or interactive mode
|
|
51
|
+
if yes:
|
|
52
|
+
# Add all results
|
|
53
|
+
toadd = list(range(len(data["message"]["items"])))
|
|
54
|
+
richprint("[green]Adding all results (--yes mode)[/green]")
|
|
55
|
+
elif not sys.stdin.isatty():
|
|
56
|
+
# Non-interactive mode: skip to avoid hanging
|
|
57
|
+
richprint("[yellow]Non-interactive mode, skipping selection.[/yellow]")
|
|
58
|
+
return
|
|
59
|
+
else:
|
|
60
|
+
# Interactive prompt
|
|
61
|
+
toadd = input("Enter space separated numbers to add, or return to quit. ")
|
|
62
|
+
if toadd:
|
|
63
|
+
toadd = [int(x) for x in toadd.split(" ")]
|
|
64
|
+
else:
|
|
65
|
+
return
|
|
47
66
|
|
|
48
67
|
if toadd:
|
|
49
|
-
toadd = [int(x) for x in toadd.split(" ")]
|
|
50
68
|
dois = [
|
|
51
69
|
"https://doi.org/" + data["message"]["items"][i]["DOI"] for i in toadd
|
|
52
70
|
]
|
|
@@ -58,6 +76,7 @@ def crossref(query, references, related, citing):
|
|
|
58
76
|
related=related,
|
|
59
77
|
references=references,
|
|
60
78
|
citing=citing,
|
|
79
|
+
yes=yes,
|
|
61
80
|
)
|
|
62
81
|
|
|
63
82
|
|
|
@@ -66,8 +85,11 @@ def crossref(query, references, related, citing):
|
|
|
66
85
|
@click.option("--references", is_flag=True, help="Add references too.")
|
|
67
86
|
@click.option("--related", is_flag=True, help="Add related too.")
|
|
68
87
|
@click.option("--citing", is_flag=True, help="Add citing too.")
|
|
88
|
+
@click.option(
|
|
89
|
+
"-y", "--yes", is_flag=True, help="Auto-confirm all matches (bypass prompts)."
|
|
90
|
+
)
|
|
69
91
|
@click.option("--model", default=None, help="LLM model to use for parsing.")
|
|
70
|
-
def fromtext(text, references, related, citing, model):
|
|
92
|
+
def fromtext(text, references, related, citing, yes, model):
|
|
71
93
|
"""Extract and add references from pasted TEXT.
|
|
72
94
|
|
|
73
95
|
TEXT should be a string containing academic references (from PDFs, websites, etc.).
|
|
@@ -238,6 +260,16 @@ Text:
|
|
|
238
260
|
" [green]High confidence match, adding automatically[/green]"
|
|
239
261
|
)
|
|
240
262
|
dois_to_add.append(best_doi)
|
|
263
|
+
# Auto-add if --yes flag is set
|
|
264
|
+
elif yes:
|
|
265
|
+
richprint(" [green]Adding (--yes mode)[/green]")
|
|
266
|
+
dois_to_add.append(best_doi)
|
|
267
|
+
# Non-interactive mode: skip to avoid hanging
|
|
268
|
+
elif not sys.stdin.isatty():
|
|
269
|
+
richprint(
|
|
270
|
+
" [yellow]Non-interactive mode, skipping low-confidence match[/yellow]"
|
|
271
|
+
)
|
|
272
|
+
# Interactive prompt
|
|
241
273
|
else:
|
|
242
274
|
confirm = input(" Add this? [y/N]: ")
|
|
243
275
|
if confirm.lower().startswith("y"):
|
|
@@ -262,6 +294,7 @@ Text:
|
|
|
262
294
|
references=references,
|
|
263
295
|
related=related,
|
|
264
296
|
citing=citing,
|
|
297
|
+
yes=yes,
|
|
265
298
|
)
|
|
266
299
|
richprint("[green]✓ Done![/green]")
|
|
267
300
|
else:
|
|
@@ -53,13 +53,41 @@ def init():
|
|
|
53
53
|
@click.option("--related", is_flag=True, help="Add related too.")
|
|
54
54
|
@click.option("--citing", is_flag=True, help="Add citing too.")
|
|
55
55
|
@click.option("--all", is_flag=True, help="Add references, related and citing.")
|
|
56
|
+
@click.option(
|
|
57
|
+
"-y", "--yes", is_flag=True, help="Auto-confirm prompts (bypass interactive mode)."
|
|
58
|
+
)
|
|
59
|
+
@click.option(
|
|
60
|
+
"--max-citing",
|
|
61
|
+
type=int,
|
|
62
|
+
default=None,
|
|
63
|
+
help="Max citing works to download (None=prompt, -1=no limit).",
|
|
64
|
+
)
|
|
65
|
+
@click.option(
|
|
66
|
+
"--max-references",
|
|
67
|
+
type=int,
|
|
68
|
+
default=None,
|
|
69
|
+
help="Max references to download (None=all, -1=no limit).",
|
|
70
|
+
)
|
|
71
|
+
@click.option(
|
|
72
|
+
"--max-related",
|
|
73
|
+
type=int,
|
|
74
|
+
default=None,
|
|
75
|
+
help="Max related works to download (None=all, -1=no limit).",
|
|
76
|
+
)
|
|
56
77
|
@click.option("-t", "--tag", "tags", multiple=True)
|
|
78
|
+
@click.option(
|
|
79
|
+
"-v", "--verbose", is_flag=True, help="Show details of works being added."
|
|
80
|
+
)
|
|
57
81
|
def add(
|
|
58
82
|
sources,
|
|
59
83
|
references=False,
|
|
60
84
|
citing=False,
|
|
61
85
|
related=False,
|
|
62
86
|
all=False,
|
|
87
|
+
yes=False,
|
|
88
|
+
max_citing=None,
|
|
89
|
+
max_references=None,
|
|
90
|
+
max_related=None,
|
|
63
91
|
verbose=False,
|
|
64
92
|
tags=None,
|
|
65
93
|
):
|
|
@@ -86,7 +114,17 @@ def add(
|
|
|
86
114
|
if all:
|
|
87
115
|
references, citing, related = True, True, True
|
|
88
116
|
|
|
89
|
-
add_work(
|
|
117
|
+
add_work(
|
|
118
|
+
source,
|
|
119
|
+
references,
|
|
120
|
+
citing,
|
|
121
|
+
related,
|
|
122
|
+
yes,
|
|
123
|
+
max_citing,
|
|
124
|
+
max_references,
|
|
125
|
+
max_related,
|
|
126
|
+
verbose,
|
|
127
|
+
)
|
|
90
128
|
|
|
91
129
|
# works from an author
|
|
92
130
|
elif "orcid" in source or source.lower().startswith("https://openalex.org/a"):
|
|
@@ -295,4 +333,21 @@ def update_embeddings():
|
|
|
295
333
|
db.execute(
|
|
296
334
|
"""create index if not exists embedding_idx ON sources (libsql_vector_idx(embedding))"""
|
|
297
335
|
)
|
|
336
|
+
|
|
337
|
+
# Update stored embedding config so the warning doesn't persist
|
|
338
|
+
import json
|
|
339
|
+
|
|
340
|
+
emb_config = json.dumps(
|
|
341
|
+
{
|
|
342
|
+
"model": config["embedding"]["model"],
|
|
343
|
+
"chunk_size": config["embedding"]["chunk_size"],
|
|
344
|
+
"chunk_overlap": config["embedding"]["chunk_overlap"],
|
|
345
|
+
},
|
|
346
|
+
sort_keys=True,
|
|
347
|
+
)
|
|
348
|
+
db.execute(
|
|
349
|
+
"""insert or replace into metadata(key, value)
|
|
350
|
+
values ('embedding_config', ?)""",
|
|
351
|
+
(emb_config,),
|
|
352
|
+
)
|
|
298
353
|
db.commit()
|
|
@@ -97,9 +97,23 @@ def audio(playback=False):
|
|
|
97
97
|
print("\n" + text + "\n")
|
|
98
98
|
|
|
99
99
|
if playback:
|
|
100
|
-
import
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
import platform
|
|
101
|
+
import subprocess
|
|
102
|
+
|
|
103
|
+
try:
|
|
104
|
+
system = platform.system()
|
|
105
|
+
if system == "Darwin":
|
|
106
|
+
subprocess.run(["afplay", afile], check=True)
|
|
107
|
+
elif system == "Linux":
|
|
108
|
+
subprocess.run(["aplay", afile], check=True)
|
|
109
|
+
elif system == "Windows":
|
|
110
|
+
import winsound
|
|
111
|
+
|
|
112
|
+
winsound.PlaySound(afile, winsound.SND_FILENAME)
|
|
113
|
+
else:
|
|
114
|
+
print("Audio playback not supported on this platform.")
|
|
115
|
+
except (FileNotFoundError, subprocess.CalledProcessError):
|
|
116
|
+
print("Audio playback failed. Required system tool not found.")
|
|
103
117
|
|
|
104
118
|
response = input("Is that what you want to search? ([y]/n/q): ")
|
|
105
119
|
if response.lower().startswith("q"):
|
|
@@ -133,10 +147,9 @@ def app():
|
|
|
133
147
|
@click.command()
|
|
134
148
|
def version():
|
|
135
149
|
"""Print the version of litdb."""
|
|
136
|
-
import
|
|
150
|
+
from importlib.metadata import version as dist_version
|
|
137
151
|
|
|
138
|
-
version
|
|
139
|
-
print(f"Litdb: version {version}")
|
|
152
|
+
print(f"Litdb: version {dist_version('litdb')}")
|
|
140
153
|
|
|
141
154
|
|
|
142
155
|
@click.command()
|
|
@@ -1,23 +1,68 @@
|
|
|
1
1
|
"""Setup and add things to the database."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import logging
|
|
4
5
|
import os
|
|
6
|
+
import sys
|
|
5
7
|
from pathlib import Path
|
|
6
8
|
|
|
7
9
|
import libsql
|
|
8
10
|
|
|
9
11
|
from .utils import get_config
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
import datetime
|
|
16
|
-
import bibtexparser
|
|
17
|
-
from rich import print
|
|
13
|
+
# Suppress sentence_transformers INFO logs and progress bars
|
|
14
|
+
logging.getLogger("sentence_transformers").setLevel(logging.WARNING)
|
|
15
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
16
|
+
os.environ["TQDM_DISABLE"] = "1"
|
|
18
17
|
|
|
19
|
-
|
|
20
|
-
from
|
|
18
|
+
import numpy as np # noqa: E402
|
|
19
|
+
from sentence_transformers import SentenceTransformer # noqa: E402
|
|
20
|
+
from langchain_text_splitters import RecursiveCharacterTextSplitter # noqa: E402
|
|
21
|
+
from tqdm import tqdm # noqa: E402
|
|
22
|
+
import datetime # noqa: E402
|
|
23
|
+
import bibtexparser # noqa: E402
|
|
24
|
+
from rich import print # noqa: E402
|
|
25
|
+
|
|
26
|
+
from litdb.openalex import get_data, get_text # noqa: E402
|
|
27
|
+
from litdb.bibtex import dump_bibtex # noqa: E402
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _check_embedding_config(db, config):
|
|
31
|
+
"""Check if embedding config has changed and warn the user.
|
|
32
|
+
|
|
33
|
+
Stores the embedding model, chunk_size, and chunk_overlap in the metadata
|
|
34
|
+
table. If any of these differ from what's in the config file, prints a
|
|
35
|
+
warning telling the user to run 'litdb update-embeddings'.
|
|
36
|
+
"""
|
|
37
|
+
emb_config = {
|
|
38
|
+
"model": config["embedding"]["model"],
|
|
39
|
+
"chunk_size": config["embedding"]["chunk_size"],
|
|
40
|
+
"chunk_overlap": config["embedding"]["chunk_overlap"],
|
|
41
|
+
}
|
|
42
|
+
current = json.dumps(emb_config, sort_keys=True)
|
|
43
|
+
|
|
44
|
+
row = db.execute(
|
|
45
|
+
"select value from metadata where key = 'embedding_config'"
|
|
46
|
+
).fetchone()
|
|
47
|
+
|
|
48
|
+
if row is None:
|
|
49
|
+
# First time: store the config
|
|
50
|
+
db.execute(
|
|
51
|
+
"insert into metadata(key, value) values ('embedding_config', ?)",
|
|
52
|
+
(current,),
|
|
53
|
+
)
|
|
54
|
+
db.commit()
|
|
55
|
+
elif row[0] != current:
|
|
56
|
+
stored = json.loads(row[0])
|
|
57
|
+
changes = []
|
|
58
|
+
for k in emb_config:
|
|
59
|
+
if emb_config[k] != stored.get(k):
|
|
60
|
+
changes.append(f" {k}: {stored.get(k)} -> {emb_config[k]}")
|
|
61
|
+
print(
|
|
62
|
+
"[bold yellow]Warning:[/bold yellow] Embedding config has changed:\n"
|
|
63
|
+
+ "\n".join(changes)
|
|
64
|
+
+ "\nRun [bold]litdb update-embeddings[/bold] to rebuild embeddings."
|
|
65
|
+
)
|
|
21
66
|
|
|
22
67
|
|
|
23
68
|
def get_db():
|
|
@@ -107,6 +152,14 @@ def get_db():
|
|
|
107
152
|
date_added text)"""
|
|
108
153
|
)
|
|
109
154
|
|
|
155
|
+
db.execute(
|
|
156
|
+
"""create table if not exists
|
|
157
|
+
metadata(key text primary key, value text)"""
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# Store embedding config and warn if it changed
|
|
161
|
+
_check_embedding_config(db, config)
|
|
162
|
+
|
|
110
163
|
db.execute(
|
|
111
164
|
"""create index if not exists image_idx
|
|
112
165
|
ON images (libsql_vector_idx(embedding))"""
|
|
@@ -187,8 +240,9 @@ def get_citation(openalex_data):
|
|
|
187
240
|
parts.append(", ".join(authors))
|
|
188
241
|
|
|
189
242
|
# Journal/venue
|
|
190
|
-
|
|
191
|
-
|
|
243
|
+
pl = openalex_data.get("primary_location") or {}
|
|
244
|
+
src = pl.get("source") or {}
|
|
245
|
+
venue = src.get("display_name")
|
|
192
246
|
if venue:
|
|
193
247
|
parts.append(venue)
|
|
194
248
|
|
|
@@ -229,7 +283,17 @@ def get_citation(openalex_data):
|
|
|
229
283
|
return None
|
|
230
284
|
|
|
231
285
|
|
|
232
|
-
def add_work(
|
|
286
|
+
def add_work(
|
|
287
|
+
workid,
|
|
288
|
+
references=False,
|
|
289
|
+
citing=False,
|
|
290
|
+
related=False,
|
|
291
|
+
yes=False,
|
|
292
|
+
max_citing=None,
|
|
293
|
+
max_references=None,
|
|
294
|
+
max_related=None,
|
|
295
|
+
verbose=False,
|
|
296
|
+
):
|
|
233
297
|
"""Add a single work to litdb.
|
|
234
298
|
|
|
235
299
|
workid is the doi (full url), or it could be an OpenAlex workid. DOI is
|
|
@@ -238,6 +302,11 @@ def add_work(workid, references=False, citing=False, related=False):
|
|
|
238
302
|
if references is truthy, also add them.
|
|
239
303
|
if citing is truthy, also add them.
|
|
240
304
|
if related is truthy, also add them.
|
|
305
|
+
if yes is truthy, bypass interactive prompts and proceed automatically.
|
|
306
|
+
max_citing limits the number of citing works to download (None = use prompt logic, -1 = no limit).
|
|
307
|
+
max_references limits the number of references to download (None = all, -1 = no limit).
|
|
308
|
+
max_related limits the number of related works to download (None = all, -1 = no limit).
|
|
309
|
+
verbose prints details about each work being added.
|
|
241
310
|
|
|
242
311
|
"""
|
|
243
312
|
config = get_config()
|
|
@@ -257,36 +326,76 @@ def add_work(workid, references=False, citing=False, related=False):
|
|
|
257
326
|
data["citation"] = get_citation(data)
|
|
258
327
|
data["bibtex"] = dump_bibtex(data)
|
|
259
328
|
|
|
329
|
+
if verbose:
|
|
330
|
+
title = data.get("title", "No title")
|
|
331
|
+
doi = data.get("doi", "No DOI")
|
|
332
|
+
year = data.get("publication_year", "Unknown year")
|
|
333
|
+
print(f"Adding: {title[:80]}{'...' if len(title) > 80 else ''}")
|
|
334
|
+
print(f" DOI: {doi} | Year: {year}")
|
|
335
|
+
|
|
260
336
|
add_source(workid, get_text(data), data)
|
|
261
337
|
|
|
262
338
|
if references:
|
|
263
|
-
|
|
264
|
-
|
|
339
|
+
# Apply limit: None or -1 means all, otherwise limit to max_references
|
|
340
|
+
ref_works = data["referenced_works"]
|
|
341
|
+
if max_references is not None and max_references != -1:
|
|
342
|
+
ref_works = ref_works[:max_references]
|
|
343
|
+
|
|
344
|
+
for wid in tqdm(ref_works):
|
|
345
|
+
# Extract work ID from full URL (e.g., https://openalex.org/W3103133276 -> W3103133276)
|
|
346
|
+
work_id = wid.split("/")[-1] if "/" in wid else wid
|
|
347
|
+
api_url = "https://api.openalex.org/works/" + work_id
|
|
348
|
+
rdata = get_data(api_url, params)
|
|
265
349
|
source = rdata.get("doi") or rdata.get("id")
|
|
266
350
|
if source is None:
|
|
267
|
-
print(f"
|
|
351
|
+
print(f"Referenced work not found (may be removed/merged): {work_id}")
|
|
352
|
+
print(f" API URL: {api_url}")
|
|
268
353
|
continue
|
|
354
|
+
if verbose:
|
|
355
|
+
title = rdata.get("title", "No title") or "No title"
|
|
356
|
+
year = rdata.get("publication_year", "Unknown")
|
|
357
|
+
print(
|
|
358
|
+
f" [ref] {title[:70]}{'...' if len(title) > 70 else ''} ({year})"
|
|
359
|
+
)
|
|
269
360
|
text = get_text(rdata)
|
|
270
361
|
rdata["citation"] = get_citation(rdata)
|
|
271
362
|
rdata["bibtex"] = dump_bibtex(rdata)
|
|
272
363
|
add_source(source, text, rdata)
|
|
273
364
|
|
|
274
365
|
if related:
|
|
275
|
-
|
|
276
|
-
|
|
366
|
+
# Apply limit: None or -1 means all, otherwise limit to max_related
|
|
367
|
+
rel_works = data["related_works"]
|
|
368
|
+
if max_related is not None and max_related != -1:
|
|
369
|
+
rel_works = rel_works[:max_related]
|
|
370
|
+
|
|
371
|
+
for wid in tqdm(rel_works):
|
|
372
|
+
# Extract work ID from full URL (e.g., https://openalex.org/W3103133276 -> W3103133276)
|
|
373
|
+
work_id = wid.split("/")[-1] if "/" in wid else wid
|
|
374
|
+
api_url = "https://api.openalex.org/works/" + work_id
|
|
375
|
+
rdata = get_data(api_url, params)
|
|
277
376
|
source = rdata.get("doi") or rdata.get("id")
|
|
278
377
|
if source is None:
|
|
279
|
-
print(f"
|
|
378
|
+
print(f"Related work not found (may be removed/merged): {work_id}")
|
|
379
|
+
print(f" API URL: {api_url}")
|
|
280
380
|
continue
|
|
381
|
+
if verbose:
|
|
382
|
+
title = rdata.get("title", "No title") or "No title"
|
|
383
|
+
year = rdata.get("publication_year", "Unknown")
|
|
384
|
+
print(
|
|
385
|
+
f" [rel] {title[:70]}{'...' if len(title) > 70 else ''} ({year})"
|
|
386
|
+
)
|
|
281
387
|
text = get_text(rdata)
|
|
282
388
|
rdata["citation"] = get_citation(rdata)
|
|
283
389
|
rdata["bibtex"] = dump_bibtex(rdata)
|
|
284
390
|
add_source(source, text, rdata)
|
|
285
391
|
|
|
286
392
|
if citing:
|
|
287
|
-
|
|
393
|
+
# Construct the citing works URL manually since OpenAlex API no longer includes cited_by_api_url
|
|
394
|
+
work_id = data["id"].split("/")[-1] # Extract work ID from the full URL
|
|
395
|
+
CURL = f"https://api.openalex.org/works?filter=cites:{work_id}"
|
|
288
396
|
next_cursor = "*"
|
|
289
397
|
params.update(cursor=next_cursor)
|
|
398
|
+
citing_downloaded = 0
|
|
290
399
|
while next_cursor:
|
|
291
400
|
cdata = get_data(CURL, params)
|
|
292
401
|
next_cursor = cdata["meta"]["next_cursor"]
|
|
@@ -294,24 +403,64 @@ def add_work(workid, references=False, citing=False, related=False):
|
|
|
294
403
|
if next_cursor is None:
|
|
295
404
|
break
|
|
296
405
|
|
|
297
|
-
# TODO: should the max citations to trigger this be configurable in litdb.toml
|
|
298
|
-
trigger = config["openalex"].get("citation_count_trigger", 100)
|
|
299
406
|
count = cdata["meta"]["count"]
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
if
|
|
305
|
-
|
|
306
|
-
|
|
407
|
+
|
|
408
|
+
# Handle max_citing limit
|
|
409
|
+
if max_citing is not None:
|
|
410
|
+
# -1 means no limit, download all
|
|
411
|
+
if max_citing == -1:
|
|
412
|
+
pass # Download all
|
|
413
|
+
# Positive number: limit to that many
|
|
414
|
+
elif max_citing > 0:
|
|
415
|
+
if citing_downloaded >= max_citing:
|
|
416
|
+
print(f"Reached limit of {max_citing} citing works.")
|
|
417
|
+
break
|
|
418
|
+
# Don't prompt if we're within the limit
|
|
307
419
|
pass
|
|
420
|
+
# No max_citing specified, use existing trigger logic
|
|
421
|
+
else:
|
|
422
|
+
# TODO: should the max citations to trigger this be configurable in litdb.toml
|
|
423
|
+
trigger = config["openalex"].get("citation_count_trigger", 100)
|
|
424
|
+
if count > trigger:
|
|
425
|
+
# Option 1: --yes flag bypasses prompt
|
|
426
|
+
if yes:
|
|
427
|
+
# Download all
|
|
428
|
+
pass
|
|
429
|
+
# Option 2: Check if non-interactive mode (piped, scripted, etc.)
|
|
430
|
+
elif not sys.stdin.isatty():
|
|
431
|
+
# Non-interactive: default to NO (safer)
|
|
432
|
+
print(
|
|
433
|
+
f"Found {count} citations. Non-interactive mode, skipping."
|
|
434
|
+
)
|
|
435
|
+
break
|
|
436
|
+
# Option 3: Normal interactive prompt
|
|
437
|
+
else:
|
|
438
|
+
r = input(
|
|
439
|
+
f"Found {count} citations. Do you want to download them all? (n/y): "
|
|
440
|
+
)
|
|
441
|
+
if r.lower().startswith("n"):
|
|
442
|
+
break
|
|
443
|
+
else:
|
|
444
|
+
pass
|
|
308
445
|
|
|
309
446
|
for work in tqdm(cdata["results"]):
|
|
447
|
+
# Check if we've reached the limit
|
|
448
|
+
if max_citing is not None and max_citing > 0:
|
|
449
|
+
if citing_downloaded >= max_citing:
|
|
450
|
+
break
|
|
451
|
+
|
|
310
452
|
source = work.get("doi") or work["id"]
|
|
453
|
+
if verbose:
|
|
454
|
+
title = work.get("title", "No title") or "No title"
|
|
455
|
+
year = work.get("publication_year", "Unknown")
|
|
456
|
+
print(
|
|
457
|
+
f" [cit] {title[:70]}{'...' if len(title) > 70 else ''} ({year})"
|
|
458
|
+
)
|
|
311
459
|
text = get_text(work)
|
|
312
460
|
work["citation"] = get_citation(work)
|
|
313
461
|
work["bibtex"] = dump_bibtex(work)
|
|
314
462
|
add_source(source, text, work)
|
|
463
|
+
citing_downloaded += 1
|
|
315
464
|
|
|
316
465
|
|
|
317
466
|
def add_author(oaid):
|
|
@@ -10,9 +10,10 @@ import requests
|
|
|
10
10
|
import base64
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def get_coa(orcid):
|
|
13
|
+
def get_coa(orcid, email=None):
|
|
14
14
|
"""Generate Table 4 for the NSF COA.
|
|
15
15
|
ORCID: str the author orcid to retrieve results for.
|
|
16
|
+
email: str optional email for OpenAlex API (for polite pool)
|
|
16
17
|
"""
|
|
17
18
|
|
|
18
19
|
url = "https://api.openalex.org/works"
|
|
@@ -32,14 +33,14 @@ def get_coa(orcid):
|
|
|
32
33
|
f",publication_year:>{four_years_ago - 1}"
|
|
33
34
|
)
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
)
|
|
36
|
+
params = {
|
|
37
|
+
"filter": _filter,
|
|
38
|
+
"cursor": next_cursor,
|
|
39
|
+
}
|
|
40
|
+
if email:
|
|
41
|
+
params["mailto"] = email
|
|
42
|
+
|
|
43
|
+
r = requests.get(url, params=params)
|
|
43
44
|
data = r.json()
|
|
44
45
|
pubs += data["results"]
|
|
45
46
|
next_cursor = data["meta"].get("next_cursor", None)
|
|
@@ -71,7 +72,9 @@ def get_coa(orcid):
|
|
|
71
72
|
for batch in batched(oaids, 50):
|
|
72
73
|
url = f"https://api.openalex.org/authors?filter=id:{'|'.join(batch)}"
|
|
73
74
|
|
|
74
|
-
params = {"per-page": 50
|
|
75
|
+
params = {"per-page": 50}
|
|
76
|
+
if email:
|
|
77
|
+
params["mailto"] = email
|
|
75
78
|
|
|
76
79
|
d = requests.get(url, params=params)
|
|
77
80
|
|
|
@@ -156,5 +159,6 @@ if __name__ == "__main__":
|
|
|
156
159
|
import sys
|
|
157
160
|
|
|
158
161
|
orcid = sys.argv[1]
|
|
162
|
+
email = sys.argv[2] if len(sys.argv) > 2 else None
|
|
159
163
|
|
|
160
|
-
get_coa(orcid)
|
|
164
|
+
get_coa(orcid, email=email)
|
|
@@ -159,14 +159,20 @@ def vsearch(query: str, n: int = 3) -> str:
|
|
|
159
159
|
|
|
160
160
|
|
|
161
161
|
@mcp.tool()
|
|
162
|
-
def openalex(query: str, n: int = 5):
|
|
162
|
+
def openalex(query: str, n: int = 5, email: str = None, api_key: str = None):
|
|
163
163
|
"""Run a simple keyword query in OpenAlex.
|
|
164
164
|
|
|
165
165
|
Args:
|
|
166
166
|
query: string, natural language query.
|
|
167
167
|
n: int, number of results to return.
|
|
168
|
+
email: optional email for OpenAlex polite pool.
|
|
169
|
+
api_key: optional OpenAlex API key.
|
|
168
170
|
"""
|
|
169
171
|
params = {"filter": f"default.search:{query}", "per_page": n}
|
|
172
|
+
if email:
|
|
173
|
+
params["mailto"] = email
|
|
174
|
+
if api_key:
|
|
175
|
+
params["api_key"] = api_key
|
|
170
176
|
|
|
171
177
|
resp = requests.get("https://api.openalex.org/works", params)
|
|
172
178
|
data = resp.json()
|
|
@@ -210,9 +216,7 @@ def openalex(query: str, n: int = 5):
|
|
|
210
216
|
|
|
211
217
|
# Get journal/venue
|
|
212
218
|
venue = ""
|
|
213
|
-
if work.get("
|
|
214
|
-
venue = work["host_venue"].get("display_name", "")
|
|
215
|
-
elif work.get("primary_location", {}).get("source"):
|
|
219
|
+
if work.get("primary_location", {}).get("source"):
|
|
216
220
|
venue = work["primary_location"]["source"].get("display_name", "")
|
|
217
221
|
|
|
218
222
|
result_text = f"{i}. {title}"
|
|
@@ -418,9 +422,7 @@ def get_source_details(source: str) -> str:
|
|
|
418
422
|
citation = extra_data.get("citation", "")
|
|
419
423
|
abstract = extra_data.get("abstract", "")
|
|
420
424
|
venue = ""
|
|
421
|
-
if extra_data.get("
|
|
422
|
-
venue = extra_data["host_venue"].get("display_name", "")
|
|
423
|
-
elif extra_data.get("primary_location", {}).get("source"):
|
|
425
|
+
if extra_data.get("primary_location", {}).get("source"):
|
|
424
426
|
venue = extra_data["primary_location"]["source"].get("display_name", "")
|
|
425
427
|
|
|
426
428
|
cited_by_count = extra_data.get("cited_by_count", 0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,7 +11,7 @@ License-File: LICENSE
|
|
|
11
11
|
Requires-Dist: toml
|
|
12
12
|
Requires-Dist: tomlkit
|
|
13
13
|
Requires-Dist: requests
|
|
14
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: pymupdf4llm<2
|
|
15
15
|
Requires-Dist: docling
|
|
16
16
|
Requires-Dist: langchain-core
|
|
17
17
|
Requires-Dist: langchain-text-splitters
|
|
@@ -33,7 +33,7 @@ Requires-Dist: dateparser
|
|
|
33
33
|
Requires-Dist: jinja2
|
|
34
34
|
Requires-Dist: pytesseract
|
|
35
35
|
Requires-Dist: pillow
|
|
36
|
-
Requires-Dist: bibtexparser
|
|
36
|
+
Requires-Dist: bibtexparser<2
|
|
37
37
|
Requires-Dist: pandas
|
|
38
38
|
Requires-Dist: more-itertools
|
|
39
39
|
Requires-Dist: nameparser
|
|
@@ -41,6 +41,9 @@ Requires-Dist: tabulate
|
|
|
41
41
|
Requires-Dist: IPython
|
|
42
42
|
Requires-Dist: gmft
|
|
43
43
|
Requires-Dist: ruff>=0.13.3
|
|
44
|
+
Requires-Dist: torch
|
|
45
|
+
Requires-Dist: torchvision
|
|
46
|
+
Requires-Dist: fastmcp>=2.13.3
|
|
44
47
|
Provides-Extra: test
|
|
45
48
|
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
46
49
|
Requires-Dist: pytest-cov>=4.1.0; extra == "test"
|
|
@@ -574,7 +577,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
574
577
|
|
|
575
578
|
Note that you need to install these libraries for this feature to work:
|
|
576
579
|
|
|
577
|
-
pyaudio,
|
|
580
|
+
pyaudio, SpeechRecognition
|
|
578
581
|
|
|
579
582
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
580
583
|
|
|
@@ -1189,11 +1192,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1189
1192
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1190
1193
|
|
|
1191
1194
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1192
|
-
|
|
1193
|
-
* Tasks
|
|
1194
|
-
|
|
1195
|
-
** DONE create a Claude skill for litdb
|
|
1196
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1197
|
-
|
|
1198
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1199
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
toml
|
|
2
2
|
tomlkit
|
|
3
3
|
requests
|
|
4
|
-
pymupdf4llm
|
|
4
|
+
pymupdf4llm<2
|
|
5
5
|
docling
|
|
6
6
|
langchain-core
|
|
7
7
|
langchain-text-splitters
|
|
@@ -23,7 +23,7 @@ dateparser
|
|
|
23
23
|
jinja2
|
|
24
24
|
pytesseract
|
|
25
25
|
pillow
|
|
26
|
-
bibtexparser
|
|
26
|
+
bibtexparser<2
|
|
27
27
|
pandas
|
|
28
28
|
more-itertools
|
|
29
29
|
nameparser
|
|
@@ -31,6 +31,9 @@ tabulate
|
|
|
31
31
|
IPython
|
|
32
32
|
gmft
|
|
33
33
|
ruff>=0.13.3
|
|
34
|
+
torch
|
|
35
|
+
torchvision
|
|
36
|
+
fastmcp>=2.13.3
|
|
34
37
|
|
|
35
38
|
[all]
|
|
36
39
|
futurehouse_client
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|