litdb 2.2.0__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {litdb-2.2.0/src/litdb.egg-info → litdb-2.2.2}/PKG-INFO +7 -12
- {litdb-2.2.0 → litdb-2.2.2}/README.org +1 -9
- {litdb-2.2.0 → litdb-2.2.2}/pyproject.toml +6 -3
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/app.py +10 -8
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/bibtex.py +23 -8
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/chat.py +10 -8
- litdb-2.2.2/src/litdb/coa.py +251 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/data_processing.py +37 -4
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/manage.py +56 -1
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/utilities.py +19 -6
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/db.py +177 -28
- litdb-2.2.0/src/litdb/coa.py → litdb-2.2.2/src/litdb/doe_collaborators.py +9 -12
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/mcp_server.py +26 -16
- {litdb-2.2.0 → litdb-2.2.2/src/litdb.egg-info}/PKG-INFO +7 -12
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb.egg-info/requires.txt +5 -2
- litdb-2.2.0/src/litdb/doe_collaborators.py +0 -160
- {litdb-2.2.0 → litdb-2.2.2}/LICENSE +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/setup.cfg +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/__init__.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/audio.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/cli.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/__init__.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/export.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/filters.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/install.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/openalex_commands.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/research_commands.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/review.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/search.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/commands/tags.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/crawl.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/extract.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/gpt.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/images.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/lab.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/lsearch.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/openalex.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/pdf.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/research.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/server.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/summary.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/utils.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb/youtube.py +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb.egg-info/SOURCES.txt +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb.egg-info/dependency_links.txt +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb.egg-info/entry_points.txt +0 -0
- {litdb-2.2.0 → litdb-2.2.2}/src/litdb.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: litdb
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: A literature database tool with GPT integration.
|
|
5
5
|
Author-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
6
6
|
Maintainer-email: John Kitchin <jkitchin@andrew.cmu.edu>
|
|
@@ -11,7 +11,7 @@ License-File: LICENSE
|
|
|
11
11
|
Requires-Dist: toml
|
|
12
12
|
Requires-Dist: tomlkit
|
|
13
13
|
Requires-Dist: requests
|
|
14
|
-
Requires-Dist: pymupdf4llm
|
|
14
|
+
Requires-Dist: pymupdf4llm<2
|
|
15
15
|
Requires-Dist: docling
|
|
16
16
|
Requires-Dist: langchain-core
|
|
17
17
|
Requires-Dist: langchain-text-splitters
|
|
@@ -33,7 +33,7 @@ Requires-Dist: dateparser
|
|
|
33
33
|
Requires-Dist: jinja2
|
|
34
34
|
Requires-Dist: pytesseract
|
|
35
35
|
Requires-Dist: pillow
|
|
36
|
-
Requires-Dist: bibtexparser
|
|
36
|
+
Requires-Dist: bibtexparser<2
|
|
37
37
|
Requires-Dist: pandas
|
|
38
38
|
Requires-Dist: more-itertools
|
|
39
39
|
Requires-Dist: nameparser
|
|
@@ -41,6 +41,9 @@ Requires-Dist: tabulate
|
|
|
41
41
|
Requires-Dist: IPython
|
|
42
42
|
Requires-Dist: gmft
|
|
43
43
|
Requires-Dist: ruff>=0.13.3
|
|
44
|
+
Requires-Dist: torch
|
|
45
|
+
Requires-Dist: torchvision
|
|
46
|
+
Requires-Dist: fastmcp>=2.13.3
|
|
44
47
|
Provides-Extra: test
|
|
45
48
|
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
46
49
|
Requires-Dist: pytest-cov>=4.1.0; extra == "test"
|
|
@@ -574,7 +577,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
574
577
|
|
|
575
578
|
Note that you need to install these libraries for this feature to work:
|
|
576
579
|
|
|
577
|
-
pyaudio,
|
|
580
|
+
pyaudio, SpeechRecognition
|
|
578
581
|
|
|
579
582
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
580
583
|
|
|
@@ -1189,11 +1192,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1189
1192
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1190
1193
|
|
|
1191
1194
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1192
|
-
|
|
1193
|
-
* Tasks
|
|
1194
|
-
|
|
1195
|
-
** DONE create a Claude skill for litdb
|
|
1196
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1197
|
-
|
|
1198
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1199
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -472,7 +472,7 @@ I haven't found the transcription to be that good on technical scientific terms.
|
|
|
472
472
|
|
|
473
473
|
Note that you need to install these libraries for this feature to work:
|
|
474
474
|
|
|
475
|
-
pyaudio,
|
|
475
|
+
pyaudio, SpeechRecognition
|
|
476
476
|
|
|
477
477
|
These are not trivial to install, and pyaudio relies on external libraries like portaudio that may not be easy to install. These are currently commented out in pyproject.toml because of these difficulties.
|
|
478
478
|
|
|
@@ -1087,11 +1087,3 @@ Many of these projects require you to make an account. There are freemium levels
|
|
|
1087
1087
|
- Another tool that runs LLMs locally, and says it can index your files so you can chat with them.
|
|
1088
1088
|
|
|
1089
1089
|
With all these options, why does litdb exist? There are a lot of answers to that. First, I wanted to make it. I learned a lot about vector search by doing it. Second, I wanted a free, extensible solution for literature search that could also work for my local files while never putting data in the cloud, and that would work in Emacs. The projects above are very nice, easy to use, no or low-code solutions, and if that is what you are looking for, look there! If you want to hack on things yourself, look here.
|
|
1090
|
-
|
|
1091
|
-
* Tasks
|
|
1092
|
-
|
|
1093
|
-
** DONE create a Claude skill for litdb
|
|
1094
|
-
CLOSED: [2025-10-21 Tue 20:26] DEADLINE: <2025-10-22 Wed>
|
|
1095
|
-
|
|
1096
|
-
** DONE Make a nicer streamlit app for the front end.
|
|
1097
|
-
CLOSED: [2025-10-28 Tue 10:40]
|
|
@@ -7,7 +7,7 @@ where = ["src"]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "litdb"
|
|
10
|
-
version = "2.2.
|
|
10
|
+
version = "2.2.2"
|
|
11
11
|
authors = [
|
|
12
12
|
{name = "John Kitchin", email = "jkitchin@andrew.cmu.edu"},
|
|
13
13
|
]
|
|
@@ -22,7 +22,7 @@ dependencies = [
|
|
|
22
22
|
"toml",
|
|
23
23
|
"tomlkit",
|
|
24
24
|
"requests",
|
|
25
|
-
"pymupdf4llm",
|
|
25
|
+
"pymupdf4llm<2",
|
|
26
26
|
"docling",
|
|
27
27
|
"langchain-core",
|
|
28
28
|
"langchain-text-splitters",
|
|
@@ -44,7 +44,7 @@ dependencies = [
|
|
|
44
44
|
"jinja2",
|
|
45
45
|
"pytesseract",
|
|
46
46
|
"pillow",
|
|
47
|
-
"bibtexparser",
|
|
47
|
+
"bibtexparser<2",
|
|
48
48
|
# Data processing and COA
|
|
49
49
|
"pandas",
|
|
50
50
|
"more-itertools",
|
|
@@ -55,6 +55,9 @@ dependencies = [
|
|
|
55
55
|
"gmft",
|
|
56
56
|
# Development tools
|
|
57
57
|
"ruff>=0.13.3",
|
|
58
|
+
"torch",
|
|
59
|
+
"torchvision",
|
|
60
|
+
"fastmcp>=2.13.3",
|
|
58
61
|
]
|
|
59
62
|
|
|
60
63
|
[project.scripts]
|
|
@@ -205,10 +205,11 @@ def format_citation(extra):
|
|
|
205
205
|
parts.append(data["title"])
|
|
206
206
|
|
|
207
207
|
# Journal/venue
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
208
|
+
pl = data.get("primary_location") or {}
|
|
209
|
+
src = pl.get("source") or {}
|
|
210
|
+
venue = src.get("display_name", "")
|
|
211
|
+
if venue:
|
|
212
|
+
parts.append(f"*{venue}*")
|
|
212
213
|
|
|
213
214
|
return ". ".join(parts) if parts else None
|
|
214
215
|
except Exception:
|
|
@@ -1061,10 +1062,11 @@ def tab_openalex_search():
|
|
|
1061
1062
|
st.markdown(f"**Citations:** {citation_count}")
|
|
1062
1063
|
|
|
1063
1064
|
# Venue
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
)
|
|
1067
|
-
|
|
1065
|
+
pl = work.get("primary_location") or {}
|
|
1066
|
+
src = pl.get("source") or {}
|
|
1067
|
+
venue_name = src.get("display_name")
|
|
1068
|
+
if venue_name:
|
|
1069
|
+
st.markdown(f"**Venue:** {venue_name}")
|
|
1068
1070
|
|
|
1069
1071
|
# Abstract preview
|
|
1070
1072
|
if work.get("abstract_inverted_index"):
|
|
@@ -15,6 +15,11 @@ entry_type_by_crossref_type = {
|
|
|
15
15
|
"book-part": "inbook",
|
|
16
16
|
"other": "misc",
|
|
17
17
|
"paratext": "misc", # new line added
|
|
18
|
+
"erratum": "misc", # new line added
|
|
19
|
+
"letter": "article", # new line added
|
|
20
|
+
"editorial": "article", # new line added
|
|
21
|
+
"libguides": "misc", # new line added
|
|
22
|
+
"supplementary-materials": "misc", # new line added
|
|
18
23
|
"book": "book",
|
|
19
24
|
"journal-volume": "misc",
|
|
20
25
|
"book-set": "misc",
|
|
@@ -46,7 +51,8 @@ def dump_bibtex(work):
|
|
|
46
51
|
"Unable to generate a bibtex entry. This was probably an entry added by url or other non-doi source."
|
|
47
52
|
)
|
|
48
53
|
return None
|
|
49
|
-
|
|
54
|
+
# Try type_crossref first (for backwards compatibility), then fall back to type
|
|
55
|
+
wtype = work.get("type_crossref") or work.get("type")
|
|
50
56
|
if wtype is None:
|
|
51
57
|
print(f"Unable to generate a bibtex entry for {work.get('id')}")
|
|
52
58
|
return None
|
|
@@ -108,7 +114,7 @@ def _populate_entry(entry, work):
|
|
|
108
114
|
elif entry_type == "misc":
|
|
109
115
|
_set_if(entry, "author", _author(work))
|
|
110
116
|
_set_if(entry, "title", work.get("title"))
|
|
111
|
-
_set_if(entry, "howpublished", (work
|
|
117
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
112
118
|
_set_if(entry, "year", _year(work))
|
|
113
119
|
elif entry_type == "phdthesis":
|
|
114
120
|
_set_if(entry, "author", _author(work))
|
|
@@ -118,7 +124,7 @@ def _populate_entry(entry, work):
|
|
|
118
124
|
elif entry_type == "proceedings":
|
|
119
125
|
_set_if(entry, "editor", _author(work))
|
|
120
126
|
_set_if(entry, "title", work.get("title"))
|
|
121
|
-
_set_if(entry, "series",
|
|
127
|
+
_set_if(entry, "series", _primary_location_display_name(work))
|
|
122
128
|
_set_if(entry, "volume", _volume(work))
|
|
123
129
|
_set_if(entry, "publisher", _publisher(work))
|
|
124
130
|
_set_if(entry, "year", _year(work))
|
|
@@ -126,7 +132,7 @@ def _populate_entry(entry, work):
|
|
|
126
132
|
_set_if(entry, "author", _author(work))
|
|
127
133
|
_set_if(entry, "title", work.get("title"))
|
|
128
134
|
_set_if(entry, "year", _year(work))
|
|
129
|
-
_set_if(entry, "howpublished", (work
|
|
135
|
+
_set_if(entry, "howpublished", _primary_location_url(work))
|
|
130
136
|
|
|
131
137
|
|
|
132
138
|
def _set_if(entry, key, value):
|
|
@@ -139,7 +145,7 @@ def _year(work):
|
|
|
139
145
|
|
|
140
146
|
|
|
141
147
|
def _book_title(work):
|
|
142
|
-
return (work
|
|
148
|
+
return _primary_location_display_name(work)
|
|
143
149
|
|
|
144
150
|
|
|
145
151
|
def _school(work):
|
|
@@ -168,11 +174,20 @@ def _journal_name(work):
|
|
|
168
174
|
|
|
169
175
|
|
|
170
176
|
def _publisher(work):
|
|
171
|
-
|
|
177
|
+
pl = work.get("primary_location") or {}
|
|
178
|
+
src = pl.get("source") or {}
|
|
179
|
+
return src.get("host_organization_name")
|
|
172
180
|
|
|
173
181
|
|
|
174
|
-
def
|
|
175
|
-
|
|
182
|
+
def _primary_location_display_name(work):
|
|
183
|
+
pl = work.get("primary_location") or {}
|
|
184
|
+
src = pl.get("source") or {}
|
|
185
|
+
return src.get("display_name")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _primary_location_url(work):
|
|
189
|
+
pl = work.get("primary_location") or {}
|
|
190
|
+
return pl.get("landing_page_url") or pl.get("pdf_url")
|
|
176
191
|
|
|
177
192
|
|
|
178
193
|
def _volume(work):
|
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
"""LitGPT using litellm."""
|
|
2
2
|
|
|
3
|
+
import importlib
|
|
4
|
+
import logging
|
|
3
5
|
import os
|
|
6
|
+
import pydoc
|
|
7
|
+
import re
|
|
4
8
|
import readline
|
|
5
9
|
import subprocess
|
|
6
|
-
|
|
7
10
|
import warnings
|
|
8
11
|
|
|
9
12
|
import numpy as np
|
|
13
|
+
from docling.document_converter import DocumentConverter
|
|
14
|
+
from docling.exceptions import ConversionError
|
|
10
15
|
from litellm import completion
|
|
11
16
|
from rich import print as richprint
|
|
12
17
|
from sentence_transformers import SentenceTransformer
|
|
13
18
|
|
|
14
|
-
import pydoc
|
|
15
|
-
import re
|
|
16
|
-
import importlib
|
|
17
|
-
from docling.document_converter import DocumentConverter
|
|
18
|
-
from docling.exceptions import ConversionError
|
|
19
|
-
import logging
|
|
20
|
-
|
|
21
19
|
# Try to import backoff (optional dependency in crawl extra)
|
|
22
20
|
try:
|
|
23
21
|
import backoff
|
|
@@ -220,6 +218,10 @@ def get_completion(model, messages, max_tokens=None):
|
|
|
220
218
|
|
|
221
219
|
except KeyboardInterrupt:
|
|
222
220
|
pass
|
|
221
|
+
except AttributeError as e:
|
|
222
|
+
# Suppress litellm TranscriptionCreateParams.__annotations__ errors
|
|
223
|
+
if "__annotations__" not in str(e):
|
|
224
|
+
raise
|
|
223
225
|
|
|
224
226
|
return output
|
|
225
227
|
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
from more_itertools import batched
|
|
4
|
+
import datetime
|
|
5
|
+
from IPython import get_ipython
|
|
6
|
+
from IPython.display import HTML, display
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from nameparser import HumanName
|
|
9
|
+
import requests
|
|
10
|
+
import base64
|
|
11
|
+
from io import BytesIO
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_coa(orcid, email=None):
|
|
15
|
+
"""Generate Table 4 for the NSF COA.
|
|
16
|
+
ORCID: str the author orcid to retrieve results for.
|
|
17
|
+
email: str optional email for OpenAlex API (for polite pool)
|
|
18
|
+
"""
|
|
19
|
+
print(f"Starting COA generation for ORCID: {orcid}")
|
|
20
|
+
|
|
21
|
+
# First, find the correct author profile (handle duplicate ORCID profiles)
|
|
22
|
+
orcid = orcid.replace("https://orcid.org/", "")
|
|
23
|
+
orcid_url = f"https://orcid.org/{orcid}"
|
|
24
|
+
|
|
25
|
+
print(f"Finding author profile for ORCID {orcid}...")
|
|
26
|
+
authors_url = "https://api.openalex.org/authors"
|
|
27
|
+
author_params = {"filter": f"orcid:{orcid_url}"}
|
|
28
|
+
if email:
|
|
29
|
+
author_params["mailto"] = email
|
|
30
|
+
|
|
31
|
+
author_resp = requests.get(authors_url, params=author_params)
|
|
32
|
+
author_resp.raise_for_status()
|
|
33
|
+
author_data = author_resp.json()
|
|
34
|
+
|
|
35
|
+
if not author_data["results"]:
|
|
36
|
+
raise ValueError(f"No author found for ORCID {orcid_url}")
|
|
37
|
+
|
|
38
|
+
# If multiple profiles exist with same ORCID, use the one with most works
|
|
39
|
+
if len(author_data["results"]) > 1:
|
|
40
|
+
print(
|
|
41
|
+
f" Warning: Found {len(author_data['results'])} author profiles with this ORCID"
|
|
42
|
+
)
|
|
43
|
+
for a in author_data["results"]:
|
|
44
|
+
print(
|
|
45
|
+
f" - {a['display_name']}: {a['works_count']} works (ID: {a['id']})"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Select author with most works
|
|
49
|
+
author = max(author_data["results"], key=lambda x: x.get("works_count", 0))
|
|
50
|
+
author_id = author["id"].replace("https://openalex.org/", "")
|
|
51
|
+
print(
|
|
52
|
+
f" Using profile: {author['display_name']} ({author['works_count']} total works)"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
url = "https://api.openalex.org/works"
|
|
56
|
+
next_cursor = "*"
|
|
57
|
+
pubs = []
|
|
58
|
+
|
|
59
|
+
current_year = datetime.datetime.now().year
|
|
60
|
+
four_years_ago = current_year - 4
|
|
61
|
+
print(f"Fetching publications from {four_years_ago} to {current_year}...")
|
|
62
|
+
|
|
63
|
+
while next_cursor:
|
|
64
|
+
_filter = (
|
|
65
|
+
f"author.id:https://openalex.org/{author_id}"
|
|
66
|
+
f",publication_year:>{four_years_ago - 1}"
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
params = {
|
|
70
|
+
"filter": _filter,
|
|
71
|
+
"cursor": next_cursor,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
# Only include mailto if email is provided
|
|
75
|
+
if email:
|
|
76
|
+
params["mailto"] = email
|
|
77
|
+
|
|
78
|
+
r = requests.get(url, params=params)
|
|
79
|
+
r.raise_for_status() # Raise exception for bad status codes
|
|
80
|
+
data = r.json()
|
|
81
|
+
|
|
82
|
+
# Check if the response has the expected structure
|
|
83
|
+
if "results" not in data:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"Unexpected API response. Status: {r.status_code}, Response: {data}"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
pubs += data["results"]
|
|
89
|
+
next_cursor = data["meta"].get("next_cursor", None)
|
|
90
|
+
|
|
91
|
+
print(f"Found {len(pubs)} publications")
|
|
92
|
+
|
|
93
|
+
# Show publication years distribution
|
|
94
|
+
year_counts = {}
|
|
95
|
+
for pub in pubs:
|
|
96
|
+
year = pub.get("publication_year", "Unknown")
|
|
97
|
+
year_counts[year] = year_counts.get(year, 0) + 1
|
|
98
|
+
print("Publications by year:", dict(sorted(year_counts.items(), reverse=True)))
|
|
99
|
+
|
|
100
|
+
# We get all the authors from all the papers first.
|
|
101
|
+
print("Extracting authors from publications...")
|
|
102
|
+
authors = []
|
|
103
|
+
|
|
104
|
+
for pub in pubs:
|
|
105
|
+
year = int(pub.get("publication_year", -1))
|
|
106
|
+
last_active = datetime.datetime.strptime(
|
|
107
|
+
pub.get("publication_date", f"{year}-01-01"), "%Y-%m-%d"
|
|
108
|
+
).strftime("%m/%m/%Y")
|
|
109
|
+
|
|
110
|
+
aus = pub["authorships"]
|
|
111
|
+
for au in aus:
|
|
112
|
+
# Skip authors with no display name or no OpenAlex ID
|
|
113
|
+
display_name = au["author"].get("display_name")
|
|
114
|
+
if not display_name or not au["author"].get("id"):
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
hn = HumanName(display_name)
|
|
118
|
+
name = f"{hn.last}, {hn.first} {hn.middle or ''}"
|
|
119
|
+
|
|
120
|
+
authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
|
|
121
|
+
|
|
122
|
+
# sort authors alphabetically, then by year descending
|
|
123
|
+
authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
|
|
124
|
+
print(f"Found {len(authors)} total author entries (including duplicates)")
|
|
125
|
+
|
|
126
|
+
# Now, get all the affiliations using a two-strategy approach:
|
|
127
|
+
# 1. Use last_known_institutions (from most recent publication)
|
|
128
|
+
# 2. Fall back to affiliation with most recent year in affiliations array
|
|
129
|
+
oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
|
|
130
|
+
print(f"Found {len(oaids)} unique authors")
|
|
131
|
+
print("Fetching affiliation information from OpenAlex...")
|
|
132
|
+
affiliations = {}
|
|
133
|
+
batch_num = 0
|
|
134
|
+
for batch in batched(oaids, 50):
|
|
135
|
+
batch_num += 1
|
|
136
|
+
print(f" Fetching batch {batch_num} of authors...")
|
|
137
|
+
url = f"https://api.openalex.org/authors?filter=id:{'|'.join(batch)}"
|
|
138
|
+
|
|
139
|
+
params = {"per-page": 50}
|
|
140
|
+
|
|
141
|
+
# Only include mailto if email is provided
|
|
142
|
+
if email:
|
|
143
|
+
params["mailto"] = email
|
|
144
|
+
|
|
145
|
+
d = requests.get(url, params=params)
|
|
146
|
+
|
|
147
|
+
for au in d.json()["results"]:
|
|
148
|
+
affiliation_name = ""
|
|
149
|
+
|
|
150
|
+
# Strategy 1: Use last_known_institutions (most recent publication)
|
|
151
|
+
last_known = au.get("last_known_institutions", [])
|
|
152
|
+
if last_known and len(last_known) > 0:
|
|
153
|
+
affiliation_name = last_known[0].get("display_name", "")
|
|
154
|
+
|
|
155
|
+
# Strategy 2: If no last_known_institutions, use affiliation with most recent year
|
|
156
|
+
if not affiliation_name:
|
|
157
|
+
affils = au.get("affiliations", [])
|
|
158
|
+
if len(affils) > 0:
|
|
159
|
+
# Find affiliation with most recent year
|
|
160
|
+
best_affil = max(affils, key=lambda a: max(a.get("years", [0])))
|
|
161
|
+
affiliation_name = best_affil["institution"]["display_name"]
|
|
162
|
+
|
|
163
|
+
affiliations[au["id"]] = affiliation_name
|
|
164
|
+
|
|
165
|
+
print("Building unique author list...")
|
|
166
|
+
uniq = {}
|
|
167
|
+
uniq_authors = [] # by openalex id
|
|
168
|
+
all_authors = []
|
|
169
|
+
for name, year, last_active, oa_id, pub_id in authors:
|
|
170
|
+
if oa_id not in uniq:
|
|
171
|
+
uniq[oa_id] = 1
|
|
172
|
+
affil = affiliations.get(oa_id, "No affiliation known")
|
|
173
|
+
# now we build the tables
|
|
174
|
+
uniq_authors += [["A:", name, affil, "", last_active]]
|
|
175
|
+
all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
|
|
176
|
+
|
|
177
|
+
print(f"Creating Excel file with {len(uniq_authors)} unique authors...")
|
|
178
|
+
|
|
179
|
+
# unique authors
|
|
180
|
+
df = pd.DataFrame(
|
|
181
|
+
uniq_authors,
|
|
182
|
+
columns=[
|
|
183
|
+
"4",
|
|
184
|
+
"Name:",
|
|
185
|
+
"Organizational Affiliation",
|
|
186
|
+
"Optional (email, Department)",
|
|
187
|
+
"Last Active",
|
|
188
|
+
],
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
today = datetime.date.today().strftime("%Y-%m-%d.xlsx")
|
|
192
|
+
filename = f"{orcid}-{today}"
|
|
193
|
+
|
|
194
|
+
# Determine output path
|
|
195
|
+
if get_ipython():
|
|
196
|
+
# Jupyter: use in-memory buffer
|
|
197
|
+
buffer = BytesIO()
|
|
198
|
+
xw = pd.ExcelWriter(buffer, engine="xlsxwriter")
|
|
199
|
+
else:
|
|
200
|
+
# MCP/CLI: save to Downloads folder
|
|
201
|
+
from pathlib import Path
|
|
202
|
+
|
|
203
|
+
downloads_path = Path.home() / "Downloads" / filename
|
|
204
|
+
xw = pd.ExcelWriter(str(downloads_path), engine="xlsxwriter")
|
|
205
|
+
|
|
206
|
+
# Table 4
|
|
207
|
+
df.to_excel(xw, index=False, sheet_name="Table 4")
|
|
208
|
+
|
|
209
|
+
sheet = xw.sheets["Table 4"]
|
|
210
|
+
|
|
211
|
+
for column in df:
|
|
212
|
+
column_length = max(df[column].astype(str).map(len).max(), len(column))
|
|
213
|
+
col_idx = df.columns.get_loc(column)
|
|
214
|
+
sheet.set_column(col_idx, col_idx, column_length + 2)
|
|
215
|
+
|
|
216
|
+
# Save all authors for debugging
|
|
217
|
+
authors_df = pd.DataFrame(
|
|
218
|
+
all_authors,
|
|
219
|
+
columns=[
|
|
220
|
+
"4",
|
|
221
|
+
"Name:",
|
|
222
|
+
"Organizational Affiliation",
|
|
223
|
+
"Optional (email, Department)",
|
|
224
|
+
"Last Active",
|
|
225
|
+
"OpenAlex pub id",
|
|
226
|
+
"OpenAlex author id",
|
|
227
|
+
],
|
|
228
|
+
)
|
|
229
|
+
authors_df.to_excel(xw, index=False, sheet_name="all authors")
|
|
230
|
+
|
|
231
|
+
sheet = xw.sheets["all authors"]
|
|
232
|
+
for column in df:
|
|
233
|
+
column_length = max(df[column].astype(str).map(len).max(), len(column))
|
|
234
|
+
col_idx = df.columns.get_loc(column)
|
|
235
|
+
sheet.set_column(col_idx, col_idx, column_length + 2)
|
|
236
|
+
|
|
237
|
+
# Save the Excel file
|
|
238
|
+
xw.close()
|
|
239
|
+
|
|
240
|
+
if get_ipython():
|
|
241
|
+
# Jupyter: display download link
|
|
242
|
+
excel_bytes = buffer.getvalue()
|
|
243
|
+
b64 = base64.b64encode(excel_bytes).decode("utf-8")
|
|
244
|
+
uri = f'<pre>{filename}</pre><br><a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download COA</a>'
|
|
245
|
+
display(HTML(uri))
|
|
246
|
+
print("✓ COA generation complete!")
|
|
247
|
+
return None
|
|
248
|
+
else:
|
|
249
|
+
# Return path for MCP server / CLI
|
|
250
|
+
print(f"✓ COA saved to: {downloads_path}")
|
|
251
|
+
return str(downloads_path)
|
|
@@ -9,6 +9,7 @@ Commands:
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import sys
|
|
12
13
|
from difflib import SequenceMatcher
|
|
13
14
|
|
|
14
15
|
import click
|
|
@@ -24,7 +25,10 @@ from . import manage
|
|
|
24
25
|
@click.option("--references", is_flag=True)
|
|
25
26
|
@click.option("--related", is_flag=True)
|
|
26
27
|
@click.option("--citing", is_flag=True)
|
|
27
|
-
|
|
28
|
+
@click.option(
|
|
29
|
+
"-y", "--yes", is_flag=True, help="Auto-add all results (bypass selection prompt)."
|
|
30
|
+
)
|
|
31
|
+
def crossref(query, references, related, citing, yes):
|
|
28
32
|
"""Add entries to litdb from a crossref query."""
|
|
29
33
|
query = " ".join(query)
|
|
30
34
|
resp = requests.get("https://api.crossref.org/works", params={"query": query})
|
|
@@ -43,10 +47,24 @@ def crossref(query, references, related, citing):
|
|
|
43
47
|
f"{i}. {' '.join(title)}, {authors}, {source} ({year}), https://doi.org/{item['DOI']}."
|
|
44
48
|
)
|
|
45
49
|
|
|
46
|
-
|
|
50
|
+
# Handle selection based on yes flag or interactive mode
|
|
51
|
+
if yes:
|
|
52
|
+
# Add all results
|
|
53
|
+
toadd = list(range(len(data["message"]["items"])))
|
|
54
|
+
richprint("[green]Adding all results (--yes mode)[/green]")
|
|
55
|
+
elif not sys.stdin.isatty():
|
|
56
|
+
# Non-interactive mode: skip to avoid hanging
|
|
57
|
+
richprint("[yellow]Non-interactive mode, skipping selection.[/yellow]")
|
|
58
|
+
return
|
|
59
|
+
else:
|
|
60
|
+
# Interactive prompt
|
|
61
|
+
toadd = input("Enter space separated numbers to add, or return to quit. ")
|
|
62
|
+
if toadd:
|
|
63
|
+
toadd = [int(x) for x in toadd.split(" ")]
|
|
64
|
+
else:
|
|
65
|
+
return
|
|
47
66
|
|
|
48
67
|
if toadd:
|
|
49
|
-
toadd = [int(x) for x in toadd.split(" ")]
|
|
50
68
|
dois = [
|
|
51
69
|
"https://doi.org/" + data["message"]["items"][i]["DOI"] for i in toadd
|
|
52
70
|
]
|
|
@@ -58,6 +76,7 @@ def crossref(query, references, related, citing):
|
|
|
58
76
|
related=related,
|
|
59
77
|
references=references,
|
|
60
78
|
citing=citing,
|
|
79
|
+
yes=yes,
|
|
61
80
|
)
|
|
62
81
|
|
|
63
82
|
|
|
@@ -66,8 +85,11 @@ def crossref(query, references, related, citing):
|
|
|
66
85
|
@click.option("--references", is_flag=True, help="Add references too.")
|
|
67
86
|
@click.option("--related", is_flag=True, help="Add related too.")
|
|
68
87
|
@click.option("--citing", is_flag=True, help="Add citing too.")
|
|
88
|
+
@click.option(
|
|
89
|
+
"-y", "--yes", is_flag=True, help="Auto-confirm all matches (bypass prompts)."
|
|
90
|
+
)
|
|
69
91
|
@click.option("--model", default=None, help="LLM model to use for parsing.")
|
|
70
|
-
def fromtext(text, references, related, citing, model):
|
|
92
|
+
def fromtext(text, references, related, citing, yes, model):
|
|
71
93
|
"""Extract and add references from pasted TEXT.
|
|
72
94
|
|
|
73
95
|
TEXT should be a string containing academic references (from PDFs, websites, etc.).
|
|
@@ -238,6 +260,16 @@ Text:
|
|
|
238
260
|
" [green]High confidence match, adding automatically[/green]"
|
|
239
261
|
)
|
|
240
262
|
dois_to_add.append(best_doi)
|
|
263
|
+
# Auto-add if --yes flag is set
|
|
264
|
+
elif yes:
|
|
265
|
+
richprint(" [green]Adding (--yes mode)[/green]")
|
|
266
|
+
dois_to_add.append(best_doi)
|
|
267
|
+
# Non-interactive mode: skip to avoid hanging
|
|
268
|
+
elif not sys.stdin.isatty():
|
|
269
|
+
richprint(
|
|
270
|
+
" [yellow]Non-interactive mode, skipping low-confidence match[/yellow]"
|
|
271
|
+
)
|
|
272
|
+
# Interactive prompt
|
|
241
273
|
else:
|
|
242
274
|
confirm = input(" Add this? [y/N]: ")
|
|
243
275
|
if confirm.lower().startswith("y"):
|
|
@@ -262,6 +294,7 @@ Text:
|
|
|
262
294
|
references=references,
|
|
263
295
|
related=related,
|
|
264
296
|
citing=citing,
|
|
297
|
+
yes=yes,
|
|
265
298
|
)
|
|
266
299
|
richprint("[green]✓ Done![/green]")
|
|
267
300
|
else:
|