litdb 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litdb/__init__.py +1 -0
- litdb/bibtex.py +182 -0
- litdb/cli.py +1510 -0
- litdb/coa.py +152 -0
- litdb/db.py +362 -0
- litdb/gpt.py +150 -0
- litdb/lab.py +23 -0
- litdb/openalex.py +81 -0
- litdb/pdf.py +19 -0
- litdb/utils.py +64 -0
- litdb-2.0.0.dist-info/METADATA +804 -0
- litdb-2.0.0.dist-info/RECORD +15 -0
- litdb-2.0.0.dist-info/WHEEL +5 -0
- litdb-2.0.0.dist-info/entry_points.txt +2 -0
- litdb-2.0.0.dist-info/top_level.txt +1 -0
litdb/coa.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
from more_itertools import batched
|
|
4
|
+
import datetime
|
|
5
|
+
from IPython import get_ipython
|
|
6
|
+
from IPython.display import HTML, display
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from nameparser import HumanName
|
|
9
|
+
import requests
|
|
10
|
+
import base64
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_coa(orcid):
|
|
14
|
+
"""Generate Table 4 for the NSF COA.
|
|
15
|
+
ORCID: str the author orcid to retrieve results for.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
url = "https://api.openalex.org/works"
|
|
19
|
+
|
|
20
|
+
next_cursor = "*"
|
|
21
|
+
|
|
22
|
+
pubs = []
|
|
23
|
+
|
|
24
|
+
current_year = datetime.datetime.now().year
|
|
25
|
+
four_years_ago = current_year - 4
|
|
26
|
+
|
|
27
|
+
orcid = orcid.replace("https://orcid.org/", "")
|
|
28
|
+
|
|
29
|
+
while next_cursor:
|
|
30
|
+
_filter = (
|
|
31
|
+
f"author.orcid:https://orcid.org/{orcid}"
|
|
32
|
+
f",publication_year:>{four_years_ago - 1}"
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
r = requests.get(
|
|
36
|
+
url,
|
|
37
|
+
params={
|
|
38
|
+
"filter": _filter,
|
|
39
|
+
"email": "jkitchin@andrew.cmu.edu",
|
|
40
|
+
"cursor": next_cursor,
|
|
41
|
+
},
|
|
42
|
+
)
|
|
43
|
+
data = r.json()
|
|
44
|
+
pubs += data["results"]
|
|
45
|
+
next_cursor = data["meta"].get("next_cursor", None)
|
|
46
|
+
|
|
47
|
+
# We get all the authors from all the papers first.
|
|
48
|
+
authors = []
|
|
49
|
+
|
|
50
|
+
for pub in pubs:
|
|
51
|
+
year = int(pub.get("publication_year", -1))
|
|
52
|
+
last_active = datetime.datetime.strptime(
|
|
53
|
+
pub.get("publication_date", f"{year}-01-01"), "%Y-%m-%d"
|
|
54
|
+
).strftime("%m/%m/%Y")
|
|
55
|
+
|
|
56
|
+
aus = pub["authorships"]
|
|
57
|
+
for au in aus:
|
|
58
|
+
hn = HumanName(au["author"]["display_name"])
|
|
59
|
+
name = f'{hn.last}, {hn.first} {hn.middle or ""}'
|
|
60
|
+
|
|
61
|
+
authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
|
|
62
|
+
|
|
63
|
+
# sort authors alphabetically, then by year descending
|
|
64
|
+
authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
|
|
65
|
+
|
|
66
|
+
# Now, get all the affiliations. This assumes the first one is most recent.
|
|
67
|
+
# I could also use the last known institution, but this is sometimes empty
|
|
68
|
+
# too.
|
|
69
|
+
oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
|
|
70
|
+
affiliations = {}
|
|
71
|
+
for batch in batched(oaids, 50):
|
|
72
|
+
url = f'https://api.openalex.org/authors?filter=id:{"|".join(batch)}'
|
|
73
|
+
|
|
74
|
+
params = {"per-page": 50, "email": "jkitchin@andrew.cmu.edu"}
|
|
75
|
+
|
|
76
|
+
d = requests.get(url, params=params)
|
|
77
|
+
|
|
78
|
+
for au in d.json()["results"]:
|
|
79
|
+
affils = au["affiliations"]
|
|
80
|
+
if len(affils) > 0:
|
|
81
|
+
affiliations[au["id"]] = affils[0]["institution"]["display_name"]
|
|
82
|
+
else:
|
|
83
|
+
affiliations[au["id"]] = ""
|
|
84
|
+
|
|
85
|
+
uniq = {}
|
|
86
|
+
uniq_authors = [] # by openalex id
|
|
87
|
+
all_authors = []
|
|
88
|
+
for name, year, last_active, oa_id, pub_id in authors:
|
|
89
|
+
if oa_id not in uniq:
|
|
90
|
+
uniq[oa_id] = 1
|
|
91
|
+
affil = affiliations.get(oa_id, "No affiliation known")
|
|
92
|
+
# now we build the tables
|
|
93
|
+
uniq_authors += [["A:", name, affil, "", last_active]]
|
|
94
|
+
all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
|
|
95
|
+
|
|
96
|
+
# unique authors
|
|
97
|
+
df = pd.DataFrame(
|
|
98
|
+
uniq_authors,
|
|
99
|
+
columns=[
|
|
100
|
+
"4",
|
|
101
|
+
"Name:",
|
|
102
|
+
"Organizational Affiliation",
|
|
103
|
+
"Optional (email, Department)",
|
|
104
|
+
"Last Active",
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
today = datetime.date.today().strftime("%Y-%m-%d.xlsx")
|
|
109
|
+
coa_file = f"{orcid}-{today}"
|
|
110
|
+
|
|
111
|
+
# Table 4
|
|
112
|
+
xw = pd.ExcelWriter(coa_file, engine="xlsxwriter")
|
|
113
|
+
df.to_excel(xw, index=False, sheet_name="Table 4")
|
|
114
|
+
|
|
115
|
+
sheet = xw.sheets["Table 4"]
|
|
116
|
+
|
|
117
|
+
for column in df:
|
|
118
|
+
column_length = max(df[column].astype(str).map(len).max(), len(column))
|
|
119
|
+
col_idx = df.columns.get_loc(column)
|
|
120
|
+
sheet.set_column(col_idx, col_idx, column_length + 2)
|
|
121
|
+
|
|
122
|
+
# Save all authors for debugging
|
|
123
|
+
authors_df = pd.DataFrame(
|
|
124
|
+
all_authors,
|
|
125
|
+
columns=[
|
|
126
|
+
"4",
|
|
127
|
+
"Name:",
|
|
128
|
+
"Organizational Affiliation",
|
|
129
|
+
"Optional (email, Department)",
|
|
130
|
+
"Last Active",
|
|
131
|
+
"OpenAlex pub id",
|
|
132
|
+
"OpenAlex author id",
|
|
133
|
+
],
|
|
134
|
+
)
|
|
135
|
+
authors_df.to_excel(xw, index=False, sheet_name="all authors")
|
|
136
|
+
|
|
137
|
+
sheet = xw.sheets["all authors"]
|
|
138
|
+
for column in df:
|
|
139
|
+
column_length = max(df[column].astype(str).map(len).max(), len(column))
|
|
140
|
+
col_idx = df.columns.get_loc(column)
|
|
141
|
+
sheet.set_column(col_idx, col_idx, column_length + 2)
|
|
142
|
+
|
|
143
|
+
# Save the Excel file
|
|
144
|
+
xw.close()
|
|
145
|
+
|
|
146
|
+
if get_ipython():
|
|
147
|
+
with open(coa_file, "rb") as f:
|
|
148
|
+
b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
149
|
+
uri = f'<pre>{coa_file}</pre><br><a href="data:text/plain;base64,{b64}" download="{coa_file}">Download COA</a>'
|
|
150
|
+
display(HTML(uri))
|
|
151
|
+
else:
|
|
152
|
+
print(f"Created {coa_file}")
|
litdb/db.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""Setup and add things to the database."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import libsql_experimental as libsql
|
|
8
|
+
|
|
9
|
+
from .utils import get_config
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from sentence_transformers import SentenceTransformer
|
|
13
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
14
|
+
from tqdm import tqdm
|
|
15
|
+
import datetime
|
|
16
|
+
import requests
|
|
17
|
+
import bibtexparser
|
|
18
|
+
from rich import print
|
|
19
|
+
|
|
20
|
+
from litdb.openalex import get_data, get_text
|
|
21
|
+
from litdb.bibtex import dump_bibtex
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_db():
|
|
25
|
+
"""Get or create the database."""
|
|
26
|
+
|
|
27
|
+
config = get_config()
|
|
28
|
+
|
|
29
|
+
root = Path(config["root"])
|
|
30
|
+
|
|
31
|
+
DB = str(root / "litdb.libsql")
|
|
32
|
+
|
|
33
|
+
if os.path.exists(DB):
|
|
34
|
+
db = libsql.connect(DB)
|
|
35
|
+
db.execute("PRAGMA foreign_keys = ON")
|
|
36
|
+
db.execute("PRAGMA journal_mode=WAL")
|
|
37
|
+
return db
|
|
38
|
+
else:
|
|
39
|
+
db = libsql.connect(DB)
|
|
40
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
41
|
+
|
|
42
|
+
_, dim = model.encode(["test"]).shape
|
|
43
|
+
|
|
44
|
+
db.execute(
|
|
45
|
+
f"""create table if not exists
|
|
46
|
+
sources(rowid integer primary key,
|
|
47
|
+
source text unique,
|
|
48
|
+
text text,
|
|
49
|
+
extra text,
|
|
50
|
+
embedding F32_BLOB({dim}),
|
|
51
|
+
date_added text)"""
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
db.execute(
|
|
55
|
+
"""create virtual table if not exists fulltext using fts5(source, text)"""
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
db.execute(
|
|
59
|
+
"""create table if not exists
|
|
60
|
+
tags(rowid integer primary key,
|
|
61
|
+
tag text unique)"""
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
db.execute(
|
|
65
|
+
"""create table if not exists
|
|
66
|
+
source_tag(rowid integer primary key,
|
|
67
|
+
source_id integer,
|
|
68
|
+
tag_id integer,
|
|
69
|
+
foreign key(source_id) references sources(rowid) on delete cascade,
|
|
70
|
+
foreign key(tag_id) references tags(rowid) on delete cascade)"""
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
db.execute(
|
|
74
|
+
"""create table if not exists
|
|
75
|
+
queries(rowid integer primary key,
|
|
76
|
+
filter text unique,
|
|
77
|
+
description text,
|
|
78
|
+
last_updated text)"""
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
db.execute(
|
|
82
|
+
"""create table if not exists
|
|
83
|
+
directories(rowid integer primary key,
|
|
84
|
+
path text unique,
|
|
85
|
+
last_updated text)"""
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
db.execute(
|
|
89
|
+
"""create index if not exists embedding_idx ON sources (libsql_vector_idx(embedding))"""
|
|
90
|
+
)
|
|
91
|
+
return db
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def add_source(source, text, extra=None):
|
|
95
|
+
"""Add a row to the sources table.
|
|
96
|
+
|
|
97
|
+
source : url or path to document
|
|
98
|
+
text: string of the document contents
|
|
99
|
+
extra: jsonable information, usually a dictionary.
|
|
100
|
+
|
|
101
|
+
We generate a document level embedding by averaging the embeddings of the
|
|
102
|
+
document chunks.
|
|
103
|
+
"""
|
|
104
|
+
db = get_db()
|
|
105
|
+
config = get_config()
|
|
106
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
107
|
+
splitter = RecursiveCharacterTextSplitter(
|
|
108
|
+
chunk_size=config["embedding"]["chunk_size"],
|
|
109
|
+
chunk_overlap=config["embedding"]["chunk_overlap"],
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
chunks = splitter.split_text(text)
|
|
113
|
+
embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
|
|
114
|
+
|
|
115
|
+
c = db.execute(
|
|
116
|
+
"""insert or ignore into sources(source, text, extra, embedding, date_added) values (?, ?, ?, ?, ?)""",
|
|
117
|
+
(
|
|
118
|
+
source,
|
|
119
|
+
text,
|
|
120
|
+
json.dumps(extra),
|
|
121
|
+
embedding,
|
|
122
|
+
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
123
|
+
),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# I think we should only do this if the previous line succeeded
|
|
127
|
+
if c.rowcount > 0:
|
|
128
|
+
db.execute(
|
|
129
|
+
"""insert into fulltext(source, text) values (?, ?)""", (source, text)
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
db.commit()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def get_citation(doi):
|
|
136
|
+
"""Get a citation string for doi."""
|
|
137
|
+
config = get_config()
|
|
138
|
+
citeas = "https://api.citeas.org/product/"
|
|
139
|
+
cp = {"email": config["openalex"]["email"]}
|
|
140
|
+
|
|
141
|
+
resp = requests.get(citeas + doi, cp)
|
|
142
|
+
if resp.status_code == 200:
|
|
143
|
+
cdata = resp.json()
|
|
144
|
+
citations = cdata.get("citations", [])
|
|
145
|
+
return citations[0]["citation"]
|
|
146
|
+
else:
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def add_work(workid, references=False, citing=False, related=False):
|
|
151
|
+
"""Add a single work to litdb.
|
|
152
|
+
|
|
153
|
+
workid is the doi (full url), or it could be an OpenAlex workid. DOI is
|
|
154
|
+
preferrable.
|
|
155
|
+
|
|
156
|
+
if references is truthy, also add them.
|
|
157
|
+
if citing is truthy, also add them.
|
|
158
|
+
if related is truthy, also add them.
|
|
159
|
+
|
|
160
|
+
"""
|
|
161
|
+
config = get_config()
|
|
162
|
+
params = {"email": config["openalex"]["email"], "per_page": 200}
|
|
163
|
+
if config["openalex"].get("api_key"):
|
|
164
|
+
params.update(api_key=config["openalex"].get("api_key"))
|
|
165
|
+
|
|
166
|
+
data = get_data("https://api.openalex.org/works/" + workid, params)
|
|
167
|
+
|
|
168
|
+
# I check to make sure we got something
|
|
169
|
+
source = data.get("id", None)
|
|
170
|
+
if source is None:
|
|
171
|
+
# I guess this could happen for a bad DOI.
|
|
172
|
+
print(f"No id found for {workid}.\n{data}")
|
|
173
|
+
return
|
|
174
|
+
|
|
175
|
+
data["citation"] = get_citation(workid)
|
|
176
|
+
data["bibtex"] = dump_bibtex(data)
|
|
177
|
+
|
|
178
|
+
add_source(workid, get_text(data), data)
|
|
179
|
+
|
|
180
|
+
if references:
|
|
181
|
+
for wid in tqdm(data["referenced_works"]):
|
|
182
|
+
rdata = get_data("https://api.openalex.org/works/" + wid, params)
|
|
183
|
+
source = rdata.get("doi") or rdata.get("id")
|
|
184
|
+
if source is None:
|
|
185
|
+
print(f"Something failed for {wid}. continuing")
|
|
186
|
+
continue
|
|
187
|
+
text = get_text(rdata)
|
|
188
|
+
rdata["citation"] = get_citation(source)
|
|
189
|
+
rdata["bibtex"] = dump_bibtex(rdata)
|
|
190
|
+
add_source(source, text, rdata)
|
|
191
|
+
|
|
192
|
+
if related:
|
|
193
|
+
for wid in tqdm(data["related_works"]):
|
|
194
|
+
rdata = get_data("https://api.openalex.org/works/" + wid, params)
|
|
195
|
+
source = rdata.get("doi") or rdata.get("id")
|
|
196
|
+
if source is None:
|
|
197
|
+
print(f"Something failed for {wid}. continuing")
|
|
198
|
+
continue
|
|
199
|
+
text = get_text(rdata)
|
|
200
|
+
rdata["citation"] = get_citation(source)
|
|
201
|
+
rdata["bibtex"] = dump_bibtex(rdata)
|
|
202
|
+
add_source(source, text, rdata)
|
|
203
|
+
|
|
204
|
+
if citing:
|
|
205
|
+
CURL = data["cited_by_api_url"]
|
|
206
|
+
next_cursor = "*"
|
|
207
|
+
params.update(cursor=next_cursor)
|
|
208
|
+
while next_cursor:
|
|
209
|
+
cdata = get_data(CURL, params)
|
|
210
|
+
next_cursor = cdata["meta"]["next_cursor"]
|
|
211
|
+
params.update(cursor=next_cursor)
|
|
212
|
+
if next_cursor is None:
|
|
213
|
+
break
|
|
214
|
+
|
|
215
|
+
# TODO: should the max citations to trigger this be configurable in litdb.toml
|
|
216
|
+
trigger = config["openalex"].get("citation_count_trigger", 100)
|
|
217
|
+
count = cdata["meta"]["count"]
|
|
218
|
+
if count > trigger:
|
|
219
|
+
r = input(
|
|
220
|
+
f"Found {count} citations. Do you want to download them all? (n/y): "
|
|
221
|
+
)
|
|
222
|
+
if r.lower().startswith("n"):
|
|
223
|
+
break
|
|
224
|
+
else:
|
|
225
|
+
pass
|
|
226
|
+
|
|
227
|
+
for work in tqdm(cdata["results"]):
|
|
228
|
+
source = work.get("doi") or work["id"]
|
|
229
|
+
text = get_text(work)
|
|
230
|
+
work["citation"] = get_citation(source)
|
|
231
|
+
work["bibtex"] = dump_bibtex(work)
|
|
232
|
+
add_source(source, text, work)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def add_author(oaid):
|
|
236
|
+
"""Add all works from an author.
|
|
237
|
+
"id": "https://openalex.org/A5003442464",
|
|
238
|
+
"orcid": "https://orcid.org/0000-0003-2625-9232",
|
|
239
|
+
"""
|
|
240
|
+
config = get_config()
|
|
241
|
+
aurl = "https://api.openalex.org/authors/" + oaid
|
|
242
|
+
|
|
243
|
+
params = {
|
|
244
|
+
"email": config["openalex"]["email"],
|
|
245
|
+
"per_page": 200,
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
if config["openalex"].get("api_key"):
|
|
249
|
+
params.update(api_key=config["openalex"].get("api_key"))
|
|
250
|
+
|
|
251
|
+
data = get_data(aurl, params)
|
|
252
|
+
|
|
253
|
+
wurl = data["works_api_url"]
|
|
254
|
+
next_cursor = "*"
|
|
255
|
+
params.update(cursor=next_cursor)
|
|
256
|
+
while next_cursor:
|
|
257
|
+
wdata = get_data(wurl, params)
|
|
258
|
+
next_cursor = wdata["meta"]["next_cursor"]
|
|
259
|
+
params.update(cursor=next_cursor)
|
|
260
|
+
for work in tqdm(wdata["results"]):
|
|
261
|
+
add_work(work.get("doi") or work["id"])
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def update_filter(f, last_updated=None, silent=False):
|
|
265
|
+
"""Update one filter (f). f should be a string that goes in the filter=
|
|
266
|
+
param for openalex.
|
|
267
|
+
|
|
268
|
+
last_updated should be a string %Y-%m-%d
|
|
269
|
+
|
|
270
|
+
If last_updated is None, we only go back one year.
|
|
271
|
+
That might still lead to a lot of hits.
|
|
272
|
+
|
|
273
|
+
"""
|
|
274
|
+
config = get_config()
|
|
275
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
276
|
+
splitter = RecursiveCharacterTextSplitter(
|
|
277
|
+
chunk_size=config["embedding"]["chunk_size"],
|
|
278
|
+
chunk_overlap=config["embedding"]["chunk_overlap"],
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
db = get_db()
|
|
282
|
+
|
|
283
|
+
_filter = f # make a copy because we modify it later.
|
|
284
|
+
|
|
285
|
+
wurl = "https://api.openalex.org/works"
|
|
286
|
+
|
|
287
|
+
# If last_updated is None, we limit to the past year
|
|
288
|
+
if last_updated is None:
|
|
289
|
+
now = datetime.date.today()
|
|
290
|
+
last_updated = (now - datetime.timedelta(days=365)).strftime("%Y-%m-%d")
|
|
291
|
+
|
|
292
|
+
_filter += f",from_created_date:{last_updated}"
|
|
293
|
+
|
|
294
|
+
params = {
|
|
295
|
+
"email": config["openalex"]["email"],
|
|
296
|
+
"api_key": config["openalex"]["api_key"],
|
|
297
|
+
"per_page": 200,
|
|
298
|
+
"filter": _filter,
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
next_cursor = "*"
|
|
302
|
+
params.update(cursor=next_cursor)
|
|
303
|
+
|
|
304
|
+
results = []
|
|
305
|
+
while next_cursor:
|
|
306
|
+
data = get_data(wurl, params)
|
|
307
|
+
next_cursor = data["meta"]["next_cursor"]
|
|
308
|
+
params.update(cursor=next_cursor)
|
|
309
|
+
|
|
310
|
+
for work in tqdm(data["results"], disable=silent):
|
|
311
|
+
source = work.get("doi") or work.get("id")
|
|
312
|
+
citation = get_citation(source)
|
|
313
|
+
work["citation"] = citation
|
|
314
|
+
|
|
315
|
+
bibtex = dump_bibtex(work)
|
|
316
|
+
work["bibtex"] = bibtex
|
|
317
|
+
|
|
318
|
+
text = get_text(work)
|
|
319
|
+
|
|
320
|
+
chunks = splitter.split_text(text)
|
|
321
|
+
embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
|
|
322
|
+
|
|
323
|
+
results += [[source, text, work]]
|
|
324
|
+
|
|
325
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
326
|
+
c = db.execute(
|
|
327
|
+
"""insert or ignore into
|
|
328
|
+
sources(source, text, extra, embedding, date_added)
|
|
329
|
+
values(?, ?, ?, ?, ?)""",
|
|
330
|
+
(source, text, json.dumps(work), embedding, now),
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
if c.rowcount > 0:
|
|
334
|
+
db.execute(
|
|
335
|
+
"""insert into fulltext(source, text) values (?, ?)""",
|
|
336
|
+
(source, text),
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
db.execute(
|
|
340
|
+
"""update queries set last_updated = ? where filter = ?""",
|
|
341
|
+
(datetime.date.today().strftime("%Y-%m-%d"), f),
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
db.commit()
|
|
345
|
+
return results
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def add_bibtex(bibfile):
|
|
349
|
+
"""Add entries with a DOI from bibfile."""
|
|
350
|
+
|
|
351
|
+
with open(bibfile, "r", encoding="utf-8", errors="replace") as bibfile:
|
|
352
|
+
bib_database = bibtexparser.load(bibfile)
|
|
353
|
+
|
|
354
|
+
for entry in tqdm(bib_database.entries):
|
|
355
|
+
if "doi" in entry:
|
|
356
|
+
doi = entry["doi"]
|
|
357
|
+
if doi.startswith("http"):
|
|
358
|
+
add_work(doi)
|
|
359
|
+
elif doi.startswith("10"):
|
|
360
|
+
add_work(f"https://doi.org/{doi}")
|
|
361
|
+
else:
|
|
362
|
+
print(f'I do not know what to do with "{doi}"')
|
litdb/gpt.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""The gpt command for litdb."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import readline
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import ollama
|
|
9
|
+
from rich import print as richprint
|
|
10
|
+
from sentence_transformers import SentenceTransformer
|
|
11
|
+
|
|
12
|
+
from .utils import get_config
|
|
13
|
+
from .db import get_db
|
|
14
|
+
|
|
15
|
+
# Enable command history
|
|
16
|
+
readline.parse_and_bind("tab: complete")
|
|
17
|
+
readline.parse_and_bind("set editing-mode emacs")
|
|
18
|
+
|
|
19
|
+
# Disable parallelism in tokenizers
|
|
20
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def gpt():
|
|
24
|
+
"""Start a LitGPT chat session. The prompt is interactive.
|
|
25
|
+
|
|
26
|
+
If the prompt starts with > the rest of the prompt will be run as a shell
|
|
27
|
+
command. Use this to add references, citations, etc, during the chat.
|
|
28
|
+
|
|
29
|
+
If the prompt starts with < the rest of the prompt will be run as a shell
|
|
30
|
+
command and the output used for RAG.
|
|
31
|
+
|
|
32
|
+
!save it will save the chat to a file.
|
|
33
|
+
!restart will reset the messages and restart the chat
|
|
34
|
+
!messages will output the current chat history to the shell
|
|
35
|
+
!help to print a help message
|
|
36
|
+
|
|
37
|
+
Use Ctrl-d to exit.
|
|
38
|
+
"""
|
|
39
|
+
config = get_config()
|
|
40
|
+
db = get_db()
|
|
41
|
+
model = SentenceTransformer(config["embedding"]["model"])
|
|
42
|
+
|
|
43
|
+
gpt = config.get("gpt", {"model": "llama2"})
|
|
44
|
+
gpt_model = gpt["model"]
|
|
45
|
+
|
|
46
|
+
messages = []
|
|
47
|
+
|
|
48
|
+
while prompt := input("LitGPT (Ctrl-d to quit)> "):
|
|
49
|
+
rag_content = ""
|
|
50
|
+
|
|
51
|
+
if prompt == "!help":
|
|
52
|
+
print("""\
|
|
53
|
+
If the prompt starts with >, run the rest as a shell command, e.g.
|
|
54
|
+
> litdb add doi
|
|
55
|
+
then continue to a new prompt.
|
|
56
|
+
|
|
57
|
+
If the prompt starts with < run the shell command but capture the output as
|
|
58
|
+
context for the next prompt.
|
|
59
|
+
|
|
60
|
+
The following subcommands can be used:
|
|
61
|
+
|
|
62
|
+
!save to save the chat to a file
|
|
63
|
+
!restart to reset the chat
|
|
64
|
+
!help for this message
|
|
65
|
+
""")
|
|
66
|
+
continue
|
|
67
|
+
|
|
68
|
+
elif prompt.startswith(">"):
|
|
69
|
+
# This means run the rest of the prompt as a shell command
|
|
70
|
+
# > litdb add some-id
|
|
71
|
+
os.system(prompt[1:].strip())
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
# a little sub-language of commands
|
|
75
|
+
elif prompt == "!save":
|
|
76
|
+
with open(input("Filename (chat.txt): ") or "chat.txt", "w") as f:
|
|
77
|
+
for message in messages:
|
|
78
|
+
f.write(f'{message["role"]}: {message["content"]}\n\n')
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
elif prompt == "!restart":
|
|
82
|
+
messages = []
|
|
83
|
+
print("Reset the chat.")
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
elif prompt == "!messages":
|
|
87
|
+
for message in messages:
|
|
88
|
+
richprint(f'{message["role"]}: {message["content"]}\n\n')
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
# Run shell command to get rag content
|
|
92
|
+
elif prompt.startswith("<"):
|
|
93
|
+
# Some command that outputs text to stdout
|
|
94
|
+
result = subprocess.run(
|
|
95
|
+
prompt[1:].strip(), shell=True, text=True, capture_output=True
|
|
96
|
+
)
|
|
97
|
+
rag_content = f"{prompt[1:]}\n\n" + result.stdout
|
|
98
|
+
prompt = input("LitGPT (Ctrl-d to quit)> ")
|
|
99
|
+
|
|
100
|
+
data = None
|
|
101
|
+
if not rag_content:
|
|
102
|
+
# RAG by vector search
|
|
103
|
+
emb = model.encode([prompt]).astype(np.float32).tobytes()
|
|
104
|
+
data = db.execute(
|
|
105
|
+
"""select sources.text, json_extract(sources.extra, '$.citation')
|
|
106
|
+
from vector_top_k('embedding_idx', ?, 3)
|
|
107
|
+
join sources on sources.rowid = id""",
|
|
108
|
+
(emb,),
|
|
109
|
+
).fetchall()
|
|
110
|
+
|
|
111
|
+
for doc, citation in data:
|
|
112
|
+
rag_content += f"\n\n{doc}"
|
|
113
|
+
|
|
114
|
+
messages += [
|
|
115
|
+
{
|
|
116
|
+
"role": "system",
|
|
117
|
+
"content": (
|
|
118
|
+
"Only use the following information and previously"
|
|
119
|
+
" provided information to respond"
|
|
120
|
+
" to the prompt. Do not use anything else:"
|
|
121
|
+
f" {rag_content}"
|
|
122
|
+
),
|
|
123
|
+
}
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
# I think we need to send this before we can use it for the user.
|
|
127
|
+
response = ollama.chat(model=gpt_model, messages=messages, stream=True)
|
|
128
|
+
for chunk in response:
|
|
129
|
+
richprint(chunk["message"]["content"], end="", flush=True)
|
|
130
|
+
|
|
131
|
+
messages += [{"role": "user", "content": prompt}]
|
|
132
|
+
|
|
133
|
+
output = ""
|
|
134
|
+
# This lets you Ctrl-c to stop streaming if it has gone way off.
|
|
135
|
+
try:
|
|
136
|
+
response = ollama.chat(model=gpt_model, messages=messages, stream=True)
|
|
137
|
+
for chunk in response:
|
|
138
|
+
output += chunk["message"]["content"]
|
|
139
|
+
richprint(chunk["message"]["content"], end="", flush=True)
|
|
140
|
+
except KeyboardInterrupt:
|
|
141
|
+
response.close()
|
|
142
|
+
richprint()
|
|
143
|
+
|
|
144
|
+
messages += [{"role": "assistant", "content": output}]
|
|
145
|
+
|
|
146
|
+
# We don't always have data here, if you use your own rag data
|
|
147
|
+
if data:
|
|
148
|
+
richprint("\nThe text was generated using these references:\n")
|
|
149
|
+
for i, (text, citation) in enumerate(data, 1):
|
|
150
|
+
richprint(f"{i:2d}. {citation}\n")
|
litdb/lab.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Jupyter Lab interface for litdb."""
|
|
2
|
+
|
|
3
|
+
import shlex
|
|
4
|
+
from IPython.core.magic import Magics, magics_class, line_cell_magic
|
|
5
|
+
from IPython import get_ipython
|
|
6
|
+
from .cli import cli
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@magics_class
|
|
10
|
+
class LitdbMagics(Magics):
|
|
11
|
+
@line_cell_magic
|
|
12
|
+
def litdb(self, line, cell=None):
|
|
13
|
+
"""Main litdb magic command using Click."""
|
|
14
|
+
args = shlex.split(line)
|
|
15
|
+
if cell is not None:
|
|
16
|
+
args += [cell]
|
|
17
|
+
cli.main(args=args, standalone_mode=False)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# Register the magic command
|
|
21
|
+
ip = get_ipython()
|
|
22
|
+
if ip:
|
|
23
|
+
ip.register_magics(LitdbMagics)
|