litdb 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
litdb/coa.py ADDED
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env python
2
+
3
+ from more_itertools import batched
4
+ import datetime
5
+ from IPython import get_ipython
6
+ from IPython.display import HTML, display
7
+ import pandas as pd
8
+ from nameparser import HumanName
9
+ import requests
10
+ import base64
11
+
12
+
13
+ def get_coa(orcid):
14
+ """Generate Table 4 for the NSF COA.
15
+ ORCID: str the author orcid to retrieve results for.
16
+ """
17
+
18
+ url = "https://api.openalex.org/works"
19
+
20
+ next_cursor = "*"
21
+
22
+ pubs = []
23
+
24
+ current_year = datetime.datetime.now().year
25
+ four_years_ago = current_year - 4
26
+
27
+ orcid = orcid.replace("https://orcid.org/", "")
28
+
29
+ while next_cursor:
30
+ _filter = (
31
+ f"author.orcid:https://orcid.org/{orcid}"
32
+ f",publication_year:>{four_years_ago - 1}"
33
+ )
34
+
35
+ r = requests.get(
36
+ url,
37
+ params={
38
+ "filter": _filter,
39
+ "email": "jkitchin@andrew.cmu.edu",
40
+ "cursor": next_cursor,
41
+ },
42
+ )
43
+ data = r.json()
44
+ pubs += data["results"]
45
+ next_cursor = data["meta"].get("next_cursor", None)
46
+
47
+ # We get all the authors from all the papers first.
48
+ authors = []
49
+
50
+ for pub in pubs:
51
+ year = int(pub.get("publication_year", -1))
52
+ last_active = datetime.datetime.strptime(
53
+ pub.get("publication_date", f"{year}-01-01"), "%Y-%m-%d"
54
+ ).strftime("%m/%m/%Y")
55
+
56
+ aus = pub["authorships"]
57
+ for au in aus:
58
+ hn = HumanName(au["author"]["display_name"])
59
+ name = f'{hn.last}, {hn.first} {hn.middle or ""}'
60
+
61
+ authors += [[name, year, last_active, au["author"]["id"], pub["id"]]]
62
+
63
+ # sort authors alphabetically, then by year descending
64
+ authors = sorted(authors, key=lambda row: (row[0].lower(), -row[1]))
65
+
66
+ # Now, get all the affiliations. This assumes the first one is most recent.
67
+ # I could also use the last known institution, but this is sometimes empty
68
+ # too.
69
+ oaids = set([row[3].replace("https://openalex.org/", "") for row in authors])
70
+ affiliations = {}
71
+ for batch in batched(oaids, 50):
72
+ url = f'https://api.openalex.org/authors?filter=id:{"|".join(batch)}'
73
+
74
+ params = {"per-page": 50, "email": "jkitchin@andrew.cmu.edu"}
75
+
76
+ d = requests.get(url, params=params)
77
+
78
+ for au in d.json()["results"]:
79
+ affils = au["affiliations"]
80
+ if len(affils) > 0:
81
+ affiliations[au["id"]] = affils[0]["institution"]["display_name"]
82
+ else:
83
+ affiliations[au["id"]] = ""
84
+
85
+ uniq = {}
86
+ uniq_authors = [] # by openalex id
87
+ all_authors = []
88
+ for name, year, last_active, oa_id, pub_id in authors:
89
+ if oa_id not in uniq:
90
+ uniq[oa_id] = 1
91
+ affil = affiliations.get(oa_id, "No affiliation known")
92
+ # now we build the tables
93
+ uniq_authors += [["A:", name, affil, "", last_active]]
94
+ all_authors += [["A:", name, affil, "", last_active, pub_id, oa_id]]
95
+
96
+ # unique authors
97
+ df = pd.DataFrame(
98
+ uniq_authors,
99
+ columns=[
100
+ "4",
101
+ "Name:",
102
+ "Organizational Affiliation",
103
+ "Optional (email, Department)",
104
+ "Last Active",
105
+ ],
106
+ )
107
+
108
+ today = datetime.date.today().strftime("%Y-%m-%d.xlsx")
109
+ coa_file = f"{orcid}-{today}"
110
+
111
+ # Table 4
112
+ xw = pd.ExcelWriter(coa_file, engine="xlsxwriter")
113
+ df.to_excel(xw, index=False, sheet_name="Table 4")
114
+
115
+ sheet = xw.sheets["Table 4"]
116
+
117
+ for column in df:
118
+ column_length = max(df[column].astype(str).map(len).max(), len(column))
119
+ col_idx = df.columns.get_loc(column)
120
+ sheet.set_column(col_idx, col_idx, column_length + 2)
121
+
122
+ # Save all authors for debugging
123
+ authors_df = pd.DataFrame(
124
+ all_authors,
125
+ columns=[
126
+ "4",
127
+ "Name:",
128
+ "Organizational Affiliation",
129
+ "Optional (email, Department)",
130
+ "Last Active",
131
+ "OpenAlex pub id",
132
+ "OpenAlex author id",
133
+ ],
134
+ )
135
+ authors_df.to_excel(xw, index=False, sheet_name="all authors")
136
+
137
+ sheet = xw.sheets["all authors"]
138
+ for column in df:
139
+ column_length = max(df[column].astype(str).map(len).max(), len(column))
140
+ col_idx = df.columns.get_loc(column)
141
+ sheet.set_column(col_idx, col_idx, column_length + 2)
142
+
143
+ # Save the Excel file
144
+ xw.close()
145
+
146
+ if get_ipython():
147
+ with open(coa_file, "rb") as f:
148
+ b64 = base64.b64encode(f.read()).decode("utf-8")
149
+ uri = f'<pre>{coa_file}</pre><br><a href="data:text/plain;base64,{b64}" download="{coa_file}">Download COA</a>'
150
+ display(HTML(uri))
151
+ else:
152
+ print(f"Created {coa_file}")
litdb/db.py ADDED
@@ -0,0 +1,362 @@
1
+ """Setup and add things to the database."""
2
+
3
+ import json
4
+ import os
5
+ from pathlib import Path
6
+
7
+ import libsql_experimental as libsql
8
+
9
+ from .utils import get_config
10
+
11
+ import numpy as np
12
+ from sentence_transformers import SentenceTransformer
13
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
14
+ from tqdm import tqdm
15
+ import datetime
16
+ import requests
17
+ import bibtexparser
18
+ from rich import print
19
+
20
+ from litdb.openalex import get_data, get_text
21
+ from litdb.bibtex import dump_bibtex
22
+
23
+
24
+ def get_db():
25
+ """Get or create the database."""
26
+
27
+ config = get_config()
28
+
29
+ root = Path(config["root"])
30
+
31
+ DB = str(root / "litdb.libsql")
32
+
33
+ if os.path.exists(DB):
34
+ db = libsql.connect(DB)
35
+ db.execute("PRAGMA foreign_keys = ON")
36
+ db.execute("PRAGMA journal_mode=WAL")
37
+ return db
38
+ else:
39
+ db = libsql.connect(DB)
40
+ model = SentenceTransformer(config["embedding"]["model"])
41
+
42
+ _, dim = model.encode(["test"]).shape
43
+
44
+ db.execute(
45
+ f"""create table if not exists
46
+ sources(rowid integer primary key,
47
+ source text unique,
48
+ text text,
49
+ extra text,
50
+ embedding F32_BLOB({dim}),
51
+ date_added text)"""
52
+ )
53
+
54
+ db.execute(
55
+ """create virtual table if not exists fulltext using fts5(source, text)"""
56
+ )
57
+
58
+ db.execute(
59
+ """create table if not exists
60
+ tags(rowid integer primary key,
61
+ tag text unique)"""
62
+ )
63
+
64
+ db.execute(
65
+ """create table if not exists
66
+ source_tag(rowid integer primary key,
67
+ source_id integer,
68
+ tag_id integer,
69
+ foreign key(source_id) references sources(rowid) on delete cascade,
70
+ foreign key(tag_id) references tags(rowid) on delete cascade)"""
71
+ )
72
+
73
+ db.execute(
74
+ """create table if not exists
75
+ queries(rowid integer primary key,
76
+ filter text unique,
77
+ description text,
78
+ last_updated text)"""
79
+ )
80
+
81
+ db.execute(
82
+ """create table if not exists
83
+ directories(rowid integer primary key,
84
+ path text unique,
85
+ last_updated text)"""
86
+ )
87
+
88
+ db.execute(
89
+ """create index if not exists embedding_idx ON sources (libsql_vector_idx(embedding))"""
90
+ )
91
+ return db
92
+
93
+
94
+ def add_source(source, text, extra=None):
95
+ """Add a row to the sources table.
96
+
97
+ source : url or path to document
98
+ text: string of the document contents
99
+ extra: jsonable information, usually a dictionary.
100
+
101
+ We generate a document level embedding by averaging the embeddings of the
102
+ document chunks.
103
+ """
104
+ db = get_db()
105
+ config = get_config()
106
+ model = SentenceTransformer(config["embedding"]["model"])
107
+ splitter = RecursiveCharacterTextSplitter(
108
+ chunk_size=config["embedding"]["chunk_size"],
109
+ chunk_overlap=config["embedding"]["chunk_overlap"],
110
+ )
111
+
112
+ chunks = splitter.split_text(text)
113
+ embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
114
+
115
+ c = db.execute(
116
+ """insert or ignore into sources(source, text, extra, embedding, date_added) values (?, ?, ?, ?, ?)""",
117
+ (
118
+ source,
119
+ text,
120
+ json.dumps(extra),
121
+ embedding,
122
+ datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
123
+ ),
124
+ )
125
+
126
+ # I think we should only do this if the previous line succeeded
127
+ if c.rowcount > 0:
128
+ db.execute(
129
+ """insert into fulltext(source, text) values (?, ?)""", (source, text)
130
+ )
131
+
132
+ db.commit()
133
+
134
+
135
+ def get_citation(doi):
136
+ """Get a citation string for doi."""
137
+ config = get_config()
138
+ citeas = "https://api.citeas.org/product/"
139
+ cp = {"email": config["openalex"]["email"]}
140
+
141
+ resp = requests.get(citeas + doi, cp)
142
+ if resp.status_code == 200:
143
+ cdata = resp.json()
144
+ citations = cdata.get("citations", [])
145
+ return citations[0]["citation"]
146
+ else:
147
+ return None
148
+
149
+
150
+ def add_work(workid, references=False, citing=False, related=False):
151
+ """Add a single work to litdb.
152
+
153
+ workid is the doi (full url), or it could be an OpenAlex workid. DOI is
154
+ preferrable.
155
+
156
+ if references is truthy, also add them.
157
+ if citing is truthy, also add them.
158
+ if related is truthy, also add them.
159
+
160
+ """
161
+ config = get_config()
162
+ params = {"email": config["openalex"]["email"], "per_page": 200}
163
+ if config["openalex"].get("api_key"):
164
+ params.update(api_key=config["openalex"].get("api_key"))
165
+
166
+ data = get_data("https://api.openalex.org/works/" + workid, params)
167
+
168
+ # I check to make sure we got something
169
+ source = data.get("id", None)
170
+ if source is None:
171
+ # I guess this could happen for a bad DOI.
172
+ print(f"No id found for {workid}.\n{data}")
173
+ return
174
+
175
+ data["citation"] = get_citation(workid)
176
+ data["bibtex"] = dump_bibtex(data)
177
+
178
+ add_source(workid, get_text(data), data)
179
+
180
+ if references:
181
+ for wid in tqdm(data["referenced_works"]):
182
+ rdata = get_data("https://api.openalex.org/works/" + wid, params)
183
+ source = rdata.get("doi") or rdata.get("id")
184
+ if source is None:
185
+ print(f"Something failed for {wid}. continuing")
186
+ continue
187
+ text = get_text(rdata)
188
+ rdata["citation"] = get_citation(source)
189
+ rdata["bibtex"] = dump_bibtex(rdata)
190
+ add_source(source, text, rdata)
191
+
192
+ if related:
193
+ for wid in tqdm(data["related_works"]):
194
+ rdata = get_data("https://api.openalex.org/works/" + wid, params)
195
+ source = rdata.get("doi") or rdata.get("id")
196
+ if source is None:
197
+ print(f"Something failed for {wid}. continuing")
198
+ continue
199
+ text = get_text(rdata)
200
+ rdata["citation"] = get_citation(source)
201
+ rdata["bibtex"] = dump_bibtex(rdata)
202
+ add_source(source, text, rdata)
203
+
204
+ if citing:
205
+ CURL = data["cited_by_api_url"]
206
+ next_cursor = "*"
207
+ params.update(cursor=next_cursor)
208
+ while next_cursor:
209
+ cdata = get_data(CURL, params)
210
+ next_cursor = cdata["meta"]["next_cursor"]
211
+ params.update(cursor=next_cursor)
212
+ if next_cursor is None:
213
+ break
214
+
215
+ # TODO: should the max citations to trigger this be configurable in litdb.toml
216
+ trigger = config["openalex"].get("citation_count_trigger", 100)
217
+ count = cdata["meta"]["count"]
218
+ if count > trigger:
219
+ r = input(
220
+ f"Found {count} citations. Do you want to download them all? (n/y): "
221
+ )
222
+ if r.lower().startswith("n"):
223
+ break
224
+ else:
225
+ pass
226
+
227
+ for work in tqdm(cdata["results"]):
228
+ source = work.get("doi") or work["id"]
229
+ text = get_text(work)
230
+ work["citation"] = get_citation(source)
231
+ work["bibtex"] = dump_bibtex(work)
232
+ add_source(source, text, work)
233
+
234
+
235
+ def add_author(oaid):
236
+ """Add all works from an author.
237
+ "id": "https://openalex.org/A5003442464",
238
+ "orcid": "https://orcid.org/0000-0003-2625-9232",
239
+ """
240
+ config = get_config()
241
+ aurl = "https://api.openalex.org/authors/" + oaid
242
+
243
+ params = {
244
+ "email": config["openalex"]["email"],
245
+ "per_page": 200,
246
+ }
247
+
248
+ if config["openalex"].get("api_key"):
249
+ params.update(api_key=config["openalex"].get("api_key"))
250
+
251
+ data = get_data(aurl, params)
252
+
253
+ wurl = data["works_api_url"]
254
+ next_cursor = "*"
255
+ params.update(cursor=next_cursor)
256
+ while next_cursor:
257
+ wdata = get_data(wurl, params)
258
+ next_cursor = wdata["meta"]["next_cursor"]
259
+ params.update(cursor=next_cursor)
260
+ for work in tqdm(wdata["results"]):
261
+ add_work(work.get("doi") or work["id"])
262
+
263
+
264
+ def update_filter(f, last_updated=None, silent=False):
265
+ """Update one filter (f). f should be a string that goes in the filter=
266
+ param for openalex.
267
+
268
+ last_updated should be a string %Y-%m-%d
269
+
270
+ If last_updated is None, we only go back one year.
271
+ That might still lead to a lot of hits.
272
+
273
+ """
274
+ config = get_config()
275
+ model = SentenceTransformer(config["embedding"]["model"])
276
+ splitter = RecursiveCharacterTextSplitter(
277
+ chunk_size=config["embedding"]["chunk_size"],
278
+ chunk_overlap=config["embedding"]["chunk_overlap"],
279
+ )
280
+
281
+ db = get_db()
282
+
283
+ _filter = f # make a copy because we modify it later.
284
+
285
+ wurl = "https://api.openalex.org/works"
286
+
287
+ # If last_updated is None, we limit to the past year
288
+ if last_updated is None:
289
+ now = datetime.date.today()
290
+ last_updated = (now - datetime.timedelta(days=365)).strftime("%Y-%m-%d")
291
+
292
+ _filter += f",from_created_date:{last_updated}"
293
+
294
+ params = {
295
+ "email": config["openalex"]["email"],
296
+ "api_key": config["openalex"]["api_key"],
297
+ "per_page": 200,
298
+ "filter": _filter,
299
+ }
300
+
301
+ next_cursor = "*"
302
+ params.update(cursor=next_cursor)
303
+
304
+ results = []
305
+ while next_cursor:
306
+ data = get_data(wurl, params)
307
+ next_cursor = data["meta"]["next_cursor"]
308
+ params.update(cursor=next_cursor)
309
+
310
+ for work in tqdm(data["results"], disable=silent):
311
+ source = work.get("doi") or work.get("id")
312
+ citation = get_citation(source)
313
+ work["citation"] = citation
314
+
315
+ bibtex = dump_bibtex(work)
316
+ work["bibtex"] = bibtex
317
+
318
+ text = get_text(work)
319
+
320
+ chunks = splitter.split_text(text)
321
+ embedding = model.encode(chunks).mean(axis=0).astype(np.float32).tobytes()
322
+
323
+ results += [[source, text, work]]
324
+
325
+ now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
326
+ c = db.execute(
327
+ """insert or ignore into
328
+ sources(source, text, extra, embedding, date_added)
329
+ values(?, ?, ?, ?, ?)""",
330
+ (source, text, json.dumps(work), embedding, now),
331
+ )
332
+
333
+ if c.rowcount > 0:
334
+ db.execute(
335
+ """insert into fulltext(source, text) values (?, ?)""",
336
+ (source, text),
337
+ )
338
+
339
+ db.execute(
340
+ """update queries set last_updated = ? where filter = ?""",
341
+ (datetime.date.today().strftime("%Y-%m-%d"), f),
342
+ )
343
+
344
+ db.commit()
345
+ return results
346
+
347
+
348
+ def add_bibtex(bibfile):
349
+ """Add entries with a DOI from bibfile."""
350
+
351
+ with open(bibfile, "r", encoding="utf-8", errors="replace") as bibfile:
352
+ bib_database = bibtexparser.load(bibfile)
353
+
354
+ for entry in tqdm(bib_database.entries):
355
+ if "doi" in entry:
356
+ doi = entry["doi"]
357
+ if doi.startswith("http"):
358
+ add_work(doi)
359
+ elif doi.startswith("10"):
360
+ add_work(f"https://doi.org/{doi}")
361
+ else:
362
+ print(f'I do not know what to do with "{doi}"')
litdb/gpt.py ADDED
@@ -0,0 +1,150 @@
1
+ """The gpt command for litdb."""
2
+
3
+ import os
4
+ import readline
5
+ import subprocess
6
+
7
+ import numpy as np
8
+ import ollama
9
+ from rich import print as richprint
10
+ from sentence_transformers import SentenceTransformer
11
+
12
+ from .utils import get_config
13
+ from .db import get_db
14
+
15
+ # Enable command history
16
+ readline.parse_and_bind("tab: complete")
17
+ readline.parse_and_bind("set editing-mode emacs")
18
+
19
+ # Disable parallelism in tokenizers
20
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
21
+
22
+
23
+ def gpt():
24
+ """Start a LitGPT chat session. The prompt is interactive.
25
+
26
+ If the prompt starts with > the rest of the prompt will be run as a shell
27
+ command. Use this to add references, citations, etc, during the chat.
28
+
29
+ If the prompt starts with < the rest of the prompt will be run as a shell
30
+ command and the output used for RAG.
31
+
32
+ !save it will save the chat to a file.
33
+ !restart will reset the messages and restart the chat
34
+ !messages will output the current chat history to the shell
35
+ !help to print a help message
36
+
37
+ Use Ctrl-d to exit.
38
+ """
39
+ config = get_config()
40
+ db = get_db()
41
+ model = SentenceTransformer(config["embedding"]["model"])
42
+
43
+ gpt = config.get("gpt", {"model": "llama2"})
44
+ gpt_model = gpt["model"]
45
+
46
+ messages = []
47
+
48
+ while prompt := input("LitGPT (Ctrl-d to quit)> "):
49
+ rag_content = ""
50
+
51
+ if prompt == "!help":
52
+ print("""\
53
+ If the prompt starts with >, run the rest as a shell command, e.g.
54
+ > litdb add doi
55
+ then continue to a new prompt.
56
+
57
+ If the prompt starts with < run the shell command but capture the output as
58
+ context for the next prompt.
59
+
60
+ The following subcommands can be used:
61
+
62
+ !save to save the chat to a file
63
+ !restart to reset the chat
64
+ !help for this message
65
+ """)
66
+ continue
67
+
68
+ elif prompt.startswith(">"):
69
+ # This means run the rest of the prompt as a shell command
70
+ # > litdb add some-id
71
+ os.system(prompt[1:].strip())
72
+ continue
73
+
74
+ # a little sub-language of commands
75
+ elif prompt == "!save":
76
+ with open(input("Filename (chat.txt): ") or "chat.txt", "w") as f:
77
+ for message in messages:
78
+ f.write(f'{message["role"]}: {message["content"]}\n\n')
79
+ continue
80
+
81
+ elif prompt == "!restart":
82
+ messages = []
83
+ print("Reset the chat.")
84
+ continue
85
+
86
+ elif prompt == "!messages":
87
+ for message in messages:
88
+ richprint(f'{message["role"]}: {message["content"]}\n\n')
89
+ continue
90
+
91
+ # Run shell command to get rag content
92
+ elif prompt.startswith("<"):
93
+ # Some command that outputs text to stdout
94
+ result = subprocess.run(
95
+ prompt[1:].strip(), shell=True, text=True, capture_output=True
96
+ )
97
+ rag_content = f"{prompt[1:]}\n\n" + result.stdout
98
+ prompt = input("LitGPT (Ctrl-d to quit)> ")
99
+
100
+ data = None
101
+ if not rag_content:
102
+ # RAG by vector search
103
+ emb = model.encode([prompt]).astype(np.float32).tobytes()
104
+ data = db.execute(
105
+ """select sources.text, json_extract(sources.extra, '$.citation')
106
+ from vector_top_k('embedding_idx', ?, 3)
107
+ join sources on sources.rowid = id""",
108
+ (emb,),
109
+ ).fetchall()
110
+
111
+ for doc, citation in data:
112
+ rag_content += f"\n\n{doc}"
113
+
114
+ messages += [
115
+ {
116
+ "role": "system",
117
+ "content": (
118
+ "Only use the following information and previously"
119
+ " provided information to respond"
120
+ " to the prompt. Do not use anything else:"
121
+ f" {rag_content}"
122
+ ),
123
+ }
124
+ ]
125
+
126
+ # I think we need to send this before we can use it for the user.
127
+ response = ollama.chat(model=gpt_model, messages=messages, stream=True)
128
+ for chunk in response:
129
+ richprint(chunk["message"]["content"], end="", flush=True)
130
+
131
+ messages += [{"role": "user", "content": prompt}]
132
+
133
+ output = ""
134
+ # This lets you Ctrl-c to stop streaming if it has gone way off.
135
+ try:
136
+ response = ollama.chat(model=gpt_model, messages=messages, stream=True)
137
+ for chunk in response:
138
+ output += chunk["message"]["content"]
139
+ richprint(chunk["message"]["content"], end="", flush=True)
140
+ except KeyboardInterrupt:
141
+ response.close()
142
+ richprint()
143
+
144
+ messages += [{"role": "assistant", "content": output}]
145
+
146
+ # We don't always have data here, if you use your own rag data
147
+ if data:
148
+ richprint("\nThe text was generated using these references:\n")
149
+ for i, (text, citation) in enumerate(data, 1):
150
+ richprint(f"{i:2d}. {citation}\n")
litdb/lab.py ADDED
@@ -0,0 +1,23 @@
1
+ """Jupyter Lab interface for litdb."""
2
+
3
+ import shlex
4
+ from IPython.core.magic import Magics, magics_class, line_cell_magic
5
+ from IPython import get_ipython
6
+ from .cli import cli
7
+
8
+
9
+ @magics_class
10
+ class LitdbMagics(Magics):
11
+ @line_cell_magic
12
+ def litdb(self, line, cell=None):
13
+ """Main litdb magic command using Click."""
14
+ args = shlex.split(line)
15
+ if cell is not None:
16
+ args += [cell]
17
+ cli.main(args=args, standalone_mode=False)
18
+
19
+
20
+ # Register the magic command
21
+ ip = get_ipython()
22
+ if ip:
23
+ ip.register_magics(LitdbMagics)