paper-manager 0.11.2 → 0.12.0

package/config.schema.json

@@ -47,6 +47,11 @@
     "defaultEmbeddingModelId": {
       "type": "string",
       "minLength": 1
+    },
+    "email": {
+      "type": "string",
+      "format": "email",
+      "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$"
     }
   },
   "required": ["embeddingModels"],

package/dist/commands/literature.js

@@ -1,9 +1,10 @@
 import * as fs from "node:fs";
+import * as os from "node:os";
 import * as path from "node:path";
 import chalk from "chalk";
 import cliProgress from "cli-progress";
 import { Command } from "commander";
-import { getFilesDir, getModelConfig, getProjectDataDir, getUserDataDir, getVectorStoreDir, } from "../config/index.js";
+import { getConfig, getFilesDir, getModelConfig, getProjectDataDir, getUserDataDir, getVectorStoreDir, } from "../config/index.js";
 import * as projectKb from "../db/project/knowledge-bases.js";
 import * as projectLit from "../db/project/literatures.js";
 import * as userKb from "../db/user/knowledge-bases.js";
@@ -12,6 +13,7 @@ import { extractContent, extractPdfMetadata } from "../extractor/index.js";
 import { convertPdfToMarkdown, isOpendataLoaderAvailable, removeImageDir, saveConvertResult, } from "../extractor/markdown.js";
 import { log } from "../logger.js";
 import { splitDocuments } from "../text-splitter.js";
+import { downloadPdf, lookupDoi, normalizeDoi, UnpaywallError } from "../unpaywall/index.js";
 import { addDocuments, createVectorStore } from "../vector-store/index.js";
 import { outputJson } from "./output.js";
 function resolveKnowledgeBase(id) {
@@ -33,11 +35,21 @@ export function createLiteratureCommand() {
     const lit = new Command("lit").description("Manage literatures");
     // ─── lit add ───────────────────────────────────────────────
     lit
-        .command("add <knowledge-base-id> <lit-path>")
-        .description("Add a literature from a file (PDF, TXT, MD, TEX, etc.)")
+        .command("add <knowledge-base-id> [lit-path]")
+        .description("Add a literature from a file (PDF, TXT, MD, TEX, etc.) or by DOI via Unpaywall")
         .option("-t, --title <title>", "Literature title")
         .option("-f, --force", "Force add even if a literature with the same DOI already exists")
+        .option("-d, --doi <doi>", "Add paper by DOI (downloads Open Access PDF via Unpaywall)")
         .action(async (kbId, litPath, options) => {
+        // Mutual exclusivity check
+        if (litPath && options.doi) {
+            log.error("Cannot specify both <lit-path> and --doi. Use one or the other.");
+            process.exit(1);
+        }
+        if (!litPath && !options.doi) {
+            log.error("Either <lit-path> or --doi is required.");
+            process.exit(1);
+        }
         const resolved = resolveKnowledgeBase(kbId);
         if (!resolved) {
             log.error(`Knowledge base not found: ${kbId}`);
@@ -46,108 +58,211 @@ export function createLiteratureCommand() {
         const { kb, scope } = resolved;
         const baseDir = getBaseDir(scope);
         const litOps = getLitOps(scope);
-        // Resolve file path
-        const absolutePath = path.resolve(litPath);
-        if (!fs.existsSync(absolutePath)) {
-            log.error(`File not found: ${absolutePath}`);
-            process.exit(1);
-        }
-        log.info("Extracting content...");
-        const docs = await extractContent(absolutePath);
-        log.step(`Extracted ${String(docs.length)} pages.`);
-        // Extract PDF metadata if available
-        const isPdf = absolutePath.toLowerCase().endsWith(".pdf");
-        const pdfMeta = isPdf ? await extractPdfMetadata(absolutePath) : null;
-        if (pdfMeta) {
-            const hasAny = pdfMeta.title ?? pdfMeta.author ?? pdfMeta.doi ?? pdfMeta.subject;
-            if (hasAny || pdfMeta.keywords.length > 0) {
-                log.info("Extracted PDF metadata:");
-                if (pdfMeta.title)
-                    log.step(`Title: ${pdfMeta.title}`);
-                if (pdfMeta.author)
-                    log.step(`Author: ${pdfMeta.author}`);
-                if (pdfMeta.subject)
-                    log.step(`Subject: ${pdfMeta.subject}`);
-                if (pdfMeta.doi)
-                    log.step(`DOI: ${pdfMeta.doi}`);
-                if (pdfMeta.keywords.length > 0)
-                    log.step(`Keywords: ${pdfMeta.keywords.join(", ")}`);
-                if (pdfMeta.creationDate)
-                    log.step(`Created: ${pdfMeta.creationDate.toISOString()}`);
-                if (pdfMeta.creator)
-                    log.step(`Creator: ${pdfMeta.creator}`);
+        let absolutePath;
+        let tempDir = null;
+        let doiFromFlag = null;
+        let unpaywallMeta = null;
+        if (options.doi) {
+            // ─── DOI mode: lookup Unpaywall and download OA PDF ───
+            const normalizedDoi = normalizeDoi(options.doi);
+            doiFromFlag = normalizedDoi;
+            const email = getConfig("email");
+            if (!email) {
+                log.error('Email is required for Unpaywall API. Set it with: paper config set email "you@example.com"');
+                process.exit(1);
             }
-        }
-        // Check for duplicate DOI in the knowledge base
-        if (pdfMeta?.doi && !options.force) {
-            const existing = litOps.findLiteratureByDoi(kbId, pdfMeta.doi);
-            if (existing) {
-                log.error(`A literature with DOI "${pdfMeta.doi}" already exists in this knowledge base: ${existing.id} (${existing.title})`);
-                log.info("Use --force to add anyway.");
+            // Check for duplicate DOI before downloading
+            if (!options.force) {
+                const existing = litOps.findLiteratureByDoi(kbId, normalizedDoi);
+                if (existing) {
+                    log.error(`A literature with DOI "${normalizedDoi}" already exists in this knowledge base: ${existing.id} (${existing.title})`);
+                    log.info("Use --force to add anyway.");
+                    process.exit(1);
+                }
+            }
+            log.info(`Looking up DOI: ${normalizedDoi}`);
+            try {
+                unpaywallMeta = await lookupDoi(normalizedDoi, email);
+            }
+            catch (err) {
+                if (err instanceof UnpaywallError) {
+                    log.error(err.message);
+                }
+                else {
+                    log.error(`Unpaywall lookup failed: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                process.exit(1);
+            }
+            if (!unpaywallMeta.is_oa) {
+                log.error(`Paper is not Open Access (status: ${unpaywallMeta.oa_status}).`);
+                log.info(`Add it manually: paper lit add ${kbId} <file>`);
+                process.exit(1);
+            }
+            const pdfUrl = unpaywallMeta.best_oa_location?.url_for_pdf;
+            if (!pdfUrl) {
+                const landingPage = unpaywallMeta.best_oa_location?.url_for_landing_page;
+                log.error("Paper is Open Access but no direct PDF URL is available.");
+                if (landingPage) {
+                    log.info(`Landing page: ${landingPage}`);
+                }
+                log.info(`Download the PDF manually and use: paper lit add ${kbId} <file>`);
+                process.exit(1);
+            }
+            // Show Unpaywall metadata
+            log.info("Unpaywall metadata:");
+            if (unpaywallMeta.title)
+                log.step(`Title: ${unpaywallMeta.title}`);
+            if (unpaywallMeta.z_authors && unpaywallMeta.z_authors.length > 0) {
+                log.step(`Authors: ${unpaywallMeta.z_authors.map((a) => a.raw_author_name).join(", ")}`);
+            }
+            if (unpaywallMeta.journal_name)
+                log.step(`Journal: ${unpaywallMeta.journal_name}`);
+            if (unpaywallMeta.year)
+                log.step(`Year: ${String(unpaywallMeta.year)}`);
+            log.step(`OA Status: ${unpaywallMeta.oa_status}`);
+            // Download PDF to temp location
+            tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "paper-unpaywall-"));
+            absolutePath = path.join(tempDir, `${normalizedDoi.replace(/\//g, "_")}.pdf`);
+            log.info("Downloading PDF...");
+            try {
+                await downloadPdf(pdfUrl, absolutePath);
+            }
+            catch (err) {
+                fs.rmSync(tempDir, { recursive: true, force: true });
+                if (err instanceof UnpaywallError) {
+                    log.error(err.message);
+                }
+                else {
+                    log.error(`PDF download failed: ${err instanceof Error ? err.message : String(err)}`);
+                }
                 process.exit(1);
             }
+            log.step("PDF downloaded.");
         }
-        const title = options.title ?? pdfMeta?.title ?? path.basename(litPath, path.extname(litPath));
-        // Create literature record
-        const literature = litOps.createLiterature({
-            title,
-            titleTranslation: null,
-            author: pdfMeta?.author ?? null,
-            abstract: pdfMeta?.subject ?? null,
-            summary: null,
-            keywords: pdfMeta?.keywords ?? [],
-            url: null,
-            doi: pdfMeta?.doi ?? null,
-            notes: {},
-            knowledgeBaseId: kbId,
-        });
-        // Copy file to storage
-        const filesDir = getFilesDir(baseDir);
-        const ext = path.extname(litPath);
-        fs.mkdirSync(filesDir, { recursive: true });
-        fs.copyFileSync(absolutePath, path.join(filesDir, `${literature.id}${ext}`));
-        // Convert PDF to Markdown if opendataloader is available
-        if (isPdf && (await isOpendataLoaderAvailable())) {
-            const result = await convertPdfToMarkdown(absolutePath);
-            if (result) {
-                saveConvertResult(filesDir, literature.id, result);
-                log.step("Converted to Markdown via opendataloader-pdf.");
+        else {
+            // ─── File mode (existing behavior) ────────────────────
+            // litPath is guaranteed to be defined here by the mutual exclusivity check above
+            const filePath = litPath ?? "";
+            absolutePath = path.resolve(filePath);
+            if (!fs.existsSync(absolutePath)) {
+                log.error(`File not found: ${absolutePath}`);
+                process.exit(1);
             }
         }
-        // Split text and add to vector store
-        log.info("Splitting text...");
-        const splitDocs = splitDocuments(docs, { chunkSize: 1000, chunkOverlap: 200 });
-        log.step(`Created ${String(splitDocs.length)} chunks.`);
-        // Add literature ID metadata to each chunk
-        for (const doc of splitDocs) {
-            doc.metadata = { ...doc.metadata, literatureId: literature.id };
-        }
-        const vectorDir = path.join(getVectorStoreDir(baseDir), kbId);
-        const modelConfig = getModelConfig(kb.embeddingModelId);
-        log.info("Embedding and storing vectors...");
-        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-        bar.start(splitDocs.length, 0);
-        // Check if both FAISS index files exist (not just the directory)
-        const hasIndex = fs.existsSync(path.join(vectorDir, "faiss.index")) &&
-            fs.existsSync(path.join(vectorDir, "docstore.json"));
-        if (hasIndex) {
-            await addDocuments(splitDocs, modelConfig, vectorDir);
+        // ─── Shared flow ──────────────────────────────────────
+        try {
+            log.info("Extracting content...");
+            const docs = await extractContent(absolutePath);
+            log.step(`Extracted ${String(docs.length)} pages.`);
+            // Extract PDF metadata if available
+            const isPdf = absolutePath.toLowerCase().endsWith(".pdf");
+            const pdfMeta = isPdf ? await extractPdfMetadata(absolutePath) : null;
+            if (pdfMeta) {
+                const hasAny = pdfMeta.title ?? pdfMeta.author ?? pdfMeta.doi ?? pdfMeta.subject;
+                if (hasAny || pdfMeta.keywords.length > 0) {
+                    log.info("Extracted PDF metadata:");
+                    if (pdfMeta.title)
+                        log.step(`Title: ${pdfMeta.title}`);
+                    if (pdfMeta.author)
+                        log.step(`Author: ${pdfMeta.author}`);
+                    if (pdfMeta.subject)
+                        log.step(`Subject: ${pdfMeta.subject}`);
+                    if (pdfMeta.doi)
+                        log.step(`DOI: ${pdfMeta.doi}`);
+                    if (pdfMeta.keywords.length > 0)
+                        log.step(`Keywords: ${pdfMeta.keywords.join(", ")}`);
+                    if (pdfMeta.creationDate)
+                        log.step(`Created: ${pdfMeta.creationDate.toISOString()}`);
+                    if (pdfMeta.creator)
+                        log.step(`Creator: ${pdfMeta.creator}`);
+                }
+            }
+            // Check for duplicate DOI (file mode only — DOI mode already checked above)
+            const effectiveDoi = doiFromFlag ?? pdfMeta?.doi ?? null;
+            if (effectiveDoi && !doiFromFlag && !options.force) {
+                const existing = litOps.findLiteratureByDoi(kbId, effectiveDoi);
+                if (existing) {
+                    log.error(`A literature with DOI "${effectiveDoi}" already exists in this knowledge base: ${existing.id} (${existing.title})`);
+                    log.info("Use --force to add anyway.");
+                    process.exit(1);
+                }
+            }
+            // Resolve metadata: CLI option > Unpaywall > PDF metadata > fallback
+            const unpaywallAuthors = unpaywallMeta?.z_authors && unpaywallMeta.z_authors.length > 0
+                ? unpaywallMeta.z_authors.map((a) => a.raw_author_name).join(", ")
+                : null;
+            const title = options.title ??
+                unpaywallMeta?.title ??
+                pdfMeta?.title ??
+                (litPath
+                    ? path.basename(litPath, path.extname(litPath))
+                    : (effectiveDoi ?? "Untitled"));
+            // Create literature record
+            const literature = litOps.createLiterature({
+                title,
+                titleTranslation: null,
+                author: pdfMeta?.author ?? unpaywallAuthors,
+                abstract: pdfMeta?.subject ?? null,
+                summary: null,
+                keywords: pdfMeta?.keywords ?? [],
+                url: null,
+                doi: effectiveDoi,
+                notes: {},
+                knowledgeBaseId: kbId,
+            });
+            // Copy file to storage
+            const filesDir = getFilesDir(baseDir);
+            const ext = path.extname(absolutePath);
+            fs.mkdirSync(filesDir, { recursive: true });
+            fs.copyFileSync(absolutePath, path.join(filesDir, `${literature.id}${ext}`));
+            // Convert PDF to Markdown if opendataloader is available
+            if (isPdf && (await isOpendataLoaderAvailable())) {
+                const result = await convertPdfToMarkdown(absolutePath);
+                if (result) {
+                    saveConvertResult(filesDir, literature.id, result);
+                    log.step("Converted to Markdown via opendataloader-pdf.");
+                }
+            }
+            // Split text and add to vector store
+            log.info("Splitting text...");
+            const splitDocs = splitDocuments(docs, { chunkSize: 1000, chunkOverlap: 200 });
+            log.step(`Created ${String(splitDocs.length)} chunks.`);
+            // Add literature ID metadata to each chunk
+            for (const doc of splitDocs) {
+                doc.metadata = { ...doc.metadata, literatureId: literature.id };
+            }
+            const vectorDir = path.join(getVectorStoreDir(baseDir), kbId);
+            const modelConfig = getModelConfig(kb.embeddingModelId);
+            log.info("Embedding and storing vectors...");
+            const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+            bar.start(splitDocs.length, 0);
+            // Check if both FAISS index files exist (not just the directory)
+            const hasIndex = fs.existsSync(path.join(vectorDir, "faiss.index")) &&
+                fs.existsSync(path.join(vectorDir, "docstore.json"));
+            if (hasIndex) {
+                await addDocuments(splitDocs, modelConfig, vectorDir);
+            }
+            else {
+                await createVectorStore(splitDocs, modelConfig, vectorDir);
+            }
+            bar.update(splitDocs.length);
+            bar.stop();
+            log.success(`Literature added: ${literature.id}`);
+            log.label("Title:", literature.title);
+            if (literature.author)
+                log.label("Author:", literature.author);
+            if (literature.abstract)
+                log.label("Abstract:", literature.abstract);
+            if (literature.doi)
+                log.label("DOI:", literature.doi);
+            if (literature.keywords.length > 0)
+                log.label("Keywords:", literature.keywords.join(", "));
+        }
+        finally {
+            if (tempDir) {
+                fs.rmSync(tempDir, { recursive: true, force: true });
+            }
         }
-        else {
-            await createVectorStore(splitDocs, modelConfig, vectorDir);
-        }
-        bar.update(splitDocs.length);
-        bar.stop();
-        log.success(`Literature added: ${literature.id}`);
-        log.label("Title:", literature.title);
-        if (literature.author)
-            log.label("Author:", literature.author);
-        if (literature.abstract)
-            log.label("Abstract:", literature.abstract);
-        if (literature.doi)
-            log.label("DOI:", literature.doi);
-        if (literature.keywords.length > 0)
-            log.label("Keywords:", literature.keywords.join(", "));
     });
     // ─── lit convert ────────────────────────────────────────────
     lit

package/dist/config/index.js

@@ -61,6 +61,7 @@ function getProjectConfigPath() {
 const configSchemas = {
     embeddingModels: z.record(z.string().min(1), EmbeddingModelConfigSchema),
     defaultEmbeddingModelId: z.string().min(1),
+    email: z.email(),
 };
 // ─── Config File I/O ────────────────────────────────────────
 export function readConfigFile(filePath) {

package/dist/types/index.js

@@ -32,4 +32,5 @@ export const ConfigSchema = z.object({
     $schema: z.string().optional(),
     embeddingModels: z.record(z.string().min(1), EmbeddingModelConfigSchema).default({}),
     defaultEmbeddingModelId: z.string().min(1).optional(),
+    email: z.email().optional(),
 });

package/dist/unpaywall/index.js

@@ -0,0 +1,71 @@
+import { writeFile } from "node:fs/promises";
+import * as z from "zod";
+// ─── Unpaywall Response Schema ─────────────────────────────
+const UnpaywallOaLocationSchema = z.object({
+    url_for_pdf: z.string().nullable(),
+    url_for_landing_page: z.string().nullable(),
+    license: z.string().nullable(),
+    version: z.string().nullable(),
+    host_type: z.string().nullable(),
+});
+const UnpaywallAuthorSchema = z.object({
+    raw_author_name: z.string(),
+});
+const UnpaywallResponseSchema = z.object({
+    is_oa: z.boolean(),
+    oa_status: z.string(),
+    title: z.string().nullable().optional(),
+    z_authors: z.array(UnpaywallAuthorSchema).nullable().optional(),
+    published_date: z.string().nullable().optional(),
+    journal_name: z.string().nullable().optional(),
+    year: z.number().nullable().optional(),
+    publisher: z.string().nullable().optional(),
+    best_oa_location: UnpaywallOaLocationSchema.nullable(),
+    doi: z.string(),
+});
+export class UnpaywallError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.name = "UnpaywallError";
+        this.code = code;
+    }
+}
+// ─── DOI Normalization ─────────────────────────────────────
+export function normalizeDoi(input) {
+    return input.replace(/^https?:\/\/(dx\.)?doi\.org\//, "").replace(/^doi:/i, "");
+}
+// ─── API Client ────────────────────────────────────────────
+export async function lookupDoi(doi, email) {
+    const url = `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=${encodeURIComponent(email)}`;
+    const response = await fetch(url, {
+        headers: { Accept: "application/json" },
+        redirect: "follow",
+    });
+    if (response.status === 404) {
+        throw new UnpaywallError(`DOI not found in Unpaywall: ${doi}`, "not_found");
+    }
+    if (!response.ok) {
+        throw new UnpaywallError(`Unpaywall API error: HTTP ${String(response.status)}`, "api_error");
+    }
+    const json = await response.json();
+    const result = UnpaywallResponseSchema.safeParse(json);
+    if (!result.success) {
+        throw new UnpaywallError(`Invalid Unpaywall API response: ${result.error.message}`, "parse_error");
+    }
+    return result.data;
+}
+// ─── PDF Download ──────────────────────────────────────────
+export async function downloadPdf(url, destPath) {
+    const response = await fetch(url, { redirect: "follow" });
+    if (!response.ok) {
+        throw new UnpaywallError(`Failed to download PDF: HTTP ${String(response.status)}`, "download_error");
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!contentType.includes("application/pdf") &&
+        !contentType.includes("application/octet-stream")) {
+        throw new UnpaywallError(`Expected PDF but received: ${contentType}`, "download_error");
+    }
+    const buffer = new Uint8Array(await response.arrayBuffer());
+    await writeFile(destPath, buffer);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "paper-manager",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "description": "A paper management system.",
   "keywords": [],
   "homepage": "https://github.com/EurFelux/paper-manager",