paper-manager 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +5 -0
- package/dist/ai/embed.js +12 -6
- package/dist/ai/embed.js.map +1 -1
- package/dist/config/index.d.ts +1 -0
- package/dist/pdf/extractor.d.ts +1 -1
- package/dist/pdf/extractor.js +15 -3
- package/dist/pdf/extractor.js.map +1 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +1 -0
- package/dist/types/index.js.map +1 -1
- package/package.json +1 -1
package/config.schema.json
CHANGED
package/dist/ai/embed.js
CHANGED
|
@@ -16,11 +16,17 @@ export async function embed(config, text) {
|
|
|
16
16
|
}
|
|
17
17
|
export async function embedMany(config, texts) {
|
|
18
18
|
const model = createEmbeddingModel(config);
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
values: texts,
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
19
|
+
const providerOptions = buildProviderOptions(config);
|
|
20
|
+
if (config.batchSize == null || texts.length <= config.batchSize) {
|
|
21
|
+
const result = await aiEmbedMany({ model, values: texts, providerOptions });
|
|
22
|
+
return result.embeddings;
|
|
23
|
+
}
|
|
24
|
+
const embeddings = [];
|
|
25
|
+
for (let i = 0; i < texts.length; i += config.batchSize) {
|
|
26
|
+
const batch = texts.slice(i, i + config.batchSize);
|
|
27
|
+
const result = await aiEmbedMany({ model, values: batch, providerOptions });
|
|
28
|
+
embeddings.push(...result.embeddings);
|
|
29
|
+
}
|
|
30
|
+
return embeddings;
|
|
25
31
|
}
|
|
26
32
|
//# sourceMappingURL=embed.js.map
|
package/dist/ai/embed.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/ai/embed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,SAAS,IAAI,WAAW,EAAE,MAAM,IAAI,CAAC;AAGhE,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,oBAAoB,CAC3B,MAA4B;IAE5B,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAChD,OAAO,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAA4B,EAAE,IAAY;IACpE,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;QAC3B,KAAK;QACL,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,oBAAoB,CAAC,MAAM,CAAC;KAC9C,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,SAAS,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAA4B,EAC5B,KAAe;IAEf,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,
|
|
1
|
+
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/ai/embed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,SAAS,IAAI,WAAW,EAAE,MAAM,IAAI,CAAC;AAGhE,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErD,SAAS,oBAAoB,CAC3B,MAA4B;IAE5B,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAChD,OAAO,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAA4B,EAAE,IAAY;IACpE,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;QAC3B,KAAK;QACL,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,oBAAoB,CAAC,MAAM,CAAC;KAC9C,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,SAAS,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAA4B,EAC5B,KAAe;IAEf,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAErD,IAAI,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAC5E,OAAO,MAAM,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,MAAM,UAAU,GAAe,EAAE,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAC5E,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
package/dist/config/index.d.ts
CHANGED
package/dist/pdf/extractor.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Document } from "@langchain/core/documents";
|
|
2
2
|
export declare function extractPdfContent(pdfPath: string): Promise<Document[]>;
|
package/dist/pdf/extractor.js
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { Document } from "@langchain/core/documents";
|
|
3
|
+
import { PDFParse } from "pdf-parse";
|
|
2
4
|
export async function extractPdfContent(pdfPath) {
|
|
3
|
-
const
|
|
4
|
-
|
|
5
|
+
const data = await readFile(pdfPath);
|
|
6
|
+
const parser = new PDFParse({ data });
|
|
7
|
+
const result = await parser.getText();
|
|
8
|
+
await parser.destroy();
|
|
9
|
+
return result.pages.map((page) => new Document({
|
|
10
|
+
pageContent: page.text,
|
|
11
|
+
metadata: {
|
|
12
|
+
source: pdfPath,
|
|
13
|
+
pdf: { totalPages: result.total },
|
|
14
|
+
loc: { pageNumber: page.num },
|
|
15
|
+
},
|
|
16
|
+
}));
|
|
5
17
|
}
|
|
6
18
|
//# sourceMappingURL=extractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/pdf/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/pdf/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IAEvB,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CACrB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,QAAQ,CAAC;QACX,WAAW,EAAE,IAAI,CAAC,IAAI;QACtB,QAAQ,EAAE;YACR,MAAM,EAAE,OAAO;YACf,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,EAAE;YACjC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;SAC9B;KACF,CAAC,CACL,CAAC;AACJ,CAAC"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export declare const EmbeddingModelConfigSchema: z.ZodObject<{
|
|
|
7
7
|
baseUrl: z.ZodOptional<z.ZodURL>;
|
|
8
8
|
apiKey: z.ZodString;
|
|
9
9
|
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
10
|
+
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
10
11
|
}, z.core.$strip>;
|
|
11
12
|
export type EmbeddingModelConfig = z.infer<typeof EmbeddingModelConfigSchema> & {
|
|
12
13
|
id: string;
|
|
@@ -69,6 +70,7 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
69
70
|
baseUrl: z.ZodOptional<z.ZodURL>;
|
|
70
71
|
apiKey: z.ZodString;
|
|
71
72
|
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
72
74
|
}, z.core.$strip>>>;
|
|
73
75
|
defaultEmbeddingModelId: z.ZodOptional<z.ZodString>;
|
|
74
76
|
}, z.core.$strip>;
|
package/dist/types/index.js
CHANGED
|
@@ -6,6 +6,7 @@ export const EmbeddingModelConfigSchema = z.object({
|
|
|
6
6
|
baseUrl: z.url().optional(),
|
|
7
7
|
apiKey: z.string(),
|
|
8
8
|
dimensions: z.number().int().positive().optional(),
|
|
9
|
+
batchSize: z.number().int().positive().optional(),
|
|
9
10
|
});
|
|
10
11
|
// ─── Knowledge Base ─────────────────────────────────────────
|
|
11
12
|
export const KnowledgeBaseMetadataSchema = z.object({
|
package/dist/types/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,+DAA+D;AAE/D,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,+DAA+D;AAE/D,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAClD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAClD,CAAC,CAAC;AAMH,+DAA+D;AAE/D,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;IACZ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE;IACnB,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE;CACpB,CAAC,CAAC;AAIH,+DAA+D;AAE/D,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;IACZ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACzC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACnD,eAAe,EAAE,CAAC,CAAC,IAAI,EAAE;IACzB,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE;IACnB,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE;CACpB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,sBAAsB,GAAG,wBAAwB,CAAC,IAAI,CAAC;IAClE,EAAE,EAAE,IAAI;IACR,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;CAChB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,sBAAsB,GAAG,sBAAsB,CAAC,OAAO,EAAE,CAAC;AAIvE,+DAA+D;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACpF,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACtD,CAAC,CAAC"}
|