plugin-knowledge-base 1.0.0 → 1.0.1

package/dist/node_modules/@langchain/textsplitters/dist/text_splitter.d.ts

@@ -1,84 +0,0 @@
-import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
-import * as tiktoken from "js-tiktoken";
-
-//#region src/text_splitter.d.ts
-interface TextSplitterParams {
-  chunkSize: number;
-  chunkOverlap: number;
-  keepSeparator: boolean;
-  lengthFunction?: ((text: string) => number) | ((text: string) => Promise<number>);
-}
-type TextSplitterChunkHeaderOptions = {
-  chunkHeader?: string;
-  chunkOverlapHeader?: string;
-  appendChunkOverlapHeader?: boolean;
-};
-declare abstract class TextSplitter extends BaseDocumentTransformer implements TextSplitterParams {
-  lc_namespace: string[];
-  chunkSize: number;
-  chunkOverlap: number;
-  keepSeparator: boolean;
-  lengthFunction: ((text: string) => number) | ((text: string) => Promise<number>);
-  constructor(fields?: Partial<TextSplitterParams>);
-  transformDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
-  abstract splitText(text: string): Promise<string[]>;
-  protected splitOnSeparator(text: string, separator: string): string[];
-  createDocuments(texts: string[],
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  metadatas?: Record<string, any>[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
-  private numberOfNewLines;
-  splitDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
-  private joinDocs;
-  mergeSplits(splits: string[], separator: string): Promise<string[]>;
-}
-interface CharacterTextSplitterParams extends TextSplitterParams {
-  separator: string;
-}
-declare class CharacterTextSplitter extends TextSplitter implements CharacterTextSplitterParams {
-  static lc_name(): string;
-  separator: string;
-  constructor(fields?: Partial<CharacterTextSplitterParams>);
-  splitText(text: string): Promise<string[]>;
-}
-interface RecursiveCharacterTextSplitterParams extends TextSplitterParams {
-  separators: string[];
-}
-declare const SupportedTextSplitterLanguages: readonly ["cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol"];
-type SupportedTextSplitterLanguage = (typeof SupportedTextSplitterLanguages)[number];
-declare class RecursiveCharacterTextSplitter extends TextSplitter implements RecursiveCharacterTextSplitterParams {
-  static lc_name(): string;
-  separators: string[];
-  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
-  private _splitText;
-  splitText(text: string): Promise<string[]>;
-  static fromLanguage(language: SupportedTextSplitterLanguage, options?: Partial<RecursiveCharacterTextSplitterParams>): RecursiveCharacterTextSplitter;
-  static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage): string[];
-}
-interface TokenTextSplitterParams extends TextSplitterParams {
-  encodingName: tiktoken.TiktokenEncoding;
-  allowedSpecial: "all" | Array<string>;
-  disallowedSpecial: "all" | Array<string>;
-}
-/**
- * Implementation of splitter which looks at tokens.
- */
-declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
-  static lc_name(): string;
-  encodingName: tiktoken.TiktokenEncoding;
-  allowedSpecial: "all" | Array<string>;
-  disallowedSpecial: "all" | Array<string>;
-  private tokenizer;
-  constructor(fields?: Partial<TokenTextSplitterParams>);
-  splitText(text: string): Promise<string[]>;
-}
-type MarkdownTextSplitterParams = TextSplitterParams;
-declare class MarkdownTextSplitter extends RecursiveCharacterTextSplitter implements MarkdownTextSplitterParams {
-  constructor(fields?: Partial<MarkdownTextSplitterParams>);
-}
-type LatexTextSplitterParams = TextSplitterParams;
-declare class LatexTextSplitter extends RecursiveCharacterTextSplitter implements LatexTextSplitterParams {
-  constructor(fields?: Partial<LatexTextSplitterParams>);
-}
-//#endregion
-export { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams };
-//# sourceMappingURL=text_splitter.d.ts.map

package/dist/node_modules/@langchain/textsplitters/dist/text_splitter.js

@@ -1,532 +0,0 @@
-import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
-import { getEncoding } from "@langchain/core/utils/tiktoken";
-
-//#region src/text_splitter.ts
-var TextSplitter = class extends BaseDocumentTransformer {
-	lc_namespace = [
-		"langchain",
-		"document_transformers",
-		"text_splitters"
-	];
-	chunkSize = 1e3;
-	chunkOverlap = 200;
-	keepSeparator = false;
-	lengthFunction;
-	constructor(fields) {
-		super(fields);
-		this.chunkSize = fields?.chunkSize ?? this.chunkSize;
-		this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
-		this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;
-		this.lengthFunction = fields?.lengthFunction ?? ((text) => text.length);
-		if (this.chunkOverlap >= this.chunkSize) throw new Error("Cannot have chunkOverlap >= chunkSize");
-	}
-	async transformDocuments(documents, chunkHeaderOptions = {}) {
-		return this.splitDocuments(documents, chunkHeaderOptions);
-	}
-	splitOnSeparator(text, separator) {
-		let splits;
-		if (separator) if (this.keepSeparator) {
-			const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, "\\$&");
-			splits = text.split(/* @__PURE__ */ new RegExp(`(?=${regexEscapedSeparator})`));
-		} else splits = text.split(separator);
-		else splits = text.split("");
-		return splits.filter((s) => s !== "");
-	}
-	async createDocuments(texts, metadatas = [], chunkHeaderOptions = {}) {
-		const _metadatas = metadatas.length > 0 ? metadatas : [...Array(texts.length)].map(() => ({}));
-		const { chunkHeader = "", chunkOverlapHeader = "(cont'd) ", appendChunkOverlapHeader = false } = chunkHeaderOptions;
-		const documents = new Array();
-		for (let i = 0; i < texts.length; i += 1) {
-			const text = texts[i];
-			let lineCounterIndex = 1;
-			let prevChunk = null;
-			let indexPrevChunk = -1;
-			for (const chunk of await this.splitText(text)) {
-				let pageContent = chunkHeader;
-				const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);
-				if (prevChunk === null) {
-					const newLinesBeforeFirstChunk = this.numberOfNewLines(text, 0, indexChunk);
-					lineCounterIndex += newLinesBeforeFirstChunk;
-				} else {
-					const indexEndPrevChunk = indexPrevChunk + await this.lengthFunction(prevChunk);
-					if (indexEndPrevChunk < indexChunk) {
-						const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexEndPrevChunk, indexChunk);
-						lineCounterIndex += numberOfIntermediateNewLines;
-					} else if (indexEndPrevChunk > indexChunk) {
-						const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
-						lineCounterIndex -= numberOfIntermediateNewLines;
-					}
-					if (appendChunkOverlapHeader) pageContent += chunkOverlapHeader;
-				}
-				const newLinesCount = this.numberOfNewLines(chunk);
-				const loc = _metadatas[i].loc && typeof _metadatas[i].loc === "object" ? { ..._metadatas[i].loc } : {};
-				loc.lines = {
-					from: lineCounterIndex,
-					to: lineCounterIndex + newLinesCount
-				};
-				const metadataWithLinesNumber = {
-					..._metadatas[i],
-					loc
-				};
-				pageContent += chunk;
-				documents.push(new Document({
-					pageContent,
-					metadata: metadataWithLinesNumber
-				}));
-				lineCounterIndex += newLinesCount;
-				prevChunk = chunk;
-				indexPrevChunk = indexChunk;
-			}
-		}
-		return documents;
-	}
-	numberOfNewLines(text, start, end) {
-		const textSection = text.slice(start, end);
-		return (textSection.match(/\n/g) || []).length;
-	}
-	async splitDocuments(documents, chunkHeaderOptions = {}) {
-		const selectedDocuments = documents.filter((doc) => doc.pageContent !== void 0);
-		const texts = selectedDocuments.map((doc) => doc.pageContent);
-		const metadatas = selectedDocuments.map((doc) => doc.metadata);
-		return this.createDocuments(texts, metadatas, chunkHeaderOptions);
-	}
-	joinDocs(docs, separator) {
-		const text = docs.join(separator).trim();
-		return text === "" ? null : text;
-	}
-	async mergeSplits(splits, separator) {
-		const docs = [];
-		const currentDoc = [];
-		let total = 0;
-		for (const d of splits) {
-			const _len = await this.lengthFunction(d);
-			if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
-				if (total > this.chunkSize) console.warn(`Created a chunk of size ${total}, +
-which is longer than the specified ${this.chunkSize}`);
-				if (currentDoc.length > 0) {
-					const doc$1 = this.joinDocs(currentDoc, separator);
-					if (doc$1 !== null) docs.push(doc$1);
-					while (total > this.chunkOverlap || total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0) {
-						total -= await this.lengthFunction(currentDoc[0]);
-						currentDoc.shift();
-					}
-				}
-			}
-			currentDoc.push(d);
-			total += _len;
-		}
-		const doc = this.joinDocs(currentDoc, separator);
-		if (doc !== null) docs.push(doc);
-		return docs;
-	}
-};
-var CharacterTextSplitter = class extends TextSplitter {
-	static lc_name() {
-		return "CharacterTextSplitter";
-	}
-	separator = "\n\n";
-	constructor(fields) {
-		super(fields);
-		this.separator = fields?.separator ?? this.separator;
-	}
-	async splitText(text) {
-		const splits = this.splitOnSeparator(text, this.separator);
-		return this.mergeSplits(splits, this.keepSeparator ? "" : this.separator);
-	}
-};
-const SupportedTextSplitterLanguages = [
-	"cpp",
-	"go",
-	"java",
-	"js",
-	"php",
-	"proto",
-	"python",
-	"rst",
-	"ruby",
-	"rust",
-	"scala",
-	"swift",
-	"markdown",
-	"latex",
-	"html",
-	"sol"
-];
-var RecursiveCharacterTextSplitter = class RecursiveCharacterTextSplitter extends TextSplitter {
-	static lc_name() {
-		return "RecursiveCharacterTextSplitter";
-	}
-	separators = [
-		"\n\n",
-		"\n",
-		" ",
-		""
-	];
-	constructor(fields) {
-		super(fields);
-		this.separators = fields?.separators ?? this.separators;
-		this.keepSeparator = fields?.keepSeparator ?? true;
-	}
-	async _splitText(text, separators) {
-		const finalChunks = [];
-		let separator = separators[separators.length - 1];
-		let newSeparators;
-		for (let i = 0; i < separators.length; i += 1) {
-			const s = separators[i];
-			if (s === "") {
-				separator = s;
-				break;
-			}
-			if (text.includes(s)) {
-				separator = s;
-				newSeparators = separators.slice(i + 1);
-				break;
-			}
-		}
-		const splits = this.splitOnSeparator(text, separator);
-		let goodSplits = [];
-		const _separator = this.keepSeparator ? "" : separator;
-		for (const s of splits) if (await this.lengthFunction(s) < this.chunkSize) goodSplits.push(s);
-		else {
-			if (goodSplits.length) {
-				const mergedText = await this.mergeSplits(goodSplits, _separator);
-				finalChunks.push(...mergedText);
-				goodSplits = [];
-			}
-			if (!newSeparators) finalChunks.push(s);
-			else {
-				const otherInfo = await this._splitText(s, newSeparators);
-				finalChunks.push(...otherInfo);
-			}
-		}
-		if (goodSplits.length) {
-			const mergedText = await this.mergeSplits(goodSplits, _separator);
-			finalChunks.push(...mergedText);
-		}
-		return finalChunks;
-	}
-	async splitText(text) {
-		return this._splitText(text, this.separators);
-	}
-	static fromLanguage(language, options) {
-		return new RecursiveCharacterTextSplitter({
-			...options,
-			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language)
-		});
-	}
-	static getSeparatorsForLanguage(language) {
-		if (language === "cpp") return [
-			"\nclass ",
-			"\nvoid ",
-			"\nint ",
-			"\nfloat ",
-			"\ndouble ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\nswitch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "go") return [
-			"\nfunc ",
-			"\nvar ",
-			"\nconst ",
-			"\ntype ",
-			"\nif ",
-			"\nfor ",
-			"\nswitch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "java") return [
-			"\nclass ",
-			"\npublic ",
-			"\nprotected ",
-			"\nprivate ",
-			"\nstatic ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\nswitch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "js") return [
-			"\nfunction ",
-			"\nconst ",
-			"\nlet ",
-			"\nvar ",
-			"\nclass ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\nswitch ",
-			"\ncase ",
-			"\ndefault ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "php") return [
-			"\nfunction ",
-			"\nclass ",
-			"\nif ",
-			"\nforeach ",
-			"\nwhile ",
-			"\ndo ",
-			"\nswitch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "proto") return [
-			"\nmessage ",
-			"\nservice ",
-			"\nenum ",
-			"\noption ",
-			"\nimport ",
-			"\nsyntax ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "python") return [
-			"\nclass ",
-			"\ndef ",
-			"\n	def ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "rst") return [
-			"\n===\n",
-			"\n---\n",
-			"\n***\n",
-			"\n.. ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "ruby") return [
-			"\ndef ",
-			"\nclass ",
-			"\nif ",
-			"\nunless ",
-			"\nwhile ",
-			"\nfor ",
-			"\ndo ",
-			"\nbegin ",
-			"\nrescue ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "rust") return [
-			"\nfn ",
-			"\nconst ",
-			"\nlet ",
-			"\nif ",
-			"\nwhile ",
-			"\nfor ",
-			"\nloop ",
-			"\nmatch ",
-			"\nconst ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "scala") return [
-			"\nclass ",
-			"\nobject ",
-			"\ndef ",
-			"\nval ",
-			"\nvar ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\nmatch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "swift") return [
-			"\nfunc ",
-			"\nclass ",
-			"\nstruct ",
-			"\nenum ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\ndo ",
-			"\nswitch ",
-			"\ncase ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "markdown") return [
-			"\n## ",
-			"\n### ",
-			"\n#### ",
-			"\n##### ",
-			"\n###### ",
-			"```\n\n",
-			"\n\n***\n\n",
-			"\n\n---\n\n",
-			"\n\n___\n\n",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "latex") return [
-			"\n\\chapter{",
-			"\n\\section{",
-			"\n\\subsection{",
-			"\n\\subsubsection{",
-			"\n\\begin{enumerate}",
-			"\n\\begin{itemize}",
-			"\n\\begin{description}",
-			"\n\\begin{list}",
-			"\n\\begin{quote}",
-			"\n\\begin{quotation}",
-			"\n\\begin{verse}",
-			"\n\\begin{verbatim}",
-			"\n\\begin{align}",
-			"$$",
-			"$",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else if (language === "html") return [
-			"<body>",
-			"<div>",
-			"<p>",
-			"<br>",
-			"<li>",
-			"<h1>",
-			"<h2>",
-			"<h3>",
-			"<h4>",
-			"<h5>",
-			"<h6>",
-			"<span>",
-			"<table>",
-			"<tr>",
-			"<td>",
-			"<th>",
-			"<ul>",
-			"<ol>",
-			"<header>",
-			"<footer>",
-			"<nav>",
-			"<head>",
-			"<style>",
-			"<script>",
-			"<meta>",
-			"<title>",
-			" ",
-			""
-		];
-		else if (language === "sol") return [
-			"\npragma ",
-			"\nusing ",
-			"\ncontract ",
-			"\ninterface ",
-			"\nlibrary ",
-			"\nconstructor ",
-			"\ntype ",
-			"\nfunction ",
-			"\nevent ",
-			"\nmodifier ",
-			"\nerror ",
-			"\nstruct ",
-			"\nenum ",
-			"\nif ",
-			"\nfor ",
-			"\nwhile ",
-			"\ndo while ",
-			"\nassembly ",
-			"\n\n",
-			"\n",
-			" ",
-			""
-		];
-		else throw new Error(`Language ${language} is not supported.`);
-	}
-};
-/**
-* Implementation of splitter which looks at tokens.
-*/
-var TokenTextSplitter = class extends TextSplitter {
-	static lc_name() {
-		return "TokenTextSplitter";
-	}
-	encodingName;
-	allowedSpecial;
-	disallowedSpecial;
-	tokenizer;
-	constructor(fields) {
-		super(fields);
-		this.encodingName = fields?.encodingName ?? "gpt2";
-		this.allowedSpecial = fields?.allowedSpecial ?? [];
-		this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
-	}
-	async splitText(text) {
-		if (!this.tokenizer) this.tokenizer = await getEncoding(this.encodingName);
-		const splits = [];
-		const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
-		let start_idx = 0;
-		while (start_idx < input_ids.length) {
-			if (start_idx > 0) start_idx -= this.chunkOverlap;
-			const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
-			const chunk_ids = input_ids.slice(start_idx, end_idx);
-			splits.push(this.tokenizer.decode(chunk_ids));
-			start_idx = end_idx;
-		}
-		return splits;
-	}
-};
-var MarkdownTextSplitter = class extends RecursiveCharacterTextSplitter {
-	constructor(fields) {
-		super({
-			...fields,
-			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("markdown")
-		});
-	}
-};
-var LatexTextSplitter = class extends RecursiveCharacterTextSplitter {
-	constructor(fields) {
-		super({
-			...fields,
-			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("latex")
-		});
-	}
-};
-
-//#endregion
-export { CharacterTextSplitter, LatexTextSplitter, MarkdownTextSplitter, RecursiveCharacterTextSplitter, SupportedTextSplitterLanguages, TextSplitter, TokenTextSplitter };
-//# sourceMappingURL=text_splitter.js.map

package/dist/node_modules/@langchain/textsplitters/package.json

@@ -1 +0,0 @@
-{"name":"@langchain/textsplitters","version":"1.0.1","description":"Various implementations of LangChain.js text splitters","author":"LangChain","license":"MIT","type":"module","engines":{"node":">=20"},"repository":{"type":"git","url":"git@github.com:langchain-ai/langchainjs.git"},"homepage":"https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-textsplitters/","dependencies":{"js-tiktoken":"^1.0.12"},"peerDependencies":{"@langchain/core":"^1.0.0"},"devDependencies":{"@tsconfig/recommended":"^1.0.3","@vitest/coverage-v8":"^3.2.4","dotenv":"^16.3.1","dpdm":"^3.14.0","eslint":"^9.34.0","prettier":"^2.8.3","rollup":"^4.5.2","typescript":"~5.8.3","vitest":"^3.2.4","@langchain/core":"1.1.0","@langchain/eslint":"0.1.1"},"publishConfig":{"access":"public"},"main":"./dist/index.cjs","types":"./dist/index.d.cts","exports":{".":{"input":"./src/index.ts","require":{"types":"./dist/index.d.cts","default":"./dist/index.cjs"},"import":{"types":"./dist/index.d.ts","default":"./dist/index.js"}},"./package.json":"./package.json"},"files":["dist/","CHANGELOG.md","README.md","LICENSE"],"module":"./dist/index.js","scripts":{"build":"turbo build:compile --filter @langchain/textsplitters --output-logs new-only","build:compile":"tsdown","lint:eslint":"eslint --cache src/","lint:dpdm":"dpdm --skip-dynamic-imports circular --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts","lint":"pnpm lint:eslint && pnpm lint:dpdm","lint:fix":"pnpm lint:eslint --fix && pnpm lint:dpdm","clean":"rm -rf .turbo dist/","test":"vitest --run","test:watch":"vitest --watch","format":"prettier --config .prettierrc --write \"src\"","format:check":"prettier --config .prettierrc --check \"src\""},"_lastModified":"2026-03-10T19:48:56.884Z"}