lfx-nextplaid 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lfx_nextplaid/__init__.py +26 -0
- lfx_nextplaid/components/nextplaid/__init__.py +15 -0
- lfx_nextplaid/components/nextplaid/nextplaid.py +278 -0
- lfx_nextplaid/components/nextplaid/vllm_multivector_embeddings.py +76 -0
- lfx_nextplaid/components/nextplaid/vllm_multivector_impl.py +183 -0
- lfx_nextplaid/extension.json +16 -0
- lfx_nextplaid-0.1.0.dist-info/METADATA +73 -0
- lfx_nextplaid-0.1.0.dist-info/RECORD +10 -0
- lfx_nextplaid-0.1.0.dist-info/WHEEL +4 -0
- lfx_nextplaid-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""lfx-nextplaid: NextPlaid multi-vector vector store bundle.
|
|
2
|
+
|
|
3
|
+
This package is the distribution unit ``lfx-nextplaid``. At runtime
|
|
4
|
+
Langflow's loader discovers ``extension.json`` shipped alongside this
|
|
5
|
+
``__init__.py`` and registers the bundle's two components under the
|
|
6
|
+
namespaced IDs:
|
|
7
|
+
|
|
8
|
+
* ``ext:nextplaid:NextPlaidVectorStoreComponent@official``
|
|
9
|
+
* ``ext:nextplaid:VllmMultivectorEmbeddingsComponent@official``
|
|
10
|
+
|
|
11
|
+
NextPlaid stores each document as a matrix of token embeddings
|
|
12
|
+
(ColBERT/ColPali-style late interaction, MaxSim scoring) backed by a
|
|
13
|
+
running NextPlaid server via the ``langchain-plaid`` client. The
|
|
14
|
+
companion ``VllmMultivectorEmbeddings`` component produces the token-matrix
|
|
15
|
+
embeddings NextPlaid ingests by calling vLLM's ``/pooling`` endpoint; the
|
|
16
|
+
two ship together because the multivector embeddings exist to feed
|
|
17
|
+
NextPlaid.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from lfx_nextplaid.components.nextplaid.nextplaid import NextPlaidVectorStoreComponent
|
|
21
|
+
from lfx_nextplaid.components.nextplaid.vllm_multivector_embeddings import VllmMultivectorEmbeddingsComponent
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"NextPlaidVectorStoreComponent",
|
|
25
|
+
"VllmMultivectorEmbeddingsComponent",
|
|
26
|
+
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Component re-exports for the ``nextplaid`` bundle.
|
|
2
|
+
|
|
3
|
+
Saved-flow migration entries that target ``lfx.components.nextplaid.<Class>``
|
|
4
|
+
(and the legacy ``lfx.components.vllm.<Class>`` path for the multivector
|
|
5
|
+
embeddings) resolve through this package, so the moved Component classes must
|
|
6
|
+
be importable from here by name.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .nextplaid import NextPlaidVectorStoreComponent
|
|
10
|
+
from .vllm_multivector_embeddings import VllmMultivectorEmbeddingsComponent
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"NextPlaidVectorStoreComponent",
|
|
14
|
+
"VllmMultivectorEmbeddingsComponent",
|
|
15
|
+
]
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import io
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
|
|
9
|
+
from lfx.helpers.data import docs_to_data
|
|
10
|
+
from lfx.io import (
|
|
11
|
+
BoolInput,
|
|
12
|
+
DropdownInput,
|
|
13
|
+
FloatInput,
|
|
14
|
+
HandleInput,
|
|
15
|
+
IntInput,
|
|
16
|
+
Output,
|
|
17
|
+
StrInput,
|
|
18
|
+
)
|
|
19
|
+
from lfx.schema.data import Data
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class NextPlaidVectorStoreComponent(LCVectorStoreComponent):
|
|
23
|
+
"""NextPlaid multi-vector (ColBERT/ColPali-style) vector store."""
|
|
24
|
+
|
|
25
|
+
display_name: str = "NextPlaid"
|
|
26
|
+
description: str = (
|
|
27
|
+
"Multi-vector (ColBERT/PLAID) vector store backed by NextPlaid. "
|
|
28
|
+
"Supports text retrieval via ColBERT models and image retrieval via "
|
|
29
|
+
"ColPali models. Connect a vLLM Multivector Embeddings component."
|
|
30
|
+
)
|
|
31
|
+
name = "NextPlaid"
|
|
32
|
+
icon = "NextPlaid"
|
|
33
|
+
|
|
34
|
+
inputs = [
|
|
35
|
+
StrInput(
|
|
36
|
+
name="url",
|
|
37
|
+
display_name="Server URL",
|
|
38
|
+
value="http://localhost:8080",
|
|
39
|
+
info="Base URL of the running NextPlaid server.",
|
|
40
|
+
),
|
|
41
|
+
StrInput(
|
|
42
|
+
name="index_name",
|
|
43
|
+
display_name="Index Name",
|
|
44
|
+
value="langflow",
|
|
45
|
+
info="Name of the index to create or connect to.",
|
|
46
|
+
),
|
|
47
|
+
DropdownInput(
|
|
48
|
+
name="nbits",
|
|
49
|
+
display_name="Quantization Bits",
|
|
50
|
+
options=["2", "4"],
|
|
51
|
+
value="4",
|
|
52
|
+
info="Bit-width for PLAID quantization. 4-bit gives better quality.",
|
|
53
|
+
advanced=True,
|
|
54
|
+
),
|
|
55
|
+
BoolInput(
|
|
56
|
+
name="create_index_if_not_exists",
|
|
57
|
+
display_name="Create Index If Not Exists",
|
|
58
|
+
value=True,
|
|
59
|
+
advanced=True,
|
|
60
|
+
),
|
|
61
|
+
FloatInput(
|
|
62
|
+
name="write_timeout",
|
|
63
|
+
display_name="Write Timeout (seconds)",
|
|
64
|
+
value=30.0,
|
|
65
|
+
info=(
|
|
66
|
+
"Seconds to wait for each batch to finish indexing. "
|
|
67
|
+
"0 = async (search may return empty on first run). "
|
|
68
|
+
"Recommended: 30+ when ingesting and searching in the same flow run."
|
|
69
|
+
),
|
|
70
|
+
advanced=True,
|
|
71
|
+
),
|
|
72
|
+
IntInput(
|
|
73
|
+
name="index_batch_size",
|
|
74
|
+
display_name="Index Batch Size",
|
|
75
|
+
value=500,
|
|
76
|
+
advanced=True,
|
|
77
|
+
info=(
|
|
78
|
+
"Documents per indexing request. PLAID builds its initial cluster "
|
|
79
|
+
"centroids from the first batch — larger batches produce better "
|
|
80
|
+
"retrieval quality. Approximate limits per 1GB request: "
|
|
81
|
+
"~26,000 text docs (ColBERT-small, 96-dim), "
|
|
82
|
+
"~2,000 PDF pages (ColPali, 1000-patch x 128-dim)."
|
|
83
|
+
),
|
|
84
|
+
),
|
|
85
|
+
*LCVectorStoreComponent.inputs,
|
|
86
|
+
HandleInput(
|
|
87
|
+
name="embedding",
|
|
88
|
+
display_name="Embedding (Multivector)",
|
|
89
|
+
input_types=["Embeddings"],
|
|
90
|
+
info=(
|
|
91
|
+
"Connect a vLLM Multivector Embeddings component. "
|
|
92
|
+
"For image indexing, the model must support embed_images() "
|
|
93
|
+
"(requires a ColPali model such as ModernVBERT/colmodernvbert)."
|
|
94
|
+
),
|
|
95
|
+
),
|
|
96
|
+
IntInput(
|
|
97
|
+
name="number_of_results",
|
|
98
|
+
display_name="Number of Results",
|
|
99
|
+
info="Number of results to return from similarity search.",
|
|
100
|
+
value=4,
|
|
101
|
+
advanced=True,
|
|
102
|
+
),
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
# Declared explicitly (rather than inherited from LCVectorStoreComponent) so the
|
|
106
|
+
# static extension validator can resolve the component's output methods; mirrors
|
|
107
|
+
# the base outputs and keeps the search output usable as an agent tool.
|
|
108
|
+
outputs = [
|
|
109
|
+
Output(
|
|
110
|
+
display_name="Search Results",
|
|
111
|
+
name="search_results",
|
|
112
|
+
method="search_documents",
|
|
113
|
+
tool_mode=True,
|
|
114
|
+
),
|
|
115
|
+
Output(
|
|
116
|
+
display_name="Table",
|
|
117
|
+
name="dataframe",
|
|
118
|
+
method="as_dataframe",
|
|
119
|
+
),
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def _data_to_pil(item: Data):
|
|
124
|
+
"""Extract a PIL Image from a Data object produced by PdfPagesToImages."""
|
|
125
|
+
from PIL import Image as PILImage
|
|
126
|
+
|
|
127
|
+
b64 = item.data.get("base64_image", "")
|
|
128
|
+
if not b64:
|
|
129
|
+
msg = f"Data object with content_type '{item.data.get('content_type')}' has no 'base64_image' field."
|
|
130
|
+
raise ValueError(msg)
|
|
131
|
+
return PILImage.open(io.BytesIO(base64.b64decode(b64)))
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def _stable_image_id(item: Data, index_name: str, position: int) -> str:
|
|
135
|
+
"""Derive a stable upsert ID from available metadata."""
|
|
136
|
+
return (
|
|
137
|
+
item.data.get("document_id")
|
|
138
|
+
or hashlib.sha256(
|
|
139
|
+
f"{index_name}:{item.data.get('source', '')}:{item.data.get('page', position)}".encode()
|
|
140
|
+
).hexdigest()
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _stable_text_id(doc, index_name: str) -> str:
|
|
145
|
+
"""Derive a stable upsert ID for a text document.
|
|
146
|
+
|
|
147
|
+
Prefers explicit id metadata; otherwise fingerprints the content
|
|
148
|
+
together with its source/page so identical chunk text from different
|
|
149
|
+
sources does not collide onto the same vector.
|
|
150
|
+
"""
|
|
151
|
+
explicit = doc.metadata.get("document_id") or doc.metadata.get("doc_id") or doc.metadata.get("id")
|
|
152
|
+
if explicit:
|
|
153
|
+
return explicit
|
|
154
|
+
fingerprint = (
|
|
155
|
+
f"{index_name}:"
|
|
156
|
+
f"{doc.metadata.get('source', '')}:"
|
|
157
|
+
f"{doc.metadata.get('page', '')}:"
|
|
158
|
+
f"{(doc.page_content or '').strip()}"
|
|
159
|
+
)
|
|
160
|
+
return hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()
|
|
161
|
+
|
|
162
|
+
@staticmethod
|
|
163
|
+
def _stable_raw_image_id(image, index_name: str) -> str:
|
|
164
|
+
"""Fingerprint a raw PIL image so re-ingesting it upserts instead of relying on batch position."""
|
|
165
|
+
if hasattr(image, "convert") and hasattr(image, "tobytes"):
|
|
166
|
+
fingerprint = hashlib.sha256(image.convert("RGB").tobytes()).hexdigest()
|
|
167
|
+
else:
|
|
168
|
+
fingerprint = hashlib.sha256(repr(image).encode("utf-8")).hexdigest()
|
|
169
|
+
return hashlib.sha256(f"{index_name}:raw_image:{fingerprint}".encode()).hexdigest()
|
|
170
|
+
|
|
171
|
+
@check_cached_vector_store
|
|
172
|
+
def build_vector_store(self):
|
|
173
|
+
try:
|
|
174
|
+
from langchain_plaid.vectorstores import NextPlaidVectorStore as LangchainNextPlaid
|
|
175
|
+
except ImportError as e:
|
|
176
|
+
msg = "Could not import langchain-plaid. Install it with: pip install langchain-plaid"
|
|
177
|
+
raise ImportError(msg) from e
|
|
178
|
+
|
|
179
|
+
if self.embedding is None:
|
|
180
|
+
msg = (
|
|
181
|
+
"No embedding model connected. Connect a vLLM Multivector Embeddings component to the Embedding input."
|
|
182
|
+
)
|
|
183
|
+
raise ValueError(msg)
|
|
184
|
+
|
|
185
|
+
nextplaid_store = LangchainNextPlaid(
|
|
186
|
+
url=self.url,
|
|
187
|
+
index_name=self.index_name,
|
|
188
|
+
embedding=self.embedding,
|
|
189
|
+
nbits=int(self.nbits),
|
|
190
|
+
create_index_if_not_exists=self.create_index_if_not_exists,
|
|
191
|
+
write_timeout=self.write_timeout,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
self.ingest_data = self._prepare_ingest_data()
|
|
195
|
+
|
|
196
|
+
# Flatten — Langflow wraps ingest data in a list: [[doc1, doc2, ...]]
|
|
197
|
+
raw_inputs: list[Any] = []
|
|
198
|
+
for item in self.ingest_data or []:
|
|
199
|
+
if isinstance(item, list):
|
|
200
|
+
raw_inputs.extend(item)
|
|
201
|
+
else:
|
|
202
|
+
raw_inputs.append(item)
|
|
203
|
+
|
|
204
|
+
if not raw_inputs:
|
|
205
|
+
return nextplaid_store
|
|
206
|
+
|
|
207
|
+
# Route items into text documents or images.
|
|
208
|
+
# A Data object is treated as an image when it carries content_type: image/*
|
|
209
|
+
# (set by PdfPagesToImages). Everything else is a text document.
|
|
210
|
+
text_docs = []
|
|
211
|
+
image_data_items: list[Data] = [] # Data objects wrapping images
|
|
212
|
+
raw_image_items: list[Any] = [] # raw PIL Images (advanced usage)
|
|
213
|
+
|
|
214
|
+
for item in raw_inputs:
|
|
215
|
+
if isinstance(item, Data):
|
|
216
|
+
if item.data.get("content_type", "").startswith("image/"):
|
|
217
|
+
image_data_items.append(item)
|
|
218
|
+
else:
|
|
219
|
+
doc = item.to_lc_document()
|
|
220
|
+
if doc.id is None:
|
|
221
|
+
doc.id = self._stable_text_id(doc, self.index_name)
|
|
222
|
+
text_docs.append(doc)
|
|
223
|
+
else:
|
|
224
|
+
# Raw PIL Image passed directly (e.g. from a custom component)
|
|
225
|
+
raw_image_items.append(item)
|
|
226
|
+
|
|
227
|
+
batch_size = max(self.index_batch_size or 500, 1)
|
|
228
|
+
|
|
229
|
+
# ── Text ingestion ────────────────────────────────────────────────────
|
|
230
|
+
if text_docs:
|
|
231
|
+
for i in range(0, len(text_docs), batch_size):
|
|
232
|
+
nextplaid_store.add_documents(text_docs[i : i + batch_size])
|
|
233
|
+
|
|
234
|
+
# ── Image ingestion from Data objects (PdfPagesToImages output) ───────
|
|
235
|
+
if image_data_items:
|
|
236
|
+
if not hasattr(self.embedding, "embed_images"):
|
|
237
|
+
msg = (
|
|
238
|
+
f"{type(self.embedding).__name__} does not implement embed_images()."
|
|
239
|
+
" Use a ColPali-compatible model (e.g. ModernVBERT/colmodernvbert)."
|
|
240
|
+
)
|
|
241
|
+
raise TypeError(msg)
|
|
242
|
+
|
|
243
|
+
for i in range(0, len(image_data_items), batch_size):
|
|
244
|
+
batch = image_data_items[i : i + batch_size]
|
|
245
|
+
|
|
246
|
+
pil_images = [self._data_to_pil(item) for item in batch]
|
|
247
|
+
metadatas = [{k: v for k, v in item.data.items() if k != "base64_image"} for item in batch]
|
|
248
|
+
ids = [self._stable_image_id(item, self.index_name, i + j) for j, item in enumerate(batch)]
|
|
249
|
+
nextplaid_store.add_images(pil_images, metadatas=metadatas, ids=ids)
|
|
250
|
+
|
|
251
|
+
# ── Raw PIL image ingestion (advanced / programmatic use) ─────────────
|
|
252
|
+
if raw_image_items:
|
|
253
|
+
if not hasattr(self.embedding, "embed_images"):
|
|
254
|
+
msg = (
|
|
255
|
+
f"{type(self.embedding).__name__} does not implement embed_images()."
|
|
256
|
+
" Use a ColPali-compatible model (e.g. ModernVBERT/colmodernvbert)."
|
|
257
|
+
)
|
|
258
|
+
raise TypeError(msg)
|
|
259
|
+
|
|
260
|
+
for i in range(0, len(raw_image_items), batch_size):
|
|
261
|
+
batch = raw_image_items[i : i + batch_size]
|
|
262
|
+
ids = [self._stable_raw_image_id(image, self.index_name) for image in batch]
|
|
263
|
+
nextplaid_store.add_images(batch, ids=ids)
|
|
264
|
+
|
|
265
|
+
return nextplaid_store
|
|
266
|
+
|
|
267
|
+
def search_documents(self) -> list[Data]:
|
|
268
|
+
vector_store = self.build_vector_store()
|
|
269
|
+
|
|
270
|
+
if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
|
|
271
|
+
docs = vector_store.similarity_search(
|
|
272
|
+
query=self.search_query,
|
|
273
|
+
k=self.number_of_results,
|
|
274
|
+
)
|
|
275
|
+
data = docs_to_data(docs)
|
|
276
|
+
self.status = data
|
|
277
|
+
return data
|
|
278
|
+
return []
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from lfx.base.embeddings.model import LCEmbeddingsModel
|
|
4
|
+
|
|
5
|
+
# Kept as a runtime import (not under TYPE_CHECKING): the framework resolves
|
|
6
|
+
# build_embeddings()'s ``-> Embeddings`` return hint via get_type_hints() at runtime.
|
|
7
|
+
from lfx.field_typing import Embeddings # noqa: TC002
|
|
8
|
+
from lfx.io import FloatInput, IntInput, MessageTextInput, SecretStrInput
|
|
9
|
+
from lfx_nextplaid.components.nextplaid.vllm_multivector_impl import VllmMultivectorEmbeddings
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class VllmMultivectorEmbeddingsComponent(LCEmbeddingsModel):
|
|
13
|
+
display_name = "vLLM Multivector Embeddings"
|
|
14
|
+
description = (
|
|
15
|
+
"Multi-vector (ColBERT/ColPali-style) token embeddings via vLLM's /pooling endpoint. "
|
|
16
|
+
"Compatible with text-only ColBERT models and multi-modal ColPali models. "
|
|
17
|
+
"Required for use with the NextPlaid vector store."
|
|
18
|
+
)
|
|
19
|
+
icon = "vLLM"
|
|
20
|
+
name = "VllmMultivectorEmbeddings"
|
|
21
|
+
|
|
22
|
+
inputs = [
|
|
23
|
+
MessageTextInput(
|
|
24
|
+
name="model_name",
|
|
25
|
+
display_name="Model Name",
|
|
26
|
+
advanced=False,
|
|
27
|
+
info=(
|
|
28
|
+
"Multi-vector model served by vLLM. Examples:\n"
|
|
29
|
+
"Text (ColBERT): answerdotai/answerai-colbert-small-v1\n"
|
|
30
|
+
"Text + Images (ColPali): ModernVBERT/colmodernvbert"
|
|
31
|
+
),
|
|
32
|
+
value="answerdotai/answerai-colbert-small-v1",
|
|
33
|
+
),
|
|
34
|
+
MessageTextInput(
|
|
35
|
+
name="api_base",
|
|
36
|
+
display_name="vLLM API Base",
|
|
37
|
+
advanced=False,
|
|
38
|
+
info=(
|
|
39
|
+
"Base URL of the vLLM server (no /v1 suffix). "
|
|
40
|
+
"Start vLLM with: vllm serve <model> --runner pooling "
|
|
41
|
+
'--pooler-config \'{"task": "token_embed"}\''
|
|
42
|
+
),
|
|
43
|
+
value="http://localhost:8000",
|
|
44
|
+
),
|
|
45
|
+
SecretStrInput(
|
|
46
|
+
name="api_key",
|
|
47
|
+
display_name="API Key",
|
|
48
|
+
info="API key for the vLLM server. Leave empty for local servers.",
|
|
49
|
+
advanced=False,
|
|
50
|
+
value="",
|
|
51
|
+
required=False,
|
|
52
|
+
),
|
|
53
|
+
FloatInput(
|
|
54
|
+
name="request_timeout",
|
|
55
|
+
display_name="Request Timeout",
|
|
56
|
+
advanced=True,
|
|
57
|
+
value=60.0,
|
|
58
|
+
info="Timeout in seconds for each request to the vLLM API.",
|
|
59
|
+
),
|
|
60
|
+
IntInput(
|
|
61
|
+
name="max_retries",
|
|
62
|
+
display_name="Max Retries",
|
|
63
|
+
advanced=True,
|
|
64
|
+
value=3,
|
|
65
|
+
info="Number of times to retry a failed request before raising an error.",
|
|
66
|
+
),
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
def build_embeddings(self) -> Embeddings:
|
|
70
|
+
return VllmMultivectorEmbeddings(
|
|
71
|
+
url=self.api_base or "http://localhost:8000",
|
|
72
|
+
model=self.model_name,
|
|
73
|
+
api_key=self.api_key or "",
|
|
74
|
+
timeout=self.request_timeout or 60.0,
|
|
75
|
+
max_retries=self.max_retries or 1,
|
|
76
|
+
)
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import io
|
|
5
|
+
import time
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
from langchain_core.embeddings import Embeddings as LCEmbeddings
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from PIL.Image import Image as PILImage
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class VllmMultivectorEmbeddings(LCEmbeddings):
|
|
16
|
+
"""Multi-vector embeddings via vLLM's /pooling endpoint.
|
|
17
|
+
|
|
18
|
+
Works with any multi-vector model served by vLLM including:
|
|
19
|
+
Text ColBERT : answerdotai/answerai-colbert-small-v1
|
|
20
|
+
lightonai/ColBERT-Zero
|
|
21
|
+
Multi-modal : ModernVBERT/colmodernvbert
|
|
22
|
+
vidore/colqwen2-v1.0
|
|
23
|
+
|
|
24
|
+
vLLM must be started with --runner pooling and the correct pooler config:
|
|
25
|
+
vllm serve <model> --runner pooling
|
|
26
|
+
|
|
27
|
+
For token-level (multi-vector) output, also pass:
|
|
28
|
+
--pooler-config '{"task": "token_embed"}'
|
|
29
|
+
|
|
30
|
+
Note: the `task` field in the HTTP request body is deprecated in vLLM >= 0.20.
|
|
31
|
+
Set it at server startup instead.
|
|
32
|
+
|
|
33
|
+
Shape contract (required by NextPlaid):
|
|
34
|
+
embed_documents(texts) -> List[List[List[float]]] (n_docs, n_tokens, dim)
|
|
35
|
+
embed_query(text) -> List[List[float]] (n_tokens, dim)
|
|
36
|
+
embed_images(images) -> List[List[List[float]]] (n_imgs, n_patches, dim)
|
|
37
|
+
— only available for multi-modal ColPali models
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
url: str,
|
|
43
|
+
model: str,
|
|
44
|
+
api_key: str = "",
|
|
45
|
+
timeout: float = 60.0,
|
|
46
|
+
max_retries: int = 1,
|
|
47
|
+
) -> None:
|
|
48
|
+
self.url = url.rstrip("/")
|
|
49
|
+
self.model = model
|
|
50
|
+
self.api_key = api_key
|
|
51
|
+
self.timeout = timeout
|
|
52
|
+
self.max_retries = max_retries
|
|
53
|
+
self._headers: dict = {
|
|
54
|
+
"Content-Type": "application/json",
|
|
55
|
+
**({"Authorization": f"Bearer {api_key}"} if api_key else {}),
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
def __repr__(self) -> str:
|
|
59
|
+
return f"VllmMultivectorEmbeddings(url={self.url!r}, model={self.model!r})"
|
|
60
|
+
|
|
61
|
+
def __hash__(self) -> int:
|
|
62
|
+
return hash((self.url, self.model, self.api_key))
|
|
63
|
+
|
|
64
|
+
def __eq__(self, other: object) -> bool:
|
|
65
|
+
if not isinstance(other, type(self)):
|
|
66
|
+
return False
|
|
67
|
+
return (self.url, self.model, self.api_key) == (other.url, other.model, other.api_key)
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def _extract_embedding_rows(payload: object) -> list[dict]:
|
|
71
|
+
"""Validate the vLLM /pooling envelope before nested indexing.
|
|
72
|
+
|
|
73
|
+
Raises a clear ``RuntimeError`` instead of letting a malformed or
|
|
74
|
+
partial response crash with ``KeyError``/``IndexError`` downstream.
|
|
75
|
+
"""
|
|
76
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
77
|
+
if not isinstance(data, list) or not data:
|
|
78
|
+
msg = "vLLM /pooling response missing non-empty 'data' list"
|
|
79
|
+
raise RuntimeError(msg)
|
|
80
|
+
return data
|
|
81
|
+
|
|
82
|
+
def _post_pooling(self, input_data: list) -> list[list[list[float]]]:
|
|
83
|
+
last_exc: Exception | None = None
|
|
84
|
+
for attempt in range(max(self.max_retries, 1)):
|
|
85
|
+
try:
|
|
86
|
+
resp = requests.post(
|
|
87
|
+
f"{self.url}/pooling",
|
|
88
|
+
json={"model": self.model, "input": input_data},
|
|
89
|
+
headers=self._headers,
|
|
90
|
+
timeout=self.timeout,
|
|
91
|
+
)
|
|
92
|
+
resp.raise_for_status()
|
|
93
|
+
rows = self._extract_embedding_rows(resp.json())
|
|
94
|
+
try:
|
|
95
|
+
results = sorted(rows, key=lambda x: x["index"])
|
|
96
|
+
return [item["data"] for item in results]
|
|
97
|
+
except (KeyError, TypeError) as exc:
|
|
98
|
+
msg = "vLLM /pooling response has invalid embedding row shape"
|
|
99
|
+
raise RuntimeError(msg) from exc
|
|
100
|
+
except requests.HTTPError as exc:
|
|
101
|
+
valid_client_status_codes = 500
|
|
102
|
+
if exc.response is not None and exc.response.status_code < valid_client_status_codes:
|
|
103
|
+
raise # don't retry 4xx — surface immediately
|
|
104
|
+
last_exc = exc
|
|
105
|
+
if attempt < self.max_retries - 1:
|
|
106
|
+
time.sleep(2**attempt)
|
|
107
|
+
except requests.RequestException as exc:
|
|
108
|
+
last_exc = exc
|
|
109
|
+
if attempt < self.max_retries - 1:
|
|
110
|
+
time.sleep(2**attempt)
|
|
111
|
+
raise last_exc # type: ignore[misc]
|
|
112
|
+
|
|
113
|
+
@staticmethod
|
|
114
|
+
def _pil_to_content_item(img: PILImage) -> dict:
|
|
115
|
+
"""Convert a PIL Image to an OpenAI Vision API content item.
|
|
116
|
+
|
|
117
|
+
This is the format vLLM expects for multi-modal pooling input.
|
|
118
|
+
"""
|
|
119
|
+
buf = io.BytesIO()
|
|
120
|
+
img.convert("RGB").save(buf, format="JPEG")
|
|
121
|
+
b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
|
122
|
+
return {
|
|
123
|
+
"type": "image_url",
|
|
124
|
+
"image_url": {"url": f"data:image/jpeg;base64,{b64}"},
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
def embed_documents(self, texts: list[str]) -> list[list[list[float]]]: # type: ignore[override]
|
|
128
|
+
"""Encode text documents. Returns one token matrix per document."""
|
|
129
|
+
if not texts:
|
|
130
|
+
return []
|
|
131
|
+
return self._post_pooling(texts)
|
|
132
|
+
|
|
133
|
+
def embed_query(self, text: str) -> list[list[float]]: # type: ignore[override]
|
|
134
|
+
"""Encode a single query string. Returns one token matrix."""
|
|
135
|
+
return self._post_pooling([text])[0]
|
|
136
|
+
|
|
137
|
+
def embed_images(self, images: list[PILImage]) -> list[list[list[float]]]:
|
|
138
|
+
if not images:
|
|
139
|
+
return []
|
|
140
|
+
|
|
141
|
+
embeddings: list[list[list[float]]] = []
|
|
142
|
+
for img in images:
|
|
143
|
+
last_exc: Exception | None = None
|
|
144
|
+
for attempt in range(max(self.max_retries, 1)):
|
|
145
|
+
try:
|
|
146
|
+
resp = requests.post(
|
|
147
|
+
f"{self.url}/pooling",
|
|
148
|
+
json={
|
|
149
|
+
"model": self.model,
|
|
150
|
+
"messages": [
|
|
151
|
+
{
|
|
152
|
+
"role": "user",
|
|
153
|
+
"content": [self._pil_to_content_item(img)],
|
|
154
|
+
}
|
|
155
|
+
],
|
|
156
|
+
},
|
|
157
|
+
headers=self._headers,
|
|
158
|
+
timeout=self.timeout,
|
|
159
|
+
)
|
|
160
|
+
resp.raise_for_status()
|
|
161
|
+
rows = self._extract_embedding_rows(resp.json())
|
|
162
|
+
try:
|
|
163
|
+
embeddings.append(rows[0]["data"])
|
|
164
|
+
except (KeyError, TypeError, IndexError) as exc:
|
|
165
|
+
msg = "vLLM /pooling image response has invalid embedding row shape"
|
|
166
|
+
raise RuntimeError(msg) from exc
|
|
167
|
+
break
|
|
168
|
+
except requests.HTTPError as exc:
|
|
169
|
+
valid_client_status_codes = 500
|
|
170
|
+
if exc.response is not None and exc.response.status_code < valid_client_status_codes:
|
|
171
|
+
msg = f"vLLM {exc.response.status_code}: {exc.response.text}"
|
|
172
|
+
raise RuntimeError(msg) from exc
|
|
173
|
+
last_exc = exc
|
|
174
|
+
if attempt < self.max_retries - 1:
|
|
175
|
+
time.sleep(2**attempt)
|
|
176
|
+
except requests.RequestException as exc:
|
|
177
|
+
last_exc = exc
|
|
178
|
+
if attempt < self.max_retries - 1:
|
|
179
|
+
time.sleep(2**attempt)
|
|
180
|
+
else:
|
|
181
|
+
raise last_exc # type: ignore[misc]
|
|
182
|
+
|
|
183
|
+
return embeddings
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://schemas.langflow.org/extension/v1.json",
|
|
3
|
+
"id": "lfx-nextplaid",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"name": "NextPlaid",
|
|
6
|
+
"description": "NextPlaid multi-vector (ColBERT/PLAID) vector store and its companion vLLM multivector embeddings as a standalone Langflow Extension Bundle.",
|
|
7
|
+
"lfx": {
|
|
8
|
+
"compat": ["1"]
|
|
9
|
+
},
|
|
10
|
+
"bundles": [
|
|
11
|
+
{
|
|
12
|
+
"name": "nextplaid",
|
|
13
|
+
"path": "components/nextplaid"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lfx-nextplaid
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: NextPlaid multi-vector vector store (+ vLLM multivector embeddings) as a standalone Langflow Extension Bundle.
|
|
5
|
+
Project-URL: Homepage, https://github.com/langflow-ai/langflow
|
|
6
|
+
Project-URL: Documentation, https://docs.langflow.org/extensions
|
|
7
|
+
Project-URL: Repository, https://github.com/langflow-ai/langflow
|
|
8
|
+
Author-email: Langflow <contact@langflow.org>
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: bundle,colbert,extension,langflow,lfx,multi-vector,nextplaid,plaid,vector-store
|
|
11
|
+
Requires-Python: <3.15,>=3.10
|
|
12
|
+
Requires-Dist: langchain-plaid<1.0.0,>=0.1.0
|
|
13
|
+
Requires-Dist: lfx<2.0.0,>=1.11.0.dev0
|
|
14
|
+
Requires-Dist: pillow>=10.0.0
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# lfx-nextplaid
|
|
18
|
+
|
|
19
|
+
NextPlaid multi-vector vector store (plus its companion vLLM multivector
|
|
20
|
+
embeddings) as a standalone Langflow Extension Bundle.
|
|
21
|
+
|
|
22
|
+
NextPlaid stores each document as a *matrix* of token embeddings rather than
|
|
23
|
+
a single vector, enabling ColBERT/ColPali-style late interaction (MaxSim
|
|
24
|
+
scoring) for significantly higher retrieval quality on semantic search.
|
|
25
|
+
The bundle ships two components:
|
|
26
|
+
|
|
27
|
+
- **`NextPlaidVectorStoreComponent`** — vector store backed by a running
|
|
28
|
+
[NextPlaid server](https://github.com/meetdoshi90/next-plaid) via the
|
|
29
|
+
[`langchain-plaid`](https://pypi.org/project/langchain-plaid/) client.
|
|
30
|
+
Supports text (ColBERT) and image (ColPali) ingestion with full upsert
|
|
31
|
+
semantics via stable document IDs.
|
|
32
|
+
- **`VllmMultivectorEmbeddingsComponent`** — produces the token-matrix
|
|
33
|
+
embeddings NextPlaid ingests by calling vLLM's `/pooling` endpoint with
|
|
34
|
+
`task: token_embed`. Compatible with ColBERT models such as
|
|
35
|
+
`answerdotai/answerai-colbert-small-v1`.
|
|
36
|
+
|
|
37
|
+
## Requirements
|
|
38
|
+
|
|
39
|
+
- A running [NextPlaid server](https://github.com/meetdoshi90/next-plaid).
|
|
40
|
+
- A running [vLLM server](https://docs.vllm.ai/) with a ColBERT-compatible
|
|
41
|
+
model loaded via `--runner pooling`.
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install lfx-nextplaid
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The bundle is registered automatically via the `langflow.extensions`
|
|
50
|
+
entry-point. After install, restart your Langflow server; the components
|
|
51
|
+
appear in the palette under the `nextplaid` bundle group.
|
|
52
|
+
|
|
53
|
+
## Develop
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
cd src/bundles/nextplaid
|
|
57
|
+
pip install -e .
|
|
58
|
+
lfx extension validate src/lfx_nextplaid
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Manifest
|
|
62
|
+
|
|
63
|
+
The extension manifest is shipped at `src/lfx_nextplaid/extension.json` and
|
|
64
|
+
points at the bundle at `components/nextplaid`. Components register under the
|
|
65
|
+
canonical namespaced IDs `ext:nextplaid:NextPlaidVectorStoreComponent@official`
|
|
66
|
+
and `ext:nextplaid:VllmMultivectorEmbeddingsComponent@official`.
|
|
67
|
+
|
|
68
|
+
## Migration
|
|
69
|
+
|
|
70
|
+
Saved flows referencing the legacy class names or the old import paths under
|
|
71
|
+
`lfx.components.nextplaid.*` / `lfx.components.vllm.VllmMultivectorEmbeddingsComponent`
|
|
72
|
+
are rewritten to the new namespaced IDs by the migration table in
|
|
73
|
+
`src/lfx/src/lfx/extension/migration/migration_table.json`.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
lfx_nextplaid/__init__.py,sha256=_nDTDeT8qzcW6kZI8UfnP-qAMx2sOwuSEJQq-utD-64,1133
|
|
2
|
+
lfx_nextplaid/extension.json,sha256=MSt-OHZJXgj7vfLQfP7NRoyKR1awX2_a5eA_2Dt0ZeA,429
|
|
3
|
+
lfx_nextplaid/components/nextplaid/__init__.py,sha256=M-huINV6lfNW3BYV4J7aB2yRlqewmkuqB6G-3iuRQAU,545
|
|
4
|
+
lfx_nextplaid/components/nextplaid/nextplaid.py,sha256=3px9zAj7_5S72IGg--9vO7DmBSdRgsNpIcnopLQqArU,10977
|
|
5
|
+
lfx_nextplaid/components/nextplaid/vllm_multivector_embeddings.py,sha256=0xNtrHbwEgJXwF7Cko7iy1i0co2DMPrW73XtEfCDD5w,2841
|
|
6
|
+
lfx_nextplaid/components/nextplaid/vllm_multivector_impl.py,sha256=7bzwf4h7fD37p2rnWHfXQ7jeP9HyM2HhP1L7Gp-HV2I,7418
|
|
7
|
+
lfx_nextplaid-0.1.0.dist-info/METADATA,sha256=55isHQFK4MdI876rpsJcbzGWSjSXg8-u2bKtmTdBq-Q,2874
|
|
8
|
+
lfx_nextplaid-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
9
|
+
lfx_nextplaid-0.1.0.dist-info/entry_points.txt,sha256=aj3ETYatcieUGi7bvAaSpOBL7wlwMJaMcsOy1n0DuJ8,52
|
|
10
|
+
lfx_nextplaid-0.1.0.dist-info/RECORD,,
|