swarmauri_parser_bertembedding 0.8.4.dev2__py3-none-any.whl → 0.11.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swarmauri_parser_bertembedding/BERTEmbeddingParser.py +17 -7
- swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/METADATA +204 -0
- swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/RECORD +7 -0
- {swarmauri_parser_bertembedding-0.8.4.dev2.dist-info → swarmauri_parser_bertembedding-0.11.0.dev1.dist-info}/WHEEL +1 -1
- swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/METADATA +0 -139
- swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/RECORD +0 -7
- {swarmauri_parser_bertembedding-0.8.4.dev2.dist-info → swarmauri_parser_bertembedding-0.11.0.dev1.dist-info}/entry_points.txt +0 -0
- {swarmauri_parser_bertembedding-0.8.4.dev2.dist-info → swarmauri_parser_bertembedding-0.11.0.dev1.dist-info}/licenses/LICENSE +0 -0
|
@@ -15,7 +15,8 @@ class BERTEmbeddingParser(ParserBase):
|
|
|
15
15
|
"""
|
|
16
16
|
A parser that transforms input text into document embeddings using BERT.
|
|
17
17
|
|
|
18
|
-
This parser tokenizes the input text, passes it through a pre-trained BERT
|
|
18
|
+
This parser tokenizes the input text, passes it through a pre-trained BERT
|
|
19
|
+
model,
|
|
19
20
|
and uses the resulting embeddings as the document content.
|
|
20
21
|
"""
|
|
21
22
|
|
|
@@ -41,34 +42,43 @@ class BERTEmbeddingParser(ParserBase):
|
|
|
41
42
|
Tokenizes input data and generates embeddings using a BERT model.
|
|
42
43
|
|
|
43
44
|
Parameters:
|
|
44
|
-
- data (Union[str, Any]): Input data, expected to be a single string or
|
|
45
|
+
- data (Union[str, Any]): Input data, expected to be a single string or
|
|
46
|
+
batch of strings.
|
|
45
47
|
|
|
46
48
|
Returns:
|
|
47
|
-
- List[IDocument]: A list containing a single IDocument instance with
|
|
49
|
+
- List[IDocument]: A list containing a single IDocument instance with
|
|
50
|
+
BERT embeddings as content.
|
|
48
51
|
"""
|
|
49
52
|
|
|
50
53
|
# Tokenization
|
|
51
54
|
inputs = self._tokenizer(
|
|
52
|
-
data,
|
|
55
|
+
data,
|
|
56
|
+
return_tensors="pt",
|
|
57
|
+
padding=True,
|
|
58
|
+
truncation=True,
|
|
59
|
+
max_length=512,
|
|
53
60
|
)
|
|
54
61
|
|
|
55
62
|
# Generate embeddings
|
|
56
63
|
with torch.no_grad():
|
|
57
64
|
outputs = self._model(**inputs)
|
|
58
65
|
|
|
59
|
-
# Use the last hidden state as document embeddings (batch_size,
|
|
66
|
+
# Use the last hidden state as document embeddings (batch_size,
|
|
67
|
+
# sequence_length, hidden_size)
|
|
60
68
|
embeddings = outputs.last_hidden_state
|
|
61
69
|
|
|
62
70
|
# Convert to list of numpy arrays
|
|
63
71
|
embeddings = embeddings.detach().cpu().numpy()
|
|
64
72
|
|
|
65
|
-
# For simplicity, let's consider the mean of embeddings across tokens
|
|
73
|
+
# For simplicity, let's consider the mean of embeddings across tokens
|
|
74
|
+
# to represent the document
|
|
66
75
|
doc_embeddings = embeddings.mean(axis=1)
|
|
67
76
|
|
|
68
77
|
# Creating document object(s)
|
|
69
78
|
documents = []
|
|
70
79
|
for i, emb in enumerate(doc_embeddings):
|
|
71
|
-
# Store the original input text as content and embeddings in
|
|
80
|
+
# Store the original input text as content and embeddings in
|
|
81
|
+
# metadata
|
|
72
82
|
input_text = data[i] if isinstance(data, list) else data
|
|
73
83
|
doc = Document(content=input_text)
|
|
74
84
|
# Add the embedding and source to metadata
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swarmauri_parser_bertembedding
|
|
3
|
+
Version: 0.11.0.dev1
|
|
4
|
+
Summary: BERT-based dense embedding parser for Swarmauri with batch text encoding and document metadata vectors.
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: swarmauri,parser,bertembedding,bert,embedding,transformers,semantic search,vector embeddings,document embeddings,hugging face,dense retrieval
|
|
8
|
+
Author: Jacob Stewart
|
|
9
|
+
Author-email: jacob@swarmauri.com
|
|
10
|
+
Requires-Python: >=3.10,<3.15
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Natural Language :: English
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Topic :: Text Processing
|
|
21
|
+
Classifier: Programming Language :: Python
|
|
22
|
+
Classifier: Programming Language :: Python :: 3
|
|
23
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
29
|
+
Requires-Dist: swarmauri_base
|
|
30
|
+
Requires-Dist: swarmauri_core
|
|
31
|
+
Requires-Dist: swarmauri_standard
|
|
32
|
+
Requires-Dist: torch
|
|
33
|
+
Requires-Dist: transformers (>=4.45.0)
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<a href="https://pepy.tech/project/swarmauri_parser_bertembedding/">
|
|
40
|
+
<img src="https://static.pepy.tech/badge/swarmauri_parser_bertembedding/month" alt="PyPI - Downloads"/></a>
|
|
41
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_parser_bertembedding/">
|
|
42
|
+
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_parser_bertembedding.svg"/></a>
|
|
43
|
+
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
44
|
+
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue" alt="PyPI - Python Version"/></a>
|
|
45
|
+
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
46
|
+
<img src="https://img.shields.io/pypi/l/swarmauri_parser_bertembedding" alt="PyPI - License"/></a>
|
|
47
|
+
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
48
|
+
<img src="https://img.shields.io/pypi/v/swarmauri_parser_bertembedding?label=swarmauri_parser_bertembedding&color=green" alt="PyPI - swarmauri_parser_bertembedding"/></a>
|
|
49
|
+
<a href="https://discord.gg/N4UpBuQv8T">
|
|
50
|
+
<img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a></p>
|
|
51
|
+
|
|
52
|
+
# Swarmauri Parser Bert Embedding
|
|
53
|
+
|
|
54
|
+
`swarmauri_parser_bertembedding` is the Swarmauri embedding parser for turning
|
|
55
|
+
text into dense vector representations with Hugging Face BERT models. It
|
|
56
|
+
returns Swarmauri `Document` objects whose `content` keeps the original text
|
|
57
|
+
and whose metadata stores the generated embedding vector.
|
|
58
|
+
|
|
59
|
+
## Why Use Swarmauri Parser Bert Embedding
|
|
60
|
+
|
|
61
|
+
- Generate dense semantic vectors inside a Swarmauri parser-style workflow.
|
|
62
|
+
- Keep original text and embedding output together in a single `Document`
|
|
63
|
+
object.
|
|
64
|
+
- Swap BERT model names when you need a different encoder surface.
|
|
65
|
+
- Feed embeddings into retrieval, clustering, semantic search, reranking, or
|
|
66
|
+
downstream vector store pipelines.
|
|
67
|
+
|
|
68
|
+
## FAQ
|
|
69
|
+
|
|
70
|
+
> **What does this parser output?**
|
|
71
|
+
> Swarmauri `Document` objects containing the original text and an averaged BERT
|
|
72
|
+
> embedding stored in `metadata["embedding"]`.
|
|
73
|
+
|
|
74
|
+
> **What model does it use by default?**
|
|
75
|
+
> `bert-base-uncased`.
|
|
76
|
+
|
|
77
|
+
> **Can it parse a batch of strings?**
|
|
78
|
+
> Yes. The current implementation accepts a single string or a list of strings.
|
|
79
|
+
|
|
80
|
+
> **Does it download model weights?**
|
|
81
|
+
> Yes. On first use, Hugging Face model and tokenizer assets are downloaded if
|
|
82
|
+
> they are not already cached locally.
|
|
83
|
+
|
|
84
|
+
## Features
|
|
85
|
+
|
|
86
|
+
- Dense embedding generation using `BertTokenizer` and `BertModel`.
|
|
87
|
+
- Supports single-string and batch-text parsing.
|
|
88
|
+
- Stores the original text alongside the embedding vector in each document.
|
|
89
|
+
- Uses inference mode with `torch.no_grad()` and mean token pooling.
|
|
90
|
+
- Supports Python 3.10, 3.11, 3.12, 3.13, and 3.14.
|
|
91
|
+
|
|
92
|
+
## Installation
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
uv add swarmauri_parser_bertembedding
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
pip install swarmauri_parser_bertembedding
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Notes:
|
|
103
|
+
|
|
104
|
+
- First-run model downloads come from Hugging Face.
|
|
105
|
+
- Install a CUDA-enabled PyTorch build separately if GPU execution is required.
|
|
106
|
+
|
|
107
|
+
## Usage
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
111
|
+
|
|
112
|
+
parser = BERTEmbeddingParser(parser_model_name="bert-base-uncased")
|
|
113
|
+
documents = parser.parse(
|
|
114
|
+
[
|
|
115
|
+
"Swarmauri agents cooperate over shared memory.",
|
|
116
|
+
"Dense embeddings power semantic search.",
|
|
117
|
+
]
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
for document in documents:
|
|
121
|
+
embedding = document.metadata["embedding"]
|
|
122
|
+
print(document.content)
|
|
123
|
+
print(len(embedding), embedding[:5])
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Examples
|
|
127
|
+
|
|
128
|
+
### Embed a single sentence
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
132
|
+
|
|
133
|
+
parser = BERTEmbeddingParser()
|
|
134
|
+
documents = parser.parse("Composable intelligence infrastructure")
|
|
135
|
+
|
|
136
|
+
print(documents[0].id)
|
|
137
|
+
print(documents[0].metadata["source"])
|
|
138
|
+
print(documents[0].metadata["embedding"].shape)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Embed a batch for downstream storage
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
145
|
+
|
|
146
|
+
texts = [
|
|
147
|
+
"Customer support workflows need retrieval.",
|
|
148
|
+
"Embeddings support semantic matching.",
|
|
149
|
+
"Vector stores preserve nearest-neighbor search state.",
|
|
150
|
+
]
|
|
151
|
+
|
|
152
|
+
parser = BERTEmbeddingParser()
|
|
153
|
+
documents = parser.parse(texts)
|
|
154
|
+
|
|
155
|
+
for document in documents:
|
|
156
|
+
vector = document.metadata["embedding"]
|
|
157
|
+
print(document.id, len(vector))
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Use an alternate BERT model name
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
164
|
+
|
|
165
|
+
parser = BERTEmbeddingParser(parser_model_name="bert-base-multilingual-cased")
|
|
166
|
+
docs = parser.parse("Bonjour tout le monde")
|
|
167
|
+
print(docs[0].metadata["embedding"][:5])
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Related Packages
|
|
171
|
+
|
|
172
|
+
- [swarmauri_embedding_mlm](https://pypi.org/project/swarmauri_embedding_mlm/)
|
|
173
|
+
- [swarmauri_vectorstore_mlm](https://pypi.org/project/swarmauri_vectorstore_mlm/)
|
|
174
|
+
- [swarmauri_vectorstore_qdrant](https://pypi.org/project/swarmauri_vectorstore_qdrant/)
|
|
175
|
+
- [swarmauri_vectorstore_pinecone](https://pypi.org/project/swarmauri_vectorstore_pinecone/)
|
|
176
|
+
|
|
177
|
+
## Swarmauri Foundations
|
|
178
|
+
|
|
179
|
+
- [swarmauri](https://pypi.org/project/swarmauri/)
|
|
180
|
+
- [swarmauri_core](https://pypi.org/project/swarmauri_core/)
|
|
181
|
+
- [swarmauri_base](https://pypi.org/project/swarmauri_base/)
|
|
182
|
+
- [swarmauri_standard](https://pypi.org/project/swarmauri_standard/)
|
|
183
|
+
|
|
184
|
+
## More Documentation
|
|
185
|
+
|
|
186
|
+
- [Hugging Face Transformers BERT docs](https://huggingface.co/docs/transformers/model_doc/bert)
|
|
187
|
+
- [Transformers AutoTokenizer and tokenization docs](https://huggingface.co/docs/transformers/main_classes/tokenizer)
|
|
188
|
+
- [PyTorch documentation](https://pytorch.org/docs/stable/index.html)
|
|
189
|
+
|
|
190
|
+
## Best Practices
|
|
191
|
+
|
|
192
|
+
- Chunk very long texts before parsing so they stay within the BERT token limit.
|
|
193
|
+
- Cache Hugging Face assets in CI and deployment environments to avoid repeated
|
|
194
|
+
model downloads.
|
|
195
|
+
- Use a model variant aligned to your language and domain.
|
|
196
|
+
- Persist vectors into a Swarmauri vector store if you plan to search or reuse
|
|
197
|
+
them beyond a single process.
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
This project is licensed under the Apache-2.0 License.
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
swarmauri_parser_bertembedding/BERTEmbeddingParser.py,sha256=RsbzESf74O7zrQkoc7xppTbZEcPTy_fCx4-dXvGgTsg,3132
|
|
2
|
+
swarmauri_parser_bertembedding/__init__.py,sha256=twyKrt-EXRRP6Vqnh9zQ0pnBJeP5j_f8LeZlNgGjl6E,519
|
|
3
|
+
swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/METADATA,sha256=lA-JQcB1P0a6PXUWUFxWqr_sY787UC73i-42urTrdJY,7770
|
|
4
|
+
swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
|
|
5
|
+
swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/entry_points.txt,sha256=wLHXhfBdkHoRqcQS0GCeYOvgOrlttrWGyQs-Fvj9Cqk,112
|
|
6
|
+
swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/licenses/LICENSE,sha256=djUXOlCxLVszShEpZXshZ7v33G-2qIC_j9KXpWKZSzQ,11359
|
|
7
|
+
swarmauri_parser_bertembedding-0.11.0.dev1.dist-info/RECORD,,
|
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: swarmauri_parser_bertembedding
|
|
3
|
-
Version: 0.8.4.dev2
|
|
4
|
-
Summary: Swarmauri Bert Embedding Parser
|
|
5
|
-
License-Expression: Apache-2.0
|
|
6
|
-
License-File: LICENSE
|
|
7
|
-
Keywords: swarmauri,parser,bertembedding,bert,embedding
|
|
8
|
-
Author: Jacob Stewart
|
|
9
|
-
Author-email: jacob@swarmauri.com
|
|
10
|
-
Requires-Python: >=3.10,<3.15
|
|
11
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
-
Classifier: Natural Language :: English
|
|
13
|
-
Classifier: Development Status :: 3 - Alpha
|
|
14
|
-
Classifier: Intended Audience :: Developers
|
|
15
|
-
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
16
|
-
Classifier: Programming Language :: Python
|
|
17
|
-
Classifier: Programming Language :: Python :: 3
|
|
18
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
-
Requires-Dist: swarmauri_base
|
|
25
|
-
Requires-Dist: swarmauri_core
|
|
26
|
-
Requires-Dist: swarmauri_standard
|
|
27
|
-
Requires-Dist: torch
|
|
28
|
-
Requires-Dist: transformers (>=4.45.0)
|
|
29
|
-
Description-Content-Type: text/markdown
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
<p align="center">
|
|
34
|
-
<a href="https://pepy.tech/project/swarmauri_parser_bertembedding/">
|
|
35
|
-
<img src="https://static.pepy.tech/badge/swarmauri_parser_bertembedding/month" alt="PyPI - Downloads"/></a>
|
|
36
|
-
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_parser_bertembedding/">
|
|
37
|
-
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_parser_bertembedding.svg"/></a>
|
|
38
|
-
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
39
|
-
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue" alt="PyPI - Python Version"/></a>
|
|
40
|
-
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
41
|
-
<img src="https://img.shields.io/pypi/l/swarmauri_parser_bertembedding" alt="PyPI - License"/></a>
|
|
42
|
-
<a href="https://pypi.org/project/swarmauri_parser_bertembedding/">
|
|
43
|
-
<img src="https://img.shields.io/pypi/v/swarmauri_parser_bertembedding?label=swarmauri_parser_bertembedding&color=green" alt="PyPI - swarmauri_parser_bertembedding"/></a>
|
|
44
|
-
</p>
|
|
45
|
-
---
|
|
46
|
-
|
|
47
|
-
# Swarmauri Parser Bert Embedding
|
|
48
|
-
|
|
49
|
-
Parser that converts text into embeddings using a Hugging Face BERT encoder. Produces `Document` objects whose metadata carries the averaged token embedding so downstream Swarmauri pipelines can work with dense vectors.
|
|
50
|
-
|
|
51
|
-
## Features
|
|
52
|
-
|
|
53
|
-
- Uses `transformers.BertModel` + `BertTokenizer` (default `bert-base-uncased`).
|
|
54
|
-
- Accepts single strings or lists of strings and emits `Document` instances with original text and embedding metadata.
|
|
55
|
-
- Runs in inference (`eval`) mode with automatic `torch.no_grad()` handling.
|
|
56
|
-
- Works on CPU by default; configure PyTorch device settings to leverage GPU.
|
|
57
|
-
|
|
58
|
-
## Prerequisites
|
|
59
|
-
|
|
60
|
-
- Python 3.10 or newer.
|
|
61
|
-
- PyTorch compatible with your hardware (installed automatically via `transformers` if not present; install CUDA-enabled wheels manually when needed).
|
|
62
|
-
- Internet access on first run so Hugging Face downloads tokenizer/model weights (or warm the cache ahead of time).
|
|
63
|
-
|
|
64
|
-
## Installation
|
|
65
|
-
|
|
66
|
-
```bash
|
|
67
|
-
# pip
|
|
68
|
-
pip install swarmauri_parser_bertembedding
|
|
69
|
-
|
|
70
|
-
# poetry
|
|
71
|
-
poetry add swarmauri_parser_bertembedding
|
|
72
|
-
|
|
73
|
-
# uv (pyproject-based projects)
|
|
74
|
-
uv add swarmauri_parser_bertembedding
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
## Quickstart
|
|
78
|
-
|
|
79
|
-
```python
|
|
80
|
-
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
81
|
-
|
|
82
|
-
parser = BERTEmbeddingParser(parser_model_name="bert-base-uncased")
|
|
83
|
-
|
|
84
|
-
documents = parser.parse([
|
|
85
|
-
"Swarmauri agents cooperate over shared memory.",
|
|
86
|
-
"Dense embeddings power semantic search.",
|
|
87
|
-
])
|
|
88
|
-
|
|
89
|
-
for doc in documents:
|
|
90
|
-
vector = doc.metadata["embedding"]
|
|
91
|
-
print(doc.content)
|
|
92
|
-
print(len(vector), vector[:5])
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
## Custom Models & Devices
|
|
96
|
-
|
|
97
|
-
```python
|
|
98
|
-
import torch
|
|
99
|
-
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
100
|
-
from transformers import BertModel
|
|
101
|
-
|
|
102
|
-
class GPUParser(BERTEmbeddingParser):
|
|
103
|
-
def __init__(self, **kwargs):
|
|
104
|
-
super().__init__(**kwargs)
|
|
105
|
-
self._model = BertModel.from_pretrained(self.parser_model_name).to("cuda")
|
|
106
|
-
|
|
107
|
-
parser = GPUParser(parser_model_name="bert-base-multilingual-cased")
|
|
108
|
-
parser._model.eval()
|
|
109
|
-
```
|
|
110
|
-
|
|
111
|
-
## Batch Embeddings at Scale
|
|
112
|
-
|
|
113
|
-
```python
|
|
114
|
-
from tqdm import tqdm
|
|
115
|
-
from swarmauri_parser_bertembedding import BERTEmbeddingParser
|
|
116
|
-
|
|
117
|
-
texts = [f"Paragraph {i}" for i in range(1000)]
|
|
118
|
-
parser = BERTEmbeddingParser()
|
|
119
|
-
|
|
120
|
-
batched_docs = []
|
|
121
|
-
batch_size = 32
|
|
122
|
-
for start in tqdm(range(0, len(texts), batch_size)):
|
|
123
|
-
batch = texts[start:start + batch_size]
|
|
124
|
-
batched_docs.extend(parser.parse(batch))
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
Persist the resulting vectors into Swarmauri vector stores (Redis, Qdrant, etc.) via the metadata field.
|
|
128
|
-
|
|
129
|
-
## Tips
|
|
130
|
-
|
|
131
|
-
- Preprocess text to match model expectations (lowercase for uncased BERT, language-specific cleanup for multilingual models).
|
|
132
|
-
- For extremely long documents, consider chunking before calling `parse` to respect the 512 token limit.
|
|
133
|
-
- Use PyTorch's `to("cuda")` or `to("mps")` to execute on GPUs or Apple silicon accelerators.
|
|
134
|
-
- Cache Hugging Face weights in CI/CD environments (`HF_HOME=/cache/hf`) to avoid repeated downloads.
|
|
135
|
-
|
|
136
|
-
## Want to help?
|
|
137
|
-
|
|
138
|
-
If you want to contribute to swarmauri-sdk, read up on our [guidelines for contributing](https://github.com/swarmauri/swarmauri-sdk/blob/master/contributing.md) that will help you get started.
|
|
139
|
-
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
swarmauri_parser_bertembedding/BERTEmbeddingParser.py,sha256=I_nEP73vL0hIg211C_947qnzb_1UjyPogPObh689COU,3025
|
|
2
|
-
swarmauri_parser_bertembedding/__init__.py,sha256=twyKrt-EXRRP6Vqnh9zQ0pnBJeP5j_f8LeZlNgGjl6E,519
|
|
3
|
-
swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/METADATA,sha256=2B38tspZTS6w2BR5zAiHYKiQgDji9tRF0UoQF2tI-Jw,5575
|
|
4
|
-
swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
|
|
5
|
-
swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/entry_points.txt,sha256=wLHXhfBdkHoRqcQS0GCeYOvgOrlttrWGyQs-Fvj9Cqk,112
|
|
6
|
-
swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/licenses/LICENSE,sha256=djUXOlCxLVszShEpZXshZ7v33G-2qIC_j9KXpWKZSzQ,11359
|
|
7
|
-
swarmauri_parser_bertembedding-0.8.4.dev2.dist-info/RECORD,,
|
|
File without changes
|