swarmauri_parser_bertembedding 0.6.0.dev154__tar.gz

swarmauri_parser_bertembedding-0.6.0.dev154/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.3
+Name: swarmauri_parser_bertembedding
+Version: 0.6.0.dev154
+Summary: Swarmauri Bert Embedding Parser
+License: Apache-2.0
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: swarmauri_base (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: swarmauri_core (>=0.6.0.dev154,<0.7.0)
+Requires-Dist: transformers (>=4.45.0)
+Project-URL: Repository, http://github.com/swarmauri/swarmauri-sdk
+Description-Content-Type: text/markdown
+
+# Swarmauri Example Community Package

swarmauri_parser_bertembedding-0.6.0.dev154/README.md

@@ -0,0 +1 @@
+# Swarmauri Example Community Package

swarmauri_parser_bertembedding-0.6.0.dev154/pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "swarmauri_parser_bertembedding"
+version = "0.6.0.dev154"
+description = "Swarmauri Bert Embedding Parser"
+authors = ["Jacob Stewart <jacob@swarmauri.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+repository = "http://github.com/swarmauri/swarmauri-sdk"
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12"
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+
+# Swarmauri
+swarmauri_core = {version = "^0.6.0.dev154"}
+swarmauri_base = {version = "^0.6.0.dev154"}
+
+# Dependencies
+transformers = ">=4.45.0"
+
+
+[tool.poetry.group.dev.dependencies]
+flake8 = "^7.0"
+pytest = "^8.0"
+pytest-asyncio = ">=0.24.0"
+pytest-xdist = "^3.6.1"
+pytest-json-report = "^1.5.0"
+python-dotenv = "*"
+requests = "^2.32.3"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+norecursedirs = ["combined", "scripts"]
+
+markers = [
+    "test: standard test",
+    "unit: Unit tests",
+    "integration: Integration tests",
+    "acceptance: Acceptance tests",
+    "experimental: Experimental tests"
+]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.poetry.plugins."swarmauri.parsers"]
+BERTEmbeddingParser = "swarmauri_parser_bertembedding.BERTEmbeddingParser:BERTEmbeddingParser"

swarmauri_parser_bertembedding-0.6.0.dev154/swarmauri_parser_bertembedding/BERTEmbeddingParser.py

@@ -0,0 +1,72 @@
+from typing import List, Union, Any, Literal
+from swarmauri_core.ComponentBase import ComponentBase
+from transformers import BertTokenizer, BertModel
+import torch
+from pydantic import PrivateAttr
+from swarmauri_core.documents.IDocument import IDocument
+from swarmauri_standard.documents.Document import Document
+from swarmauri_base.parsers.ParserBase import ParserBase
+
+@ComponentBase.register_type(ParserBase, "BERTEmbeddingParser")
+class BERTEmbeddingParser(ParserBase):
+    """
+    A parser that transforms input text into document embeddings using BERT.
+
+    This parser tokenizes the input text, passes it through a pre-trained BERT model,
+    and uses the resulting embeddings as the document content.
+    """
+
+    parser_model_name: str = "bert-base-uncased"
+    _model: Any = PrivateAttr()
+    type: Literal["BERTEmbeddingParser"] = "BERTEmbeddingParser"
+
+    def __init__(self, **kwargs):
+        """
+        Initializes the BERTEmbeddingParser with a specific BERT model.
+
+        Parameters:
+        - model_name (str): The name of the pre-trained BERT model to use.
+        """
+        super().__init__(**kwargs)
+        self.tokenizer = BertTokenizer.from_pretrained(self.parser_model_name)
+        self._model = BertModel.from_pretrained(self.parser_model_name)
+        self._model.eval()  # Set model to evaluation mode
+
+    def parse(self, data: Union[str, Any]) -> List[IDocument]:
+        """
+        Tokenizes input data and generates embeddings using a BERT model.
+
+        Parameters:
+        - data (Union[str, Any]): Input data, expected to be a single string or batch of strings.
+
+        Returns:
+        - List[IDocument]: A list containing a single IDocument instance with BERT embeddings as content.
+        """
+
+        # Tokenization
+        inputs = self.tokenizer(
+            data, return_tensors="pt", padding=True, truncation=True, max_length=512
+        )
+
+        # Generate embeddings
+        with torch.no_grad():
+            outputs = self._model(**inputs)
+
+        # Use the last hidden state as document embeddings (batch_size, sequence_length, hidden_size)
+        embeddings = outputs.last_hidden_state
+
+        # Convert to list of numpy arrays
+        embeddings = embeddings.detach().cpu().numpy()
+
+        # For simplicity, let's consider the mean of embeddings across tokens to represent the document
+        doc_embeddings = embeddings.mean(axis=1)
+
+        # Creating document object(s)
+        documents = [
+            Document(
+                doc_id=str(i), content=emb, metadata={"source": "BERTEmbeddingParser"}
+            )
+            for i, emb in enumerate(doc_embeddings)
+        ]
+
+        return documents

swarmauri_parser_bertembedding-0.6.0.dev154/swarmauri_parser_bertembedding/__init__.py

@@ -0,0 +1,12 @@
+from .BERTEmbeddingParser import BERTEmbeddingParser
+
+__version__ = "0.6.0.dev26"
+__long_desc__ = """
+
+# Swarmauri BERTEmbedding Parser Plugin
+
+Visit us at: https://swarmauri.com
+Follow us at: https://github.com/swarmauri
+Star us at: https://github.com/swarmauri/swarmauri-sdk
+
+"""