cmem-plugin-pgvector 0.5.0__tar.gz

cmem_plugin_pgvector-0.5.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2021 CMEM
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

cmem_plugin_pgvector-0.5.0/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.3
+Name: cmem-plugin-pgvector
+Version: 0.5.0
+Summary: Store embedding vectors into a Postgres vector store.
+License: Apache-2.0
+Keywords: eccenca Corporate Memory,plugin
+Author: eccenca GmbH
+Author-email: cmempy-developer@eccenca.com
+Maintainer: Edgard Marx
+Maintainer-email: edgard.marx@eccenca.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Plugins
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: cmem-plugin-base (>=4.8.0,<5.0.0)
+Requires-Dist: langchain-community (>=0.3.2,<0.4.0)
+Requires-Dist: langchain-postgres (>=0.0.12,<0.0.13)
+Requires-Dist: psycopg[binary] (>=3.2.3,<4.0.0)
+Description-Content-Type: text/markdown
+
+# cmem-plugin-pgvector
+
+[![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] 
+
+Store embedding vectors into a Postgres vector store.
+
+This plugin consumes the costumable entity's paths ```embedding```, ```text``` and ```metadata``` as following:
+
+  - The text path contain the text used to generate the embeddings, default ```text```.
+  - The embedding path contain the embedding representation of the text, default ```embedding```.
+  - The metadata path contain the information that will be associated with the embedding, default all paths.
+
+[![eccenca Corporate Memory][cmem-shield]][cmem-link]
+
+## Use
+
+Interact with Large Language Models.
+
+This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com).
+
+You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line
+clients like this:
+
+```
+cmemc admin workspace python install cmem-plugin-llm
+```
+
+### Parameters
+
+- ```collection_name```: The name of the collection where the embeddings are going to be stored, default ```my_collection```
+- ```user```:the database user
+- ```password```: the database password
+- ```host```: the databse host, i.e. locahost
+- ```port```: the database port, default ```5432```
+- ```database```: the name of the database
+- ```pre_delete_collection```: boolean parameter indicating if the collection should be cleanse before insertion, default ```false```
+- ```embedding_path```: output path that will contain the generated embedding, default ```embedding```
+- ```text_path```: path containing the text used for genereting the embedding, default ```text```
+- ```metadata_paths```: paths from the entity that will be stored along with the embedding, default all paths
+
+[cmem-link]: https://documentation.eccenca.com
+[cmem-shield]: https://img.shields.io/endpoint?url=https://dev.documentation.eccenca.com/badge.json
+[poetry-link]: https://python-poetry.org/
+[poetry-shield]: https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json
+[ruff-link]: https://docs.astral.sh/ruff/
+[ruff-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&label=Code%20Style
+[mypy-link]: https://mypy-lang.org/
+[mypy-shield]: https://www.mypy-lang.org/static/mypy_badge.svg
+[copier]: https://copier.readthedocs.io/
+[copier-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-purple.json
+
+

cmem_plugin_pgvector-0.5.0/README-public.md

@@ -0,0 +1,51 @@
+# cmem-plugin-pgvector
+
+[![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] 
+
+Store embedding vectors into a Postgres vector store.
+
+This plugin consumes the costumable entity's paths ```embedding```, ```text``` and ```metadata``` as following:
+
+  - The text path contain the text used to generate the embeddings, default ```text```.
+  - The embedding path contain the embedding representation of the text, default ```embedding```.
+  - The metadata path contain the information that will be associated with the embedding, default all paths.
+
+[![eccenca Corporate Memory][cmem-shield]][cmem-link]
+
+## Use
+
+Interact with Large Language Models.
+
+This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com).
+
+You can install it with the [cmemc](https://eccenca.com/go/cmemc) command line
+clients like this:
+
+```
+cmemc admin workspace python install cmem-plugin-llm
+```
+
+### Parameters
+
+- ```collection_name```: The name of the collection where the embeddings are going to be stored, default ```my_collection```
+- ```user```:the database user
+- ```password```: the database password
+- ```host```: the databse host, i.e. locahost
+- ```port```: the database port, default ```5432```
+- ```database```: the name of the database
+- ```pre_delete_collection```: boolean parameter indicating if the collection should be cleanse before insertion, default ```false```
+- ```embedding_path```: output path that will contain the generated embedding, default ```embedding```
+- ```text_path```: path containing the text used for genereting the embedding, default ```text```
+- ```metadata_paths```: paths from the entity that will be stored along with the embedding, default all paths
+
+[cmem-link]: https://documentation.eccenca.com
+[cmem-shield]: https://img.shields.io/endpoint?url=https://dev.documentation.eccenca.com/badge.json
+[poetry-link]: https://python-poetry.org/
+[poetry-shield]: https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json
+[ruff-link]: https://docs.astral.sh/ruff/
+[ruff-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&label=Code%20Style
+[mypy-link]: https://mypy-lang.org/
+[mypy-shield]: https://www.mypy-lang.org/static/mypy_badge.svg
+[copier]: https://copier.readthedocs.io/
+[copier-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-purple.json
+

cmem_plugin_pgvector-0.5.0/cmem_plugin_pgvector/__init__.py

@@ -0,0 +1 @@
+"""cmem-plugin-pgvector"""

cmem_plugin_pgvector-0.5.0/cmem_plugin_pgvector/cmem_plugin_pgvector.py

@@ -0,0 +1,275 @@
+"""Random values workflow plugin module
+
+Remove this and other example files after bootstrapping your project.
+"""
+
+from ast import literal_eval
+from collections.abc import Sequence
+from typing import Any
+
+from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
+from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
+from cmem_plugin_base.dataintegration.entity import Entities, Entity, EntityPath
+from cmem_plugin_base.dataintegration.parameter.password import Password, PasswordParameterType
+from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
+from cmem_plugin_base.dataintegration.ports import (
+    FixedNumberOfInputs,
+    UnknownSchemaPort,
+)
+from cmem_plugin_base.dataintegration.types import IntParameterType
+from langchain_postgres import PGVector
+
+
+class DataContainer:
+    """Encapsulate the data to be added to the database."""
+
+    def __init__(self):
+        self.texts = []
+        self.embeddings = []
+        self.metadatas = []
+
+    def add(self, text: str, embedding: list[float], metadata: dict) -> None:
+        """Add objects to the respective lists."""
+        self.texts.append(text)
+        self.embeddings.append(embedding)
+        self.metadatas.append(metadata)
+
+    def clear(self) -> None:
+        """Clear all three lists."""
+        self.texts.clear()
+        self.embeddings.clear()
+        self.metadatas.clear()
+
+    def size(self) -> int:
+        """Return the size of the lists (assuming all lists have the same length)."""
+        return len(self.texts)
+
+
+@Plugin(
+    label="Postgres Vector Store",
+    description="Store embeddings into Postgres Vector Store.",
+    documentation="""
+This plugin workflow store embeddings into Postgres Vector Store.
+
+The vector embeddings and its respective metadata are going to be stored into a collection inside
+the Postgres Vector Store.
+It is possible to specify either the name of the attributes containing the vectors as well as the
+metadata.
+""",
+    icon=Icon(package=__package__, file_name="postgresql.svg"),
+    parameters=[
+        PluginParameter(
+            name="host",
+            label="Database Host",
+            description="The hostname of the postgres database service.",
+            default_value="pgvector",
+        ),
+        PluginParameter(
+            name="port",
+            label="Database Port",
+            param_type=IntParameterType(),
+            description="The port number of the postgres database service.",
+            default_value=5432,
+        ),
+        PluginParameter(
+            name="user",
+            label="Database User",
+            description="The account name used to login to the postgres database service.",
+            default_value="pgvector",
+        ),
+        PluginParameter(
+            name="password",
+            label="Database Password",
+            param_type=PasswordParameterType(),
+            description="The password of the database account.",
+        ),
+        PluginParameter(
+            name="database",
+            label="Database Name",
+            description="The database name.",
+            default_value="pgvector",
+        ),
+        PluginParameter(
+            name="collection_name",
+            label="Collection Name",
+            description="The name of the collection, where the embeddings are going to be stored.",
+            default_value="my_collection",
+        ),
+        PluginParameter(
+            name="pre_delete_collection",
+            label="Pre Delete Collection",
+            description="If set to true, then the collection will removed at the beginning.",
+            default_value=True,
+        ),
+        PluginParameter(
+            name="source_path",
+            label="Source Path",
+            description="The name of the path to use for reading the embedding source.",
+            default_value="_embedding_source",
+            advanced=True,
+        ),
+        PluginParameter(
+            name="embedding_path",
+            label="Embedding Path",
+            description="The name of the path to use for reading the embeddings.",
+            default_value="_embedding",
+            advanced=True,
+        ),
+        PluginParameter(
+            name="metadata_paths",
+            label="Metadata Paths",
+            description="The comma separated list path names to be used as metadata. "
+            "Empty name means all paths "
+            "(except embedding source and embedding) will be used",
+            default_value="",
+            advanced=True,
+        ),
+        PluginParameter(
+            name="batch_processing_size",
+            label="Batch Processing Size",
+            description="The number of entries to be processed in batch.",
+            default_value=100,
+            advanced=True,
+        ),
+    ],
+)
+class PGVectorStorePlugin(WorkflowPlugin):
+    """PGVectorStorePlugin: Enable the storage of vectors into Postgres Vector Store."""
+
+    connection_string: str
+    user: str
+    password: str
+    host: str
+    port: int
+    database: str
+    collection_name: str
+    source_path: str
+    embedding_path: str
+    metadata_paths: str
+    batch_processing_size: int
+    inputs: Sequence[Entities]
+    db: PGVector
+    report: ExecutionReport
+
+    def __init__(  # noqa: PLR0913
+        self,
+        host: str = "pgvector",
+        port: int = 5432,
+        user: str = "pgvector",
+        password: Password | str = "",
+        database: str = "pgvector",
+        collection_name: str = "my_collection",
+        pre_delete_collection: bool = True,
+        source_path: str = "_embedding_source",
+        embedding_path: str = "_embedding",
+        metadata_paths: str = "metadata",
+        batch_processing_size: int = 1000,
+    ) -> None:
+        self.batch_processing_size = batch_processing_size
+        self.collection_name = collection_name
+        self.user = user
+        self.host = host
+        self.port = port
+        self.database = database
+        self.embedding_path = embedding_path
+        self.metadata_paths = metadata_paths
+        self.source_path = source_path
+
+        self.output_port = None
+        self.input_ports = FixedNumberOfInputs([UnknownSchemaPort()])
+        str_password = self.password = password if isinstance(password, str) else password.decrypt()
+        self.connection_string = (
+            f"postgresql+psycopg://{user}:{str_password}@{host}:{port}/{database}"
+        )
+
+        self.report = ExecutionReport()
+        self.report.operation = "store"
+        self.report.operation_desc = "vectors stored"
+
+        self.db = PGVector(
+            collection_name=self.collection_name,
+            connection=self.connection_string,
+            embeddings=None,  # type: ignore  # noqa: PGH003
+            use_jsonb=True,
+            pre_delete_collection=pre_delete_collection,
+        )
+
+    def _update_report(self, count: int) -> None:
+        self.report.entity_count = count
+        self.execution_context.report.update(self.report)
+
+    def _paths_to_metadata(
+        self, paths: list[str], entity: Entity, schema_paths: list[EntityPath]
+    ) -> dict[str, Any]:
+        metadata_paths = [path for path in schema_paths if path.path in paths]
+        return self._metadata(entity, metadata_paths)
+
+    def _index_of(self, path_name: str, paths: list[EntityPath]) -> int:
+        for path in paths:
+            if path.path == path_name:
+                return paths.index(path)
+        return -1
+
+    def _metadata(self, entity: Entity, schema_paths: list[EntityPath]) -> dict[str, Any]:
+        entity_dict: dict[str, Any] = {}
+        for path, values in zip(schema_paths, entity.values, strict=False):
+            if path.path not in (self.embedding_path, self.source_path):
+                entity_dict[path.path] = list(values)
+        return entity_dict
+
+    def _cancel_workflow(self) -> bool:
+        """Cancel workflow"""
+        try:
+            if self.execution_context.workflow.status() == "Canceling":
+                self.log.info("End task (Cancelled Workflow).")
+                return True
+        except AttributeError:
+            pass
+        return False
+
+    def _process_entities(self, entities: Entities) -> None:
+        schema_paths: list[EntityPath] = list(entities.schema.paths)
+        text_path_idx: int = self._index_of(self.source_path, schema_paths)
+        embedding_path_idx: int = self._index_of(self.embedding_path, schema_paths)
+
+        if text_path_idx == -1 or embedding_path_idx == -1:
+            raise ValueError("The data does not have a text or embedding paths defined.")
+
+        n_processed_entries: int = 0
+        container: DataContainer = DataContainer()
+        self._update_report(n_processed_entries)
+        for entity in entities.entities:
+            container.add(
+                entity.values[text_path_idx][0],
+                literal_eval(entity.values[embedding_path_idx][0]),
+                self._paths_to_metadata(self.metadata_paths.split(","), entity, schema_paths)
+                if self.metadata_paths
+                else self._metadata(entity, schema_paths),
+            )
+            if container.size() == self.batch_processing_size:
+                self.db.add_embeddings(container.texts, container.embeddings, container.metadatas)
+                n_processed_entries += container.size()
+                self._update_report(n_processed_entries)
+                container.clear()
+            if self._cancel_workflow():
+                return
+        if container.size() > 0:
+            self.db.add_embeddings(container.texts, container.embeddings, container.metadatas)
+            n_processed_entries += container.size()
+            self._update_report(n_processed_entries)
+
+    def execute(
+        self,
+        inputs: Sequence[Entities],
+        context: ExecutionContext,
+    ) -> None:
+        """Run the workflow operator."""
+        self.log.info("Start storing vectors.")
+        self.inputs = inputs
+        self.execution_context = context
+        try:
+            first_input: Entities = self.inputs[0]
+        except IndexError as error:
+            raise ValueError("Input port not connected.") from error
+        self._process_entities(first_input)
+        self.log.info("Vectors stored successfuly.")

cmem_plugin_pgvector-0.5.0/cmem_plugin_pgvector/postgresql.svg

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   viewBox="0 0 24 24"
+   version="1.1"
+   id="svg919"
+   sodipodi:docname="postgresql.svg"
+   inkscape:version="1.1.2 (b8e25be8, 2022-02-05)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs923" />
+  <sodipodi:namedview
+     id="namedview921"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="17.192681"
+     inkscape:cx="12.35991"
+     inkscape:cy="12.883389"
+     inkscape:window-width="1920"
+     inkscape:window-height="1027"
+     inkscape:window-x="1728"
+     inkscape:window-y="25"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg919" />
+  <path
+     d="m 21.296342,14.115915 a 0.40209284,0.38851625 0 0 0 -0.04584,-0.09324 Q 21.081623,13.715747 20.440687,13.84318 c -1.330124,0.264191 -1.844802,0.101015 -2.031373,-0.01554 1.079217,-1.591363 1.966234,-3.513741 2.445528,-5.3071321 0.218739,-0.8158842 0.64174,-2.7374855 0.09811,-3.6753637 A 1.2866971,1.243252 0 0 0 20.832327,4.6617605 C 19.794928,3.382765 18.272604,2.6950913 16.431019,2.6764425 15.228761,2.66401 14.203424,2.9452957 13.925176,3.0486411 a 8.0418568,7.7703249 0 0 0 -0.41496,-0.063717 6.4334854,6.2162599 0 0 0 -1.055091,-0.098683 C 11.504577,2.8714776 10.683504,3.0913778 10.002358,3.5389486 9.3139755,3.2895211 6.1527216,2.2607301 4.1961379,3.6003341 3.1016412,4.3486164 2.5982209,5.6851123 2.6955274,7.5740783 2.728499,8.2096909 3.1032495,10.164705 3.6951302,12.037353 q 0.5548881,1.756093 1.1523981,2.78333 0.6674741,1.16011 1.3783742,1.390888 c 0.3602752,0.115001 0.9111424,0.111116 1.494177,-0.566456 a 45.034398,43.51382 0 0 1 1.5641412,-1.714134 c 0.3498207,0.182603 0.7285923,0.281286 1.1178183,0.292941 v 0.0031 a 8.8460425,8.5473574 0 0 0 -0.198634,0.236995 c -0.272619,0.334124 -0.3297162,0.404057 -1.2062786,0.57889 -0.2492976,0.04973 -0.9119466,0.181048 -0.9215968,0.630173 a 0.48251141,0.4662195 0 0 0 0.073181,0.25409 c 0.1825501,0.328684 0.7414592,0.473989 0.8162485,0.491861 1.0735879,0.258752 2.0144849,0.07149 2.7117139,-0.527605 -0.01367,1.73356 0.06192,3.43293 0.277444,3.953541 0.177725,0.429699 0.61279,1.47947 1.986339,1.47947 q 0.301569,7.77e-4 0.66667,-0.07304 c 1.433059,-0.296826 2.055498,-0.909128 2.29595,-2.258056 0.120628,-0.676018 0.323282,-2.233969 0.433456,-3.18661 0.01367,-0.05439 0.02895,-0.09324 0.04584,-0.105677 0,0 0.05629,-0.0373 0.343388,0.02331 l 0.03538,0.0054 0.204263,0.01709 0.01206,7.77e-4 c 0.681145,0.0303 1.536799,-0.110339 2.035394,-0.334124 0.517895,-0.23311 1.452359,-0.802674 1.282676,-1.297644 M 4.2556476,11.903703 C 3.6573335,10.011629 3.3083169,8.1078996 3.2809746,7.5748553 3.1933183,5.8871407 3.61632,4.7130447 4.5371126,4.0836483 6.0144017,3.0742831 8.4293713,3.6640508 9.4490787,3.9826341 l -0.00804,0.00777 C 7.8141692,5.5771048 7.8527702,8.2905022 7.8583995,8.4567872 c 0,0.063717 0.00483,0.1546295 0.012867,0.2797317 0.027342,0.455341 0.080419,1.3054141 -0.05951,2.2673811 -0.1286697,0.893587 0.156012,1.768526 0.7824727,2.400253 q 0.096502,0.09791 0.2026547,0.184157 c -0.2790524,0.288279 -0.8846042,0.927 -1.5303653,1.676836 -0.4567775,0.529936 -0.7720183,0.428145 -0.874954,0.394732 C 6.076324,15.558864 5.7377618,15.20376 5.3951787,14.632641 5.0091696,13.980711 4.6207479,13.053711 4.257256,11.903703 m 4.8307434,3.953542 A 1.2866971,1.243252 0 0 1 8.7405911,15.718933 c 0.071573,-0.0303 0.1905921,-0.06993 0.3884217,-0.108785 1.0325742,-0.205913 1.1918032,-0.350441 1.5400152,-0.777032 0.07961,-0.09868 0.169684,-0.209799 0.295137,-0.344226 a 0.32167427,0.310813 0 0 0 0.05951,-0.101014 c 0.136712,-0.117332 0.218739,-0.08547 0.350625,-0.03264 0.125453,0.05051 0.247689,0.202028 0.297549,0.36909 0.02413,0.07926 0.04986,0.229225 -0.03619,0.34578 -0.726984,0.983723 -1.786901,0.97129 -2.5476606,0.787134 m 1.6839646,-3.098806 -0.04182,0.108785 c -0.106957,0.2774 -0.206676,0.535375 -0.268598,0.78014 C 9.9251567,13.64581 9.402436,13.423579 9.0059724,13.023407 8.5009438,12.514451 8.2717509,11.806574 8.376295,11.080826 8.523461,10.064467 8.4695806,9.1794274 8.4406299,8.7038835 L 8.4301755,8.5329364 C 8.6682144,8.3293539 9.7699488,7.759012 10.555638,7.9330673 c 0.358667,0.079257 0.577406,0.3154752 0.667474,0.7210862 0.470449,2.1010955 0.06273,2.9760345 -0.265381,3.6800255 a 7.2376711,6.9932924 0 0 0 -0.184963,0.42426 m 5.922024,3.552593 q -0.0193,0.20669 -0.04986,0.463111 l -0.117411,0.34034 a 0.32167427,0.310813 0 0 0 -0.01448,0.08392 c -0.0048,0.36909 -0.04343,0.504294 -0.09248,0.676018 -0.05147,0.17794 -0.108565,0.379192 -0.144753,0.821323 -0.08846,1.098724 -0.706075,1.730452 -1.943717,1.986095 -1.218341,0.252536 -1.434667,-0.385408 -1.624455,-0.948756 a 5.6292998,5.4392274 0 0 0 -0.06273,-0.176387 c -0.1729,-0.455341 -0.152796,-1.09717 -0.126258,-1.985318 0.01287,-0.435915 -0.0201,-1.477138 -0.265381,-2.056028 q 0.0048,-0.341894 0.01528,-0.693113 a 0.32167427,0.310813 0 0 0 -0.01287,-0.0878 1.6083714,1.554065 0 0 0 -0.03538,-0.161623 c -0.09811,-0.33257 -0.337758,-0.610748 -0.627265,-0.726525 -0.114195,-0.04584 -0.324087,-0.129765 -0.576601,-0.0676 0.05388,-0.214461 0.147166,-0.456118 0.248493,-0.718755 l 0.04262,-0.110339 c 0.04825,-0.124325 0.107761,-0.252535 0.171291,-0.388516 0.342583,-0.736627 0.812228,-1.745215 0.302374,-4.0234744 C 11.590625,7.6844169 10.952906,7.2679275 9.9862748,7.3642795 9.4072611,7.422557 8.8764985,7.6486734 8.6119214,7.7776608 A 4.8251141,4.662195 0 0 0 8.454301,7.8584722 C 8.5282861,6.9990742 8.8073386,5.392171 9.8503674,4.3758126 a 3.2167427,3.10813 0 0 1 0.2436686,-0.214461 0.28146499,0.27196137 0 0 0 0.116607,-0.04973 C 10.81539,3.668713 11.573737,3.4511439 12.463971,3.4643534 q 0.495378,0.00777 0.944114,0.06294 c 1.56012,0.2758465 2.608778,1.124366 3.245693,1.8516684 0.654607,0.7475053 1.009253,1.5004497 1.15079,1.9068377 -1.063938,-0.1041223 -1.787705,0.098683 -2.155218,0.6060854 -0.797752,1.101832 0.437477,3.2417791 1.030966,4.2705701 0.108565,0.188042 0.202655,0.351219 0.23241,0.419598 0.193005,0.45301 0.443106,0.755275 0.625656,0.975953 0.05629,0.0676 0.110978,0.132872 0.151992,0.190372 -0.321675,0.09014 -0.900688,0.297604 -0.848416,1.334165 -0.01046,0.121217 -0.03458,0.347334 -0.06755,0.633282 -0.03699,0.161622 -0.05629,0.357435 -0.08042,0.595207 m 0.715725,-1.25957 c -0.03217,-0.646491 0.21713,-0.714093 0.480099,-0.784803 l 0.108565,-0.03186 a 0.80418568,0.77703249 0 0 0 0.107761,0.08003 c 0.458385,0.292164 1.273026,0.327131 2.418186,0.104123 -0.162446,0.137534 -0.417372,0.310813 -0.766389,0.466996 -0.329716,0.147636 -0.881388,0.258752 -1.404912,0.28284 -0.579014,0.02642 -0.873346,-0.06216 -0.94331,-0.117332 m 0.458386,-7.203868 c -0.0048,0.2719614 -0.04343,0.5198348 -0.08444,0.7778095 -0.04423,0.2781777 -0.09007,0.5656797 -0.102131,0.9145673 -0.01126,0.3387862 0.03217,0.6915582 0.07479,1.0334532 0.08605,0.689228 0.173704,1.398658 -0.166467,2.098764 a 3.2167427,3.10813 0 0 1 -0.151187,-0.299157 6.4334854,6.2162599 0 0 0 -0.26136,-0.479429 C 16.681926,11.035752 15.523094,9.0263463 16.115779,8.2073541 16.42137,7.7854254 17.194996,7.7675537 17.8681,7.847588 m 0.183354,5.449329 -0.06836,-0.08314 -0.02815,-0.03419 c 0.583838,-0.932439 0.469644,-1.854777 0.367513,-2.672215 -0.04182,-0.335678 -0.08042,-0.652707 -0.07077,-0.9495335 0.01045,-0.3162522 0.05308,-0.5866595 0.09489,-0.8485194 0.05147,-0.3224685 0.104544,-0.6558155 0.08926,-1.0489939 a 0.48251141,0.4662195 0 0 0 0.0097,-0.1476362 C 18.408511,7.1350549 17.962993,6.0068038 17.054263,4.985006 a 6.2726483,6.0608534 0 0 0 -2.161651,-1.5851463 7.4789268,7.2264022 0 0 1 1.626867,-0.144528 c 1.650189,0.035743 2.955383,0.6325044 3.879392,1.7739652 a 0.80418568,0.77703249 0 0 1 0.05388,0.077703 C 21.034178,6.1606562 20.230796,9.982879 18.050649,13.296923 M 10.960948,8.5445919 C 10.940843,8.6844577 10.71165,8.8732766 10.461549,8.8732766 A 0.80418568,0.77703249 0 0 1 10.396409,8.8686144 0.64334854,0.62162599 0 0 1 9.9894915,8.6238492 C 9.952499,8.5772272 9.8929892,8.4855374 9.9042478,8.4023949 a 0.17692085,0.17094715 0 0 1 0.074789,-0.1157778 c 0.094894,-0.069156 0.2830732,-0.094798 0.4905532,-0.066825 0.254123,0.034189 0.516288,0.1499673 0.490554,0.3247996 m 6.377192,-0.3193604 c 0.0088,0.062163 -0.03941,0.1554065 -0.12304,0.2408801 a 0.57901369,0.55946339 0 0 1 -0.328108,0.1732782 0.80418568,0.77703249 0 0 1 -0.06031,0.00389 c -0.235626,0 -0.435064,-0.1818256 -0.450344,-0.2882791 -0.0193,-0.1375347 0.212305,-0.24088 0.450344,-0.2735154 0.239647,-0.032635 0.492162,0.00699 0.511462,0.143751"
+     id="path917"
+     style="stroke-width:0.790493" />
+</svg>

cmem_plugin_pgvector-0.5.0/pyproject.toml

@@ -0,0 +1,99 @@
+[tool.poetry]
+name = "cmem-plugin-pgvector"
+version = "0.5.0"
+license = "Apache-2.0"
+description = "Store embedding vectors into a Postgres vector store."
+authors = ["eccenca GmbH <cmempy-developer@eccenca.com>"]
+maintainers = ["Edgard Marx <edgard.marx@eccenca.com>"]
+classifiers = [
+    "Development Status :: 4 - Beta","Environment :: Plugins",
+]
+readme = "README-public.md"
+keywords = [
+    "eccenca Corporate Memory", "plugin"
+]
+
+
+[tool.poetry.dependencies]# if you need to change python version here, change it also in .python-version
+python = "^3.11"
+langchain-community = "^0.3.2"
+langchain-postgres = "^0.0.12"
+psycopg = {extras = ["binary"], version = "^3.2.3"}
+
+[tool.poetry.dependencies.cmem-plugin-base]
+version = "^4.8.0"
+allow-prereleases = false
+
+[tool.poetry.group.dev.dependencies.cmem-cmemc]
+version = "^24.3.0"
+
+[tool.poetry.group.dev.dependencies]
+genbadge = {extras = ["coverage"], version = "^1.1.1"}
+mypy = "^1.14.1"
+pip = "^25.0"
+pytest = "^8.3.4"
+pytest-cov = "^6.0.0"
+pytest-dotenv = "^0.5.2"
+pytest-html = "^4.1.1"
+pytest-memray = { version = "^1.7.0",  markers = "platform_system != 'Windows'" }
+ruff = "^0.9.4"
+safety = "^1.10.3"
+aiohttp = "^3.10.11"
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
+build-backend = "poetry_dynamic_versioning.backend"
+
+[tool.poetry-dynamic-versioning]
+enable = false
+vcs = "git"
+dirty = true
+bump = true
+
+[tool.mypy]
+warn_return_any = true
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+addopts = ""
+
+[tool.coverage.report]
+exclude_also = [
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+    ]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.format]
+line-ending = "lf" # Use `\n` line endings for all files
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "ANN204",  # Missing return type annotation for special method `__init__`
+    "COM812",  # missing-trailing-comma
+    "D107",    # Missing docstring in __init__
+    "D203",    # [*] 1 blank line required before class docstring
+    "D211",    # No blank lines allowed before class docstring
+    "D213",    # Multi-line docstring summary should start at the second line
+    "D400",    # First line should end with a period
+    "D415",    # First line should end with a period, question mark, or exclamation point
+    "EM",      # Exception texts - https://docs.astral.sh/ruff/rules/#flake8-errmsg-em
+    "FBT",     # The Boolean Trap - https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
+    "FIX002",  # Allow to add TODO notes in the code
+    "G004",    # Logging statement uses f-string
+    "ISC001",  # single-line-implicit-string-concatenation
+    "PD",      # opinionated linting for pandas code
+    "S101",    # use of assert detected
+    "TRY003",  # Avoid specifying long messages outside the exception class
+]
+