tentaclio-postgres 0.1.0__py3-none-any.whl

tentaclio_postgres/__init__.py

@@ -0,0 +1,20 @@
+"""This package implements the tentaclio postgres client"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+from tentaclio import *  # noqa
+
+from .clients.postgres_client import PostgresClient
+from .streams.postgres_handler import PostgresURLHandler
+
+try:
+    __version__ = _version("tentaclio-postgres")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+
+# Db registry
+DB_REGISTRY.register("postgresql", PostgresClient)  # type: ignore
+
+# postgres handler
+STREAM_HANDLER_REGISTRY.register("postgresql", PostgresURLHandler())  # type: ignore

tentaclio_postgres/__main__.py

@@ -0,0 +1,7 @@
+# Helper cli to encode urls.
+
+
+from tentaclio.__main__ import main
+
+if __name__ == "__main__":
+    main(prog_name="python -m tentaclio-postgres")

tentaclio_postgres/clients/__init__.py

@@ -0,0 +1,9 @@
+"""The client module defines stream and query based clients.
+
+Stream clients allow to read and write from stream based sources with ease,
+and in an interchangable manner.
+
+Query based clients unify how to access databases leveraging from sqlalchemy.
+"""
+
+from .postgres_client import *  # noqa

tentaclio_postgres/clients/postgres_client.py

@@ -0,0 +1,87 @@
+"""Postgres query client.
+Adds the convinience methods for loading csv data using copy_expert,
+which is more performant than using sql alchemy functions.
+"""
+
+import io
+import os
+from typing import Sequence
+
+import pandas as pd
+from tentaclio import protocols
+from tentaclio.clients import decorators, sqla_client
+
+__all__ = ["PostgresClient"]
+
+
+class PostgresClient(sqla_client.SQLAlchemyClient):
+    """Postgres client, backed by a SQLAlchemy connection.
+    While connecting will check TENTACLIO__PG_APPLICATION_NAME to set
+    the applicationName in the connection string.
+    """
+
+    TENTACLIO__PG_APPLICATION_NAME = "TENTACLIO__PG_APPLICATION_NAME"
+
+    allowed_schemes = ["postgresql"]
+
+    def _extract_url_params(self):
+        super()._extract_url_params()
+        application_name = os.getenv(self.TENTACLIO__PG_APPLICATION_NAME, "")
+        if application_name != "":
+            if self.url_query is None:
+                self.url_query: dict = {}
+            # overrides the value that might be set in the tentaclio file
+            self.url_query["application_name"] = application_name
+
+    # Postgres Copy Expert methods:
+    @decorators.check_conn
+    def get_df_unsafe(self, sql_query: str, params: None = None, **kwargs) -> pd.DataFrame:
+        """Run a raw SQL query and return a data frame using COPY.
+        Params:
+            sql_query: query to execute
+            params: not supported; must be None
+            **kwargs: additional kwargs to pass to `pandas.read_csv`
+        """
+        if params is not None:
+            raise NotImplementedError(
+                "Support for `params` is not implemented; please provide a pre-formatted query."
+            )
+        copy_sql = f"COPY ({sql_query.rstrip(';')}) TO STDOUT WITH CSV HEADER"
+        buffer = io.StringIO()
+        raw_conn = self._get_raw_conn()
+        raw_conn.cursor().copy_expert(copy_sql, buffer)
+        buffer.seek(0)
+        df = pd.read_csv(buffer, **kwargs)
+        return df
+
+    @decorators.check_conn
+    def dump_df(self, df: pd.DataFrame, dest_table: str) -> None:
+        """Dump a data frame into an existing Postgres table."""
+        buff = io.StringIO()
+        df.to_csv(buff, index=False)
+        buff.seek(0)
+        self._copy_expert_csv(buff, df.columns, dest_table)
+
+    @decorators.check_conn
+    def dump_csv(
+        self, csv_reader: protocols.Reader, columns: Sequence[str], dest_table: str
+    ) -> None:
+        """Dump a csv reader into the database."""
+        self._copy_expert_csv(csv_reader, columns, dest_table)
+
+    def _copy_expert_csv(
+        self, csv_reader: protocols.Reader, columns: Sequence[str], dest_table: str
+    ) -> None:
+        """Dump a csv reader into the given table."""
+        sql_columns = ",".join(columns)
+        sql_query = f"""COPY {dest_table} ({sql_columns}) FROM STDIN
+                        WITH CSV HEADER DELIMITER AS ','
+                        NULL AS 'NULL';"""
+        raw_conn = self._get_raw_conn()
+        try:
+            raw_conn.cursor().copy_expert(sql_query, csv_reader)
+        except Exception:
+            raw_conn.rollback()
+            raise
+        else:
+            raw_conn.commit()

tentaclio_postgres/streams/__init__.py

@@ -0,0 +1 @@
+from .postgres_handler import *  # noqa

tentaclio_postgres/streams/postgres_handler.py

@@ -0,0 +1,48 @@
+"""Handler for postgresql:// urls."""
+
+from typing import Tuple
+
+from tentaclio.protocols import ReaderClosable, WriterClosable
+from tentaclio.streams import csv_db_stream
+from tentaclio.urls import URL
+
+from tentaclio_postgres.clients import postgres_client
+
+__all__ = ["PostgresURLHandler"]
+
+
+def _split_url(url: URL) -> Tuple[URL, str]:
+    parts = url.path.split("::")
+    if len(parts) != 2 or parts[1] == "":
+        raise ValueError("No table provided for dumping the csv data")
+    path = parts[0]
+    table = parts[1]
+    conn_url = URL.from_components(
+        scheme=url.scheme,
+        username=url.username,
+        password=url.password,
+        hostname=url.hostname,
+        port=url.port,
+        path=path,
+        query=url.query,
+    )
+    return conn_url, table
+
+
+class PostgresURLHandler:
+    """Handler for opening writers and readers ."""
+
+    def open_reader_for(self, url: URL, mode: str, extras: dict) -> ReaderClosable:
+        """Not implemented."""
+        raise NotImplementedError("Reading csv from postgres not implemented")
+
+    def open_writer_for(self, url: URL, mode: str, extras: dict) -> WriterClosable:
+        """Open writer to dump the csv into a table in database.
+        This stream writer only works for csv data at the moment. The url must contain
+        a table as the last part of the path separated by a doble colon i.e.
+        `postgresql://user:pass@host/database::table`. The target table needs the columns
+        with the same names as the field names.
+        """
+        conn_url, table = _split_url(url)
+        client = postgres_client.PostgresClient(conn_url, **extras)
+        return csv_db_stream.DatabaseCsvWriter(client, table)

tentaclio_postgres-0.1.0.dist-info/METADATA

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: tentaclio-postgres
+Version: 0.1.0
+Summary: A python project containing all the dependencies for schema postgres for tentaclio.
+Author-email: Octopus Energy <nerds@octopus.energy>
+License-Expression: MIT
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: tentaclio
+Requires-Dist: psycopg2-binary
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: Flake8-pyproject; extra == "dev"
+Requires-Dist: isort>=5.11.2; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Dynamic: license-file
+
+
+# tentaclio-postgres
+
+A package containing all the dependencies for the postgresql tentaclio schema .
+
+## Quick Start
+
+This project comes with a `Makefile` which is ready to do basic common tasks
+
+```
+$ make help
+install                       Initalise the virtual env installing deps
+clean                         Remove all the unwanted clutter
+lock                          Lock dependencies
+update                        Update dependencies (whole tree)
+sync                          Install dependencies as per the lock file
+lint                          Lint files with flake and mypy
+format                        Run black and isort
+test                          Run unit tests
+circleci                      Validate circleci configuration (needs circleci cli)
+```

tentaclio_postgres-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+tentaclio_postgres/__init__.py,sha256=zqft6-mFQd0yzbV5gfYo8WjBys-CBBIlOw2fExNgEW4,604
+tentaclio_postgres/__main__.py,sha256=Fx9uOFe0XHNiAC_hAwLBFO-6dYIxK8fGYDCA9mXbQwc,146
+tentaclio_postgres/clients/__init__.py,sha256=lULFwiVzNtWEOdAsCbf4I9ECDaXwrSeMmo_NkmlA_iA,294
+tentaclio_postgres/clients/postgres_client.py,sha256=SDWlX6O3gz2DCIYwv0vhpGCAZ-z50s8qZiN6dCKXJvM,3173
+tentaclio_postgres/streams/__init__.py,sha256=9D2OcQ70_WgTdsQTf7_fpeESudeNbYvonwK5JApF51k,40
+tentaclio_postgres/streams/postgres_handler.py,sha256=YDetA29332SZQ4YjAOeEB8NAB4vYB0rkCBl-_eMxC2c,1694
+tentaclio_postgres-0.1.0.dist-info/licenses/LICENSE.txt,sha256=Aqv2h5R53YlGaQXl7XgUDMZPUxJK1HftERN2ExXJ_u4,1080
+tentaclio_postgres-0.1.0.dist-info/METADATA,sha256=CzCf4_ILMtNi4Dq1TSZetpSg441JxZUKGjtq_J3ZuOk,1958
+tentaclio_postgres-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+tentaclio_postgres-0.1.0.dist-info/top_level.txt,sha256=Srpz8siGsW7_npK27Ct1E3p4WizTSI0P5UyUEqWZLJE,19
+tentaclio_postgres-0.1.0.dist-info/RECORD,,

tentaclio_postgres-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tentaclio_postgres-0.1.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Octopus Energy Limited.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tentaclio_postgres-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tentaclio_postgres