pgpack-dumper 0.2.1.1__cp310-cp310-win_amd64.whl → 0.3.0.0__cp310-cp310-win_amd64.whl

pgpack_dumper/common/__init__.py

@@ -1,5 +1,13 @@
+"""Common functions and classes."""
+
+from .columns import make_columns
 from .connector import PGConnector
 from .copy import CopyBuffer
+from .diagram import (
+    DBMetadata,
+    format_table,
+    transfer_diagram,
+)
 from .errors import (
     CopyBufferError,
     CopyBufferObjectError,
@@ -29,6 +37,7 @@ __all__ = (
     "CopyBufferObjectError",
     "CopyBufferTableNotDefined",
     "CopyReader",
+    "DBMetadata",
     "DumperLogger",
     "PGConnector",
     "PGObject",
@@ -38,9 +47,12 @@ __all__ = (
     "PGPackDumperWriteError",
     "StreamReader",
     "chunk_query",
+    "format_table",
+    "make_columns",
     "query_path",
     "query_template",
     "random_name",
     "read_metadata",
     "search_object",
+    "transfer_diagram",
 )

pgpack_dumper/common/columns.py

@@ -0,0 +1,30 @@
+from collections import OrderedDict
+
+from pgcopylib import PGOid
+from pgpack.common import PGParam
+
+
+def make_columns(
+    list_columns: list[str],
+    pgtypes: list[PGOid],
+    pgparam: list[PGParam],
+) -> OrderedDict[str, str]:
+    """Make DBMetadata.columns dictionary."""
+
+    columns = OrderedDict()
+
+    for col_name, pgtype, param in zip(
+        list_columns,
+        pgtypes,
+        pgparam,
+    ):
+        col_type = pgtype.name
+
+        if pgtype is PGOid.bpchar:
+            col_type = f"{col_type}({param.length})"
+        elif pgtype is PGOid.numeric:
+            col_type = f"{col_type}({param.length}, {param.scale})"
+
+        columns[col_name] = col_type
+
+    return columns

pgpack_dumper/common/diagram.py

@@ -0,0 +1,78 @@
+from collections import OrderedDict
+from typing import NamedTuple
+
+
+class DBMetadata(NamedTuple):
+    """Database object."""
+
+    name: str
+    version: str
+    columns: OrderedDict
+
+
+def truncate_text(text: str, max_length: int) -> str:
+    """Truncate text and add ellipsis if too long."""
+
+    if len(text) > max_length:
+        return text[: max_length - 1] + "…"
+    return text
+
+
+def format_table(
+    metadata: DBMetadata,
+    direction: str,
+    table_width: int = 51,
+) -> list[str]:
+    """Format single table as list of lines."""
+
+    lines = []
+
+    title = f"{direction} [{metadata.name} {metadata.version}]"
+    lines.append(f"┌{''.ljust(table_width, '─')}┐")
+    lines.append(
+        f"│ {truncate_text(title, table_width - 1).ljust(table_width - 1)}│"
+    )
+    lines.append(f"╞{'═' * 25}╤{'═' * 25}╡")
+    lines.append(f"│ {'Column Name'.ljust(23)} │ {'Data Type'.ljust(23)} │")
+    lines.append(f"╞{'═' * 25}╪{'═' * 25}╡")
+
+    for i, (col_name, col_type) in enumerate(metadata.columns.items()):
+        truncated_name = truncate_text(col_name, 23)
+        truncated_type = truncate_text(str(col_type), 23)
+        lines.append(
+            f"│ {truncated_name.ljust(23)} │ {truncated_type.ljust(23)} │"
+        )
+        if i < len(metadata.columns) - 1:
+            lines.append(f"├{'─' * 25}┼{'─' * 25}┤")
+
+    lines.append(f"└{'─' * 25}┴{'─' * 25}┘")
+    return lines
+
+
+def transfer_diagram(source: DBMetadata, destination: DBMetadata) -> str:
+    """Make transfer diagram with two tables and arrow."""
+
+    src_lines = format_table(source, "Source")
+    dest_lines = format_table(destination, "Destination")
+    max_lines = max(len(src_lines), len(dest_lines), 9)
+
+    src_lines.extend([" " * 53] * (max_lines - len(src_lines)))
+    dest_lines.extend([" " * 53] * (max_lines - len(dest_lines)))
+
+    middle_line = max_lines // 2
+    arrow_config = [
+        (middle_line - 3, " │╲   "),
+        (middle_line - 2, " │ ╲  "),
+        (middle_line - 1, "┌┘  ╲ "),
+        (middle_line, "│    ╲"),
+        (middle_line + 1, "│    ╱"),
+        (middle_line + 2, "└┐  ╱ "),
+        (middle_line + 3, " │ ╱  "),
+        (middle_line + 4, " │╱   "),
+    ]
+    arrow_map = {line: arrow for line, arrow in arrow_config}
+
+    return "Transfer data diagram:\n" + "\n".join(
+        f"{src_lines[row]} {arrow_map.get(row, '      ')} {dest_lines[row]}"
+        for row in range(max_lines)
+    )

pgpack_dumper/common/queryes/dbname.sql

@@ -0,0 +1,2 @@
+select case when count(nspname) = 1 then 'greenplum' else 'postgres' end as dbname
+from pg_catalog.pg_namespace where nspname = 'gp_toolkit';

pgpack_dumper/common/queryes/gpversion.sql

@@ -0,0 +1 @@
+select substring(version() from 'Greenplum Database (.*?) build') as gp_version;

pgpack_dumper/dumper.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+from collections.abc import Generator
 from io import (
     BufferedReader,
     BufferedWriter,
@@ -7,7 +9,8 @@ from types import MethodType
 from typing import (
     Any,
     Iterable,
-    Union
+    Iterator,
+    Union,
 )
 
 from pgcopylib import PGCopyWriter
@@ -19,6 +22,7 @@ from pgpack import (
 )
 from psycopg import (
     Connection,
+    Copy,
     Cursor,
 )
 from pandas import DataFrame as PdFrame
@@ -27,6 +31,7 @@ from sqlparse import format as sql_format
 
 from .common import (
     CopyBuffer,
+    DBMetadata,
     DumperLogger,
     PGConnector,
     PGPackDumperError,
@@ -35,6 +40,9 @@ from .common import (
     PGPackDumperWriteBetweenError,
     StreamReader,
     chunk_query,
+    make_columns,
+    query_template,
+    transfer_diagram,
 )
 
 
@@ -61,14 +69,24 @@ class PGPackDumper:
             )
             self.cursor: Cursor = self.connect.cursor()
             self.copy_buffer: CopyBuffer = CopyBuffer(self.cursor, self.logger)
+            self._dbmeta: DBMetadata | None = None
+            self._size = 0
         except Exception as error:
             self.logger.error(f"{error.__class__.__name__}: {error}")
             raise PGPackDumperError(error)
 
+        self.cursor.execute(query_template("dbname"))
+        self.dbname = self.cursor.fetchone()[0]
         self.version = (
             f"{self.connect.info.server_version // 10000}."
             f"{self.connect.info.server_version % 1000}"
         )
+
+        if self.dbname == "greenplum":
+            self.cursor.execute(query_template("gpversion"))
+            gpversion = self.cursor.fetchone()[0]
+            self.version = f"{self.version} gp {gpversion}"
+
         self.logger.info(
             f"PGPackDumper initialized for host {self.connector.host}"
             f"[version {self.version}]"
@@ -84,7 +102,7 @@ class PGPackDumper:
             second_part: list[str]
 
             self: PGPackDumper = args[0]
-            cursor: Cursor = kwargs.get("dumper_src", self).cursor
+            cursor: Cursor = (kwargs.get("dumper_src") or self).cursor
             query: str = kwargs.get("query_src") or kwargs.get("query")
             part: int = 1
             first_part, second_part = chunk_query(self.query_formatter(query))
@@ -138,21 +156,45 @@ class PGPackDumper:
     ) -> bool:
         """Internal method read_dump for generate kwargs to decorator."""
 
+        def __read_data(
+            copy_to: Iterator[Copy],
+        ) -> Generator[bytes, None, None]:
+            """Generate bytes from copy object with calc size."""
+
+            self._size = 0
+
+            for data in copy_to:
+                chunk = bytes(data)
+                self._size += len(chunk)
+                yield chunk
+
         try:
             self.copy_buffer.query = query
             self.copy_buffer.table_name = table_name
+            metadata = self.copy_buffer.metadata
             pgpack = PGPackWriter(
                 fileobj,
-                self.copy_buffer.metadata,
+                metadata,
                 self.compression_method,
             )
+            columns = make_columns(*metadata_reader(metadata))
+            source = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=columns,
+            )
+            destination = DBMetadata(
+                name="file",
+                version=fileobj.name,
+                columns=columns,
+            )
+            self.logger.info(transfer_diagram(source, destination))
 
             with self.copy_buffer.copy_to() as copy_to:
-                pgpack.from_bytes(bytes(data) for data in copy_to)
+                pgpack.from_bytes(__read_data(copy_to))
 
-            size = pgpack.tell()
             pgpack.close()
-            self.logger.info(f"Successfully read {size} bytes.")
+            self.logger.info(f"Successfully read {self._size} bytes.")
             self.logger.info(
                 f"Read pgpack dump from {self.connector.host} done."
             )
@@ -183,10 +225,14 @@ class PGPackDumper:
                     query_src,
                     table_src,
                 )
+                src_dbname = self.dbname
+                src_version = self.version
             elif dumper_src.__class__ is PGPackDumper:
                 source_copy_buffer = dumper_src.copy_buffer
                 source_copy_buffer.table_name = table_src
                 source_copy_buffer.query = query_src
+                src_dbname = dumper_src.dbname
+                src_version = dumper_src.version
             else:
                 reader = dumper_src.to_reader(
                     query=query_src,
@@ -196,12 +242,28 @@ class PGPackDumper:
                 self.from_rows(
                     dtype_data=dtype_data,
                     table_name=table_dest,
+                    source=dumper_src._dbmeta,
                 )
                 size = reader.tell()
                 self.logger.info(f"Successfully sending {size} bytes.")
                 return reader.close()
 
             self.copy_buffer.table_name = table_dest
+            source = DBMetadata(
+                name=src_dbname,
+                version=src_version,
+                columns=make_columns(
+                    *metadata_reader(source_copy_buffer.metadata),
+                ),
+            )
+            destination = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=make_columns(
+                    *metadata_reader(self.copy_buffer.metadata),
+                ),
+            )
+            self.logger.info(transfer_diagram(source, destination))
             self.copy_buffer.copy_between(source_copy_buffer)
             self.connect.commit()
             return True
@@ -219,8 +281,16 @@ class PGPackDumper:
 
         self.copy_buffer.query = query
         self.copy_buffer.table_name = table_name
+        metadata = self.copy_buffer.metadata
+        self._dbmeta = DBMetadata(
+            name=self.dbname,
+            version=self.version,
+            columns=make_columns(
+                *metadata_reader(metadata),
+            ),
+        )
         return StreamReader(
-            self.copy_buffer.metadata,
+            metadata,
             self.copy_buffer.copy_to(),
         )
 
@@ -246,12 +316,27 @@ class PGPackDumper:
         """Write PGPack dump into PostgreSQL/GreenPlum."""
 
         try:
-            pgpack = PGPackReader(fileobj)
             self.copy_buffer.table_name = table_name
+            pgpack = PGPackReader(fileobj)
+            source = DBMetadata(
+                name="file",
+                version=fileobj.name,
+                columns=make_columns(
+                    pgpack.columns,
+                    pgpack.pgtypes,
+                    pgpack.pgparam,
+                ),
+            )
+            destination = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=make_columns(
+                    *metadata_reader(self.copy_buffer.metadata),
+                ),
+            )
+            self.logger.info(transfer_diagram(source, destination))
             self.copy_buffer.copy_from(pgpack.to_bytes())
             self.connect.commit()
-            size = pgpack.tell()
-            self.logger.info(f"Successfully sending {size} bytes.")
             pgpack.close()
             self.refresh()
         except Exception as error:
@@ -290,13 +375,31 @@ class PGPackDumper:
         self,
         dtype_data: Iterable[Any],
         table_name: str,
+        source: DBMetadata | None = None,
     ) -> None:
         """Write from python iterable object
         into PostgreSQL/GreenPlum table."""
 
+        if not source:
+            source = DBMetadata(
+                name="python",
+                version="iterable object",
+                columns={"Unknown": "Unknown"},
+            )
+
         self.copy_buffer.table_name = table_name
-        _, pgtypes, _ = metadata_reader(self.copy_buffer.metadata)
+        columns, pgtypes, pgparam = metadata_reader(self.copy_buffer.metadata)
         writer = PGCopyWriter(None, pgtypes)
+        destination = DBMetadata(
+            name=self.dbname,
+            version=self.version,
+            columns=make_columns(
+                list_columns=columns,
+                pgtypes=pgtypes,
+                pgparam=pgparam,
+            ),
+        )
+        self.logger.info(transfer_diagram(source, destination))
         self.copy_buffer.copy_from(writer.from_rows(dtype_data))
         self.connect.commit()
         self.refresh()
@@ -311,6 +414,14 @@ class PGPackDumper:
         self.from_rows(
             dtype_data=iter(data_frame.values),
             table_name=table_name,
+            source=DBMetadata(
+                name="pandas",
+                version="DataFrame",
+                columns=OrderedDict(zip(
+                    data_frame.columns,
+                    [str(dtype) for dtype in data_frame.dtypes],
+                )),
+            )
         )
 
     def from_polars(
@@ -323,6 +434,14 @@ class PGPackDumper:
         self.from_rows(
             dtype_data=data_frame.iter_rows(),
             table_name=table_name,
+            source=DBMetadata(
+                name="polars",
+                version="DataFrame",
+                columns=OrderedDict(zip(
+                    data_frame.columns,
+                    [str(dtype) for dtype in data_frame.dtypes],
+                )),
+            )
         )
 
     def refresh(self) -> None:

pgpack_dumper/version.py

@@ -1 +1 @@
-__version__ = "0.2.1.1"
+__version__ = "0.3.0.0"

{pgpack_dumper-0.2.1.1.dist-info → pgpack_dumper-0.3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgpack_dumper
-Version: 0.2.1.1
+Version: 0.3.0.0
 Summary: Library for read and write PGPack format between PostgreSQL and file.
 Author-email: 0xMihalich <bayanmobile87@gmail.com>
 Project-URL: Homepage, https://github.com/0xMihalich/pgpack_dumper
@@ -13,12 +13,13 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pgpack==0.3.0.8
-Requires-Dist: psycopg_binary>=3.2.10
-Requires-Dist: psycopg>=3.2.10
+Requires-Dist: pgpack==0.3.0.9
+Requires-Dist: psycopg_binary>=3.2.11
+Requires-Dist: psycopg>=3.2.11
 Requires-Dist: sqlparse>=0.5.3
 Dynamic: license-file
 

{pgpack_dumper-0.2.1.1.dist-info → pgpack_dumper-0.3.0.0.dist-info}/RECORD

@@ -1,14 +1,16 @@
 pgpack_dumper/__init__.py,sha256=wl4I02A6U4A6H3iZC4xNC1N-Y2f9GpgW8lSdbFAR-eA,1012
-pgpack_dumper/dumper.py,sha256=nyue-ldOjuLJU3KZZW3seWLMUE1XxD0O85UrhycxUAY,10629
-pgpack_dumper/version.py,sha256=alMv2ocBwLyPN1exsc6K4ug7N3X7Nu0Nwv7RWitPtDs,25
-pgpack_dumper/common/__init__.py,sha256=fOjEYtpkuafk8klKzPrioT1Bbf_Fp89hraHyJe8HmdQ,1054
+pgpack_dumper/dumper.py,sha256=d5JgkMGJjoK4hXte7AVpFbg-QSyDLMQy1-6uVpMYOWI,14721
+pgpack_dumper/version.py,sha256=7QLWsdocVb_VhH8H-ER-x1bX4v3i5iRdbn82QyaGJ3Q,25
+pgpack_dumper/common/__init__.py,sha256=sWiUnLyvYZZJNIZYWl6MmntIUyDm-naM3DVoV3YcNDA,1300
+pgpack_dumper/common/columns.py,sha256=sV3e0IErvuBZvIsATSQx_hajBp59VTmni5EmSAdyRBY,723
 pgpack_dumper/common/connector.py,sha256=3SIGIDTd9OCdr4k5uiPJxzHMGifvsJAEotzzY1GBZgU,187
 pgpack_dumper/common/copy.py,sha256=9YORQ05GEQgA8y-rcLEYfV5QQQvUnUpFc3wxd-7v5IE,5098
+pgpack_dumper/common/diagram.py,sha256=IkmRtKF6Jt4z0cMWpAYGCZ4LwhNWQ4qkWYEJd8Xvd9I,2583
 pgpack_dumper/common/errors.py,sha256=m0nLUBkpOrodRTesI9WYlpy_MUJ5GPY1e-TaI7RP2LU,617
 pgpack_dumper/common/logger.py,sha256=PF7L1WYWkj4qU-CDpQTraqSiCPi_AZSmaUt1nscO5gM,1745
 pgpack_dumper/common/metadata.py,sha256=keknqsofg8nUfu_e38siM5CR-bj8hiiLHSRRXpDs_qU,902
 pgpack_dumper/common/query.py,sha256=fB1XLT_vK6LBS92p_qZfQyLZTmoplzGCcG63rgPZ-T8,1421
-pgpack_dumper/common/reader.cp310-win_amd64.pyd,sha256=QtksGsG4QQpqpZ3rLDJDrXztDSzw7bMANRIUY4-8Lsg,58880
+pgpack_dumper/common/reader.cp310-win_amd64.pyd,sha256=2fXkunYfhTs8qoJH6IHTgDw29v4iq0LrMHvJd5dxNu0,58880
 pgpack_dumper/common/reader.pxd,sha256=_CVyKYdSScWrDZ7-FeCafFvXSOJDN3KwWIE0dH07Zbo,263
 pgpack_dumper/common/reader.pyi,sha256=z4ZQMiHLUm1p7lDuWrhnQnv9_sv-GVmBa_5_0Ns1UZc,710
 pgpack_dumper/common/reader.pyx,sha256=dxu2igQiJvq3EzPRoTpsirxS0ZlQxqTrJrAoz-8Waso,1888
@@ -17,10 +19,12 @@ pgpack_dumper/common/structs.py,sha256=F2K2D7V6RqSPoZCp00huL6gj0_TSR7czjTvS6fC5D
 pgpack_dumper/common/queryes/attributes.sql,sha256=im-Rgu2u2ocf0oa1QNrHn9b51t2sbei49klbtx1QLVU,397
 pgpack_dumper/common/queryes/copy_from.sql,sha256=hEHfzW7P9-q74jtgMMA94kGpiMeyURs0pnFLbmqgdW0,50
 pgpack_dumper/common/queryes/copy_to.sql,sha256=AUSsFl_WFB88UcC45SLd8lD_1La1MIkV2CKX-gCOxEE,49
+pgpack_dumper/common/queryes/dbname.sql,sha256=yJN5CJF3KOrbHdOYmoj7DjFveEojjWsmWYuPeL7GyOQ,142
+pgpack_dumper/common/queryes/gpversion.sql,sha256=iUwW40pFCVAZhDZ6ah8CIW75_KMBLyvUxMyqOI7vxtA,80
 pgpack_dumper/common/queryes/prepare.sql,sha256=1pACITTlfQo1F_tEtOGigFCkWOkbxD421B4xJjjG70Y,185
 pgpack_dumper/common/queryes/relkind.sql,sha256=wpyWaIiuCqQ0YaFPsNAEwwxSxuErNnRmP__-bfZ1vho,66
-pgpack_dumper-0.2.1.1.dist-info/licenses/LICENSE,sha256=jW7Zeev0dRSYqbEnAk6qGz3e3yok-NhdTxP0ceNPlWM,1088
-pgpack_dumper-0.2.1.1.dist-info/METADATA,sha256=QiXwRuLuRPRBhM1fPO_JrxBYS074nbfw4YaI_l_JMeg,5218
-pgpack_dumper-0.2.1.1.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
-pgpack_dumper-0.2.1.1.dist-info/top_level.txt,sha256=AhxLKWTyGtJWNLWUSyBu54xY4wjVS8vSXgW4wwq4bYk,14
-pgpack_dumper-0.2.1.1.dist-info/RECORD,,
+pgpack_dumper-0.3.0.0.dist-info/licenses/LICENSE,sha256=jW7Zeev0dRSYqbEnAk6qGz3e3yok-NhdTxP0ceNPlWM,1088
+pgpack_dumper-0.3.0.0.dist-info/METADATA,sha256=7XT-EG2TaKdjIt4ae_RAxo-ZJ4bN_B7M8yPjNTjzEcg,5270
+pgpack_dumper-0.3.0.0.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
+pgpack_dumper-0.3.0.0.dist-info/top_level.txt,sha256=AhxLKWTyGtJWNLWUSyBu54xY4wjVS8vSXgW4wwq4bYk,14
+pgpack_dumper-0.3.0.0.dist-info/RECORD,,