pgpack-dumper 0.2.1.1__cp313-cp313-macosx_11_0_arm64.whl → 0.3.0.0__cp313-cp313-macosx_11_0_arm64.whl

pgpack_dumper/common/__init__.py

@@ -1,5 +1,13 @@
+"""Common functions and classes."""
+
+from .columns import make_columns
 from .connector import PGConnector
 from .copy import CopyBuffer
+from .diagram import (
+    DBMetadata,
+    format_table,
+    transfer_diagram,
+)
 from .errors import (
     CopyBufferError,
     CopyBufferObjectError,
@@ -29,6 +37,7 @@ __all__ = (
     "CopyBufferObjectError",
     "CopyBufferTableNotDefined",
     "CopyReader",
+    "DBMetadata",
     "DumperLogger",
     "PGConnector",
     "PGObject",
@@ -38,9 +47,12 @@ __all__ = (
     "PGPackDumperWriteError",
     "StreamReader",
     "chunk_query",
+    "format_table",
+    "make_columns",
     "query_path",
     "query_template",
     "random_name",
     "read_metadata",
     "search_object",
+    "transfer_diagram",
 )

pgpack_dumper/common/columns.py

@@ -0,0 +1,30 @@
+from collections import OrderedDict
+
+from pgcopylib import PGOid
+from pgpack.common import PGParam
+
+
+def make_columns(
+    list_columns: list[str],
+    pgtypes: list[PGOid],
+    pgparam: list[PGParam],
+) -> OrderedDict[str, str]:
+    """Make DBMetadata.columns dictionary."""
+
+    columns = OrderedDict()
+
+    for col_name, pgtype, param in zip(
+        list_columns,
+        pgtypes,
+        pgparam,
+    ):
+        col_type = pgtype.name
+
+        if pgtype is PGOid.bpchar:
+            col_type = f"{col_type}({param.length})"
+        elif pgtype is PGOid.numeric:
+            col_type = f"{col_type}({param.length}, {param.scale})"
+
+        columns[col_name] = col_type
+
+    return columns

pgpack_dumper/common/diagram.py

@@ -0,0 +1,78 @@
+from collections import OrderedDict
+from typing import NamedTuple
+
+
+class DBMetadata(NamedTuple):
+    """Database object."""
+
+    name: str
+    version: str
+    columns: OrderedDict
+
+
+def truncate_text(text: str, max_length: int) -> str:
+    """Truncate text and add ellipsis if too long."""
+
+    if len(text) > max_length:
+        return text[: max_length - 1] + "…"
+    return text
+
+
+def format_table(
+    metadata: DBMetadata,
+    direction: str,
+    table_width: int = 51,
+) -> list[str]:
+    """Format single table as list of lines."""
+
+    lines = []
+
+    title = f"{direction} [{metadata.name} {metadata.version}]"
+    lines.append(f"┌{''.ljust(table_width, '─')}┐")
+    lines.append(
+        f"│ {truncate_text(title, table_width - 1).ljust(table_width - 1)}│"
+    )
+    lines.append(f"╞{'═' * 25}╤{'═' * 25}╡")
+    lines.append(f"│ {'Column Name'.ljust(23)} │ {'Data Type'.ljust(23)} │")
+    lines.append(f"╞{'═' * 25}╪{'═' * 25}╡")
+
+    for i, (col_name, col_type) in enumerate(metadata.columns.items()):
+        truncated_name = truncate_text(col_name, 23)
+        truncated_type = truncate_text(str(col_type), 23)
+        lines.append(
+            f"│ {truncated_name.ljust(23)} │ {truncated_type.ljust(23)} │"
+        )
+        if i < len(metadata.columns) - 1:
+            lines.append(f"├{'─' * 25}┼{'─' * 25}┤")
+
+    lines.append(f"└{'─' * 25}┴{'─' * 25}┘")
+    return lines
+
+
+def transfer_diagram(source: DBMetadata, destination: DBMetadata) -> str:
+    """Make transfer diagram with two tables and arrow."""
+
+    src_lines = format_table(source, "Source")
+    dest_lines = format_table(destination, "Destination")
+    max_lines = max(len(src_lines), len(dest_lines), 9)
+
+    src_lines.extend([" " * 53] * (max_lines - len(src_lines)))
+    dest_lines.extend([" " * 53] * (max_lines - len(dest_lines)))
+
+    middle_line = max_lines // 2
+    arrow_config = [
+        (middle_line - 3, " │╲   "),
+        (middle_line - 2, " │ ╲  "),
+        (middle_line - 1, "┌┘  ╲ "),
+        (middle_line, "│    ╲"),
+        (middle_line + 1, "│    ╱"),
+        (middle_line + 2, "└┐  ╱ "),
+        (middle_line + 3, " │ ╱  "),
+        (middle_line + 4, " │╱   "),
+    ]
+    arrow_map = {line: arrow for line, arrow in arrow_config}
+
+    return "Transfer data diagram:\n" + "\n".join(
+        f"{src_lines[row]} {arrow_map.get(row, '      ')} {dest_lines[row]}"
+        for row in range(max_lines)
+    )

pgpack_dumper/common/queryes/dbname.sql

@@ -0,0 +1,2 @@
+select case when count(nspname) = 1 then 'greenplum' else 'postgres' end as dbname
+from pg_catalog.pg_namespace where nspname = 'gp_toolkit';

pgpack_dumper/common/queryes/gpversion.sql

@@ -0,0 +1 @@
+select substring(version() from 'Greenplum Database (.*?) build') as gp_version;

pgpack_dumper/dumper.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+from collections.abc import Generator
 from io import (
     BufferedReader,
     BufferedWriter,
@@ -7,7 +9,8 @@ from types import MethodType
 from typing import (
     Any,
     Iterable,
-    Union
+    Iterator,
+    Union,
 )
 
 from pgcopylib import PGCopyWriter
@@ -19,6 +22,7 @@ from pgpack import (
 )
 from psycopg import (
     Connection,
+    Copy,
     Cursor,
 )
 from pandas import DataFrame as PdFrame
@@ -27,6 +31,7 @@ from sqlparse import format as sql_format
 
 from .common import (
     CopyBuffer,
+    DBMetadata,
     DumperLogger,
     PGConnector,
     PGPackDumperError,
@@ -35,6 +40,9 @@ from .common import (
     PGPackDumperWriteBetweenError,
     StreamReader,
     chunk_query,
+    make_columns,
+    query_template,
+    transfer_diagram,
 )
 
 
@@ -61,14 +69,24 @@ class PGPackDumper:
             )
             self.cursor: Cursor = self.connect.cursor()
             self.copy_buffer: CopyBuffer = CopyBuffer(self.cursor, self.logger)
+            self._dbmeta: DBMetadata | None = None
+            self._size = 0
         except Exception as error:
             self.logger.error(f"{error.__class__.__name__}: {error}")
             raise PGPackDumperError(error)
 
+        self.cursor.execute(query_template("dbname"))
+        self.dbname = self.cursor.fetchone()[0]
         self.version = (
             f"{self.connect.info.server_version // 10000}."
             f"{self.connect.info.server_version % 1000}"
         )
+
+        if self.dbname == "greenplum":
+            self.cursor.execute(query_template("gpversion"))
+            gpversion = self.cursor.fetchone()[0]
+            self.version = f"{self.version} gp {gpversion}"
+
         self.logger.info(
             f"PGPackDumper initialized for host {self.connector.host}"
             f"[version {self.version}]"
@@ -84,7 +102,7 @@ class PGPackDumper:
             second_part: list[str]
 
             self: PGPackDumper = args[0]
-            cursor: Cursor = kwargs.get("dumper_src", self).cursor
+            cursor: Cursor = (kwargs.get("dumper_src") or self).cursor
             query: str = kwargs.get("query_src") or kwargs.get("query")
             part: int = 1
             first_part, second_part = chunk_query(self.query_formatter(query))
@@ -138,21 +156,45 @@ class PGPackDumper:
     ) -> bool:
         """Internal method read_dump for generate kwargs to decorator."""
 
+        def __read_data(
+            copy_to: Iterator[Copy],
+        ) -> Generator[bytes, None, None]:
+            """Generate bytes from copy object with calc size."""
+
+            self._size = 0
+
+            for data in copy_to:
+                chunk = bytes(data)
+                self._size += len(chunk)
+                yield chunk
+
         try:
             self.copy_buffer.query = query
             self.copy_buffer.table_name = table_name
+            metadata = self.copy_buffer.metadata
             pgpack = PGPackWriter(
                 fileobj,
-                self.copy_buffer.metadata,
+                metadata,
                 self.compression_method,
             )
+            columns = make_columns(*metadata_reader(metadata))
+            source = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=columns,
+            )
+            destination = DBMetadata(
+                name="file",
+                version=fileobj.name,
+                columns=columns,
+            )
+            self.logger.info(transfer_diagram(source, destination))
 
             with self.copy_buffer.copy_to() as copy_to:
-                pgpack.from_bytes(bytes(data) for data in copy_to)
+                pgpack.from_bytes(__read_data(copy_to))
 
-            size = pgpack.tell()
             pgpack.close()
-            self.logger.info(f"Successfully read {size} bytes.")
+            self.logger.info(f"Successfully read {self._size} bytes.")
             self.logger.info(
                 f"Read pgpack dump from {self.connector.host} done."
             )
@@ -183,10 +225,14 @@ class PGPackDumper:
                     query_src,
                     table_src,
                 )
+                src_dbname = self.dbname
+                src_version = self.version
             elif dumper_src.__class__ is PGPackDumper:
                 source_copy_buffer = dumper_src.copy_buffer
                 source_copy_buffer.table_name = table_src
                 source_copy_buffer.query = query_src
+                src_dbname = dumper_src.dbname
+                src_version = dumper_src.version
             else:
                 reader = dumper_src.to_reader(
                     query=query_src,
@@ -196,12 +242,28 @@ class PGPackDumper:
                 self.from_rows(
                     dtype_data=dtype_data,
                     table_name=table_dest,
+                    source=dumper_src._dbmeta,
                 )
                 size = reader.tell()
                 self.logger.info(f"Successfully sending {size} bytes.")
                 return reader.close()
 
             self.copy_buffer.table_name = table_dest
+            source = DBMetadata(
+                name=src_dbname,
+                version=src_version,
+                columns=make_columns(
+                    *metadata_reader(source_copy_buffer.metadata),
+                ),
+            )
+            destination = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=make_columns(
+                    *metadata_reader(self.copy_buffer.metadata),
+                ),
+            )
+            self.logger.info(transfer_diagram(source, destination))
             self.copy_buffer.copy_between(source_copy_buffer)
             self.connect.commit()
             return True
@@ -219,8 +281,16 @@ class PGPackDumper:
 
         self.copy_buffer.query = query
         self.copy_buffer.table_name = table_name
+        metadata = self.copy_buffer.metadata
+        self._dbmeta = DBMetadata(
+            name=self.dbname,
+            version=self.version,
+            columns=make_columns(
+                *metadata_reader(metadata),
+            ),
+        )
         return StreamReader(
-            self.copy_buffer.metadata,
+            metadata,
             self.copy_buffer.copy_to(),
         )
 
@@ -246,12 +316,27 @@ class PGPackDumper:
         """Write PGPack dump into PostgreSQL/GreenPlum."""
 
         try:
-            pgpack = PGPackReader(fileobj)
             self.copy_buffer.table_name = table_name
+            pgpack = PGPackReader(fileobj)
+            source = DBMetadata(
+                name="file",
+                version=fileobj.name,
+                columns=make_columns(
+                    pgpack.columns,
+                    pgpack.pgtypes,
+                    pgpack.pgparam,
+                ),
+            )
+            destination = DBMetadata(
+                name=self.dbname,
+                version=self.version,
+                columns=make_columns(
+                    *metadata_reader(self.copy_buffer.metadata),
+                ),
+            )
+            self.logger.info(transfer_diagram(source, destination))
             self.copy_buffer.copy_from(pgpack.to_bytes())
             self.connect.commit()
-            size = pgpack.tell()
-            self.logger.info(f"Successfully sending {size} bytes.")
             pgpack.close()
             self.refresh()
         except Exception as error:
@@ -290,13 +375,31 @@ class PGPackDumper:
         self,
         dtype_data: Iterable[Any],
         table_name: str,
+        source: DBMetadata | None = None,
     ) -> None:
         """Write from python iterable object
         into PostgreSQL/GreenPlum table."""
 
+        if not source:
+            source = DBMetadata(
+                name="python",
+                version="iterable object",
+                columns={"Unknown": "Unknown"},
+            )
+
         self.copy_buffer.table_name = table_name
-        _, pgtypes, _ = metadata_reader(self.copy_buffer.metadata)
+        columns, pgtypes, pgparam = metadata_reader(self.copy_buffer.metadata)
         writer = PGCopyWriter(None, pgtypes)
+        destination = DBMetadata(
+            name=self.dbname,
+            version=self.version,
+            columns=make_columns(
+                list_columns=columns,
+                pgtypes=pgtypes,
+                pgparam=pgparam,
+            ),
+        )
+        self.logger.info(transfer_diagram(source, destination))
         self.copy_buffer.copy_from(writer.from_rows(dtype_data))
         self.connect.commit()
         self.refresh()
@@ -311,6 +414,14 @@ class PGPackDumper:
         self.from_rows(
             dtype_data=iter(data_frame.values),
             table_name=table_name,
+            source=DBMetadata(
+                name="pandas",
+                version="DataFrame",
+                columns=OrderedDict(zip(
+                    data_frame.columns,
+                    [str(dtype) for dtype in data_frame.dtypes],
+                )),
+            )
         )
 
     def from_polars(
@@ -323,6 +434,14 @@ class PGPackDumper:
         self.from_rows(
             dtype_data=data_frame.iter_rows(),
             table_name=table_name,
+            source=DBMetadata(
+                name="polars",
+                version="DataFrame",
+                columns=OrderedDict(zip(
+                    data_frame.columns,
+                    [str(dtype) for dtype in data_frame.dtypes],
+                )),
+            )
         )
 
     def refresh(self) -> None:

pgpack_dumper/version.py

@@ -1 +1 @@
-__version__ = "0.2.1.1"
+__version__ = "0.3.0.0"

{pgpack_dumper-0.2.1.1.dist-info → pgpack_dumper-0.3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgpack_dumper
-Version: 0.2.1.1
+Version: 0.3.0.0
 Summary: Library for read and write PGPack format between PostgreSQL and file.
 Author-email: 0xMihalich <bayanmobile87@gmail.com>
 Project-URL: Homepage, https://github.com/0xMihalich/pgpack_dumper
@@ -13,12 +13,13 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pgpack==0.3.0.8
-Requires-Dist: psycopg_binary>=3.2.10
-Requires-Dist: psycopg>=3.2.10
+Requires-Dist: pgpack==0.3.0.9
+Requires-Dist: psycopg_binary>=3.2.11
+Requires-Dist: psycopg>=3.2.11
 Requires-Dist: sqlparse>=0.5.3
 Dynamic: license-file
 

{pgpack_dumper-0.2.1.1.dist-info → pgpack_dumper-0.3.0.0.dist-info}/RECORD

@@ -1,26 +1,30 @@
-pgpack_dumper-0.2.1.1.dist-info/RECORD,,
-pgpack_dumper-0.2.1.1.dist-info/WHEEL,sha256=oqGJCpG61FZJmvyZ3C_0aCv-2mdfcY9e3fXvyUNmWfM,136
-pgpack_dumper-0.2.1.1.dist-info/top_level.txt,sha256=AhxLKWTyGtJWNLWUSyBu54xY4wjVS8vSXgW4wwq4bYk,14
-pgpack_dumper-0.2.1.1.dist-info/METADATA,sha256=tXrpK7qauZHn1aJjdB83tIHC8qb_2tG95JQTGmvbYaQ,5018
-pgpack_dumper-0.2.1.1.dist-info/licenses/LICENSE,sha256=9IYMxtPlcVkrfjeRPA9GRUmaQkLG_FL-Gx7HxLvcLDQ,1067
-pgpack_dumper/version.py,sha256=6XpKk8PqQElU3s4yNEnOro0ov1S0j4k_2mEnP9McEB4,24
+pgpack_dumper-0.3.0.0.dist-info/RECORD,,
+pgpack_dumper-0.3.0.0.dist-info/WHEEL,sha256=oqGJCpG61FZJmvyZ3C_0aCv-2mdfcY9e3fXvyUNmWfM,136
+pgpack_dumper-0.3.0.0.dist-info/top_level.txt,sha256=AhxLKWTyGtJWNLWUSyBu54xY4wjVS8vSXgW4wwq4bYk,14
+pgpack_dumper-0.3.0.0.dist-info/METADATA,sha256=m8GpfNmtDaxEgZW-imd67SfDs_LXTlOsL6ZBuvuJe6s,5069
+pgpack_dumper-0.3.0.0.dist-info/licenses/LICENSE,sha256=9IYMxtPlcVkrfjeRPA9GRUmaQkLG_FL-Gx7HxLvcLDQ,1067
+pgpack_dumper/version.py,sha256=QjDEkJ4u3QAQZ3CwLE5X_wJEerb9U3SSlJbnhJdcrxg,24
 pgpack_dumper/__init__.py,sha256=2x9LRaKPMjMuOoP9pBAeSRZoAeCo9B8fxFbnW_N7kh8,966
-pgpack_dumper/dumper.py,sha256=M6cPnDLWT9dJheduNIA6g0WqwS7vzQbWIGYKnB_QijM,10288
+pgpack_dumper/dumper.py,sha256=5WnXg69hszHjzPXRoAMAvFY5ZrVsKVtx48XMsLDG0og,14261
 pgpack_dumper/common/metadata.py,sha256=Z_vbgi1V85jgPR0h1xbUtt3Ru7vMcmG06bd13Gn_ocY,864
+pgpack_dumper/common/diagram.py,sha256=2w5-s5qwYjCd0cqqlBRbHQ8E90jU7Ay7TNLeROHjCa0,2505
 pgpack_dumper/common/query.py,sha256=5kMXFJvBWCKW3hYfqsI7F1qc1I786IPNGIhMXypIv1E,1359
 pgpack_dumper/common/reader.pyx,sha256=vRIR558enMtx4gueWtoujJbpRoWbKnLWx_foVR-rxIw,1821
-pgpack_dumper/common/__init__.py,sha256=_SvvDUz9lykOlIk7Y05f3ZpS1nMZAtQJ_7TNT_zD1IE,1008
+pgpack_dumper/common/__init__.py,sha256=gz-xk9TgMHVx5hlFnipO3kuLp7k3grb-OhoSP5Y7LvU,1242
 pgpack_dumper/common/copy.py,sha256=3mzGGVOVMdnUXqmpdXu3THQJpJB85D9LQN-MC-LX-fg,4941
 pgpack_dumper/common/logger.py,sha256=gsFCOjnCU3Hvg5UGYMVrPjKRCZKRFtBFBX-7QLZshFo,1675
 pgpack_dumper/common/stream.py,sha256=PXWcxmDmuK7ueI3MTUfQnmXKWz3A99W-knNRvq_-ZG0,2245
-pgpack_dumper/common/reader.cpython-313-darwin.so,sha256=AmP8XuHzqBLyldKpA3Kw4sf1NwRLLFN274p9HIHUwd4,115872
+pgpack_dumper/common/reader.cpython-313-darwin.so,sha256=LSD0XVgbm0VYcr406cC_WQBFjsQo76PnmKId5ZU6Pew,115872
+pgpack_dumper/common/columns.py,sha256=Pa8bzhoi8tXbm3GlgVeHYM3auPORD41q-L29JF_pRqM,693
 pgpack_dumper/common/structs.py,sha256=gS7vT-_yZ6R45j212B0kYKCJgHalxkdEN84Itctv7Ok,1033
 pgpack_dumper/common/connector.py,sha256=WGBovyXWxgeseGXploUW0L_4R0GQBtuInl89zqvbKLY,176
 pgpack_dumper/common/reader.pxd,sha256=flQddQni7UgYU_XPGpnILc7GVDFYjRgq-tbKRTUPbE8,252
 pgpack_dumper/common/reader.pyi,sha256=UoW-mEAw_G7TLBbIoPTJwpdtucv5IygHdkHc_ZcVM5Y,675
 pgpack_dumper/common/errors.py,sha256=1WIe9PUq5-Q6VNWyWAaXWr7xuXS2ixJCUB5sDOi5Xgs,591
 pgpack_dumper/common/queryes/copy_from.sql,sha256=hEHfzW7P9-q74jtgMMA94kGpiMeyURs0pnFLbmqgdW0,50
+pgpack_dumper/common/queryes/gpversion.sql,sha256=iUwW40pFCVAZhDZ6ah8CIW75_KMBLyvUxMyqOI7vxtA,80
 pgpack_dumper/common/queryes/relkind.sql,sha256=wpyWaIiuCqQ0YaFPsNAEwwxSxuErNnRmP__-bfZ1vho,66
 pgpack_dumper/common/queryes/copy_to.sql,sha256=AUSsFl_WFB88UcC45SLd8lD_1La1MIkV2CKX-gCOxEE,49
 pgpack_dumper/common/queryes/attributes.sql,sha256=TriT025-yo7k9Bba3mKmpEYZiutE6ml4-5pAnGMemSg,394
+pgpack_dumper/common/queryes/dbname.sql,sha256=Z3b6a_566PxA52ywRJw1JDSCUMC_c-FyulvW8mC1C7c,141
 pgpack_dumper/common/queryes/prepare.sql,sha256=oqRk3GU0gjFzM0nLNsKWeD8GfeThZVKsFiMY1-tdb8g,182