pyLodStorage 0.19.2__tar.gz → 0.19.4__tar.gz

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyLodStorage
-Version: 0.19.2
+Version: 0.19.4
 Summary: python List of Dict (Table) Storage library
 Project-URL: Home, https://github.com/WolfgangFahl/pyLodStorage
 Project-URL: Documentation, http://wiki.bitplan.com/index.php/PyLoDStorage
@@ -35,6 +35,8 @@ Provides-Extra: duckdb
 Requires-Dist: duckdb>=1.1.0; extra == 'duckdb'
 Provides-Extra: mysql
 Requires-Dist: pymysql>=1.1.1; extra == 'mysql'
+Provides-Extra: postgresql
+Requires-Dist: psycopg2-binary>=2.9.9; extra == 'postgresql'
 Provides-Extra: test
 Requires-Dist: green; extra == 'test'
 Description-Content-Type: text/markdown

pylodstorage-0.19.4/lodstorage/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.19.4"

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/duckdb_query.py

@@ -40,13 +40,15 @@ class DuckDBQuery:
         self.debug = debug
         self.con = duckdb.connect(self.path)
 
-    def query(self, sql: str, params: Any = None) -> list[dict]:
+    def query(self, sql: str, params: Any = None, commit: bool = False) -> list[dict]:
         """
         Execute an SQL query and return all results eagerly.
 
         Args:
             sql: the SQL query string to execute
             params: optional positional parameters (list or tuple)
+            commit: if True, commit the connection after execution
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
 
         Returns:
             list of dicts, one per row
@@ -54,9 +56,15 @@ class DuckDBQuery:
         if self.debug:
             logging.debug(f"DuckDBQuery.query: {sql!r} params={params}")
         rel = self.con.execute(sql, params or [])
+        if rel.description is None:
+            if commit:
+                self.con.commit()
+            return []
         cols = [d[0] for d in rel.description]
         rows = rel.fetchall()
         result = [dict(zip(cols, row)) for row in rows]
+        if commit:
+            self.con.commit()
         return result
 
     def query_gen(self, sql: str, params: Any = None) -> Generator[dict, None, None]:

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/mysql.py

@@ -69,20 +69,28 @@ class MySqlQuery:
                 decoded_record[key] = value
         return decoded_record
 
-    def execute_sql_query(self, query: str) -> List[Dict[str, Any]]:
+    def execute_sql_query(
+        self, query: str, commit: bool = False
+    ) -> List[Dict[str, Any]]:
         """
         Executes an SQL query using the provided connection parameters.
 
         Args:
             query (str): The SQL query to execute.
-            connection_params (dict): Database connection parameters.
+            commit (bool): if True, commit the connection before closing
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
 
         Returns:
             list: A list of dictionaries representing the query results.
         """
         connection, cursor = self.get_cursor(query)
         cursor.execute(query)
-        raw_lod = cursor.fetchall()
+        if cursor.description is None:
+            raw_lod = []
+        else:
+            raw_lod = cursor.fetchall()
+        if commit:
+            connection.commit()
         connection.close()
         lod = []
         for raw_row in raw_lod:
@@ -108,18 +116,22 @@ class MySqlQuery:
             cursor.close()
             connection.close()
 
-    def query(self, sql: str, params: Any = None) -> List[Dict[str, Any]]:
+    def query(
+        self, sql: str, params: Any = None, commit: bool = False
+    ) -> List[Dict[str, Any]]:
         """
         SQLBackend protocol alias for execute_sql_query.
 
         Args:
             sql: the SQL query to execute
             params: ignored (not yet supported at the pymysql layer)
+            commit: if True, commit the connection after execution
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
 
         Returns:
             list of dicts, one per row
         """
-        return self.execute_sql_query(sql)
+        return self.execute_sql_query(sql, commit=commit)
 
     def query_gen(
         self, sql: str, params: Any = None

pylodstorage-0.19.4/lodstorage/postgresql.py

@@ -0,0 +1,152 @@
+"""
+postgresql.py:
+
+PostgreSQL support
+
+"""
+
+import logging
+from typing import Any, Dict, Generator, List
+
+import psycopg2
+import psycopg2.extras
+
+from lodstorage.query import Endpoint
+
+
+class PostgreSqlQuery:
+    """
+    A class to manage and execute PostgreSQL queries with optional debugging.
+
+    Attributes:
+        endpoint_info (Endpoint): endpoint configuration.
+        debug (bool): Flag to enable debugging.
+    """
+
+    def __init__(self, endpoint: Endpoint, debug: bool = False):
+        """
+        Initializes the PostgreSqlQuery class.
+
+        Args:
+            endpoint (Endpoint): endpoint configuration.
+            debug (bool): Flag to enable debugging.
+        """
+        self.db_params = {
+            "host": endpoint.host or "localhost",
+            "port": endpoint.port or 5432,
+            "user": endpoint.user or "postgres",
+            "password": endpoint.password,
+            "dbname": endpoint.database,
+        }
+
+        self.debug = debug
+
+    def get_cursor(self, query: str):
+        if self.debug:
+            logging.debug(f"Executing query: {query}")
+            logging.debug(f"With connection parameters: {self.db_params}")
+
+        connection = psycopg2.connect(**self.db_params)
+        cursor = connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+        return connection, cursor
+
+    def decode_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Converts binary values to UTF-8 strings.
+
+        Args:
+            record (Dict[str, Any]): Raw database row data
+
+        Returns:
+            Dict[str, Any]: Data with binary values decoded to strings
+        """
+        decoded_record = {}
+        for key, value in record.items():
+            if isinstance(value, (bytes, memoryview)):
+                decoded_record[key] = (
+                    bytes(value).decode("utf-8", errors="replace")
+                    if isinstance(value, memoryview)
+                    else value.decode("utf-8", errors="replace")
+                )
+            else:
+                decoded_record[key] = value
+        return decoded_record
+
+    def execute_sql_query(
+        self, query: str, commit: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        Executes an SQL query using the provided connection parameters.
+
+        Args:
+            query (str): The SQL query to execute.
+            commit (bool): if True, commit the connection before closing
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
+
+        Returns:
+            list: A list of dictionaries representing the query results.
+        """
+        connection, cursor = self.get_cursor(query)
+        cursor.execute(query)
+        if cursor.description is None:
+            raw_lod = []
+        else:
+            raw_lod = cursor.fetchall()
+        if commit:
+            connection.commit()
+        connection.close()
+        lod = []
+        for raw_row in raw_lod:
+            row = self.decode_record(dict(raw_row))
+            lod.append(row)
+        return lod
+
+    def query_generator(self, query: str) -> Generator[Dict[str, Any], None, None]:
+        """
+        Generator for fetching records one by one from a SQL query.
+        """
+        connection, cursor = self.get_cursor(query)
+        try:
+            cursor.execute(query)
+            while True:
+                raw_record = cursor.fetchone()
+                if not raw_record:
+                    break
+                record = self.decode_record(dict(raw_record))
+                yield record
+
+        finally:
+            cursor.close()
+            connection.close()
+
+    def query(
+        self, sql: str, params: Any = None, commit: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        SQLBackend protocol alias for execute_sql_query.
+
+        Args:
+            sql: the SQL query to execute
+            params: ignored (not yet supported at the psycopg2 layer)
+            commit: if True, commit the connection after execution
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
+
+        Returns:
+            list of dicts, one per row
+        """
+        return self.execute_sql_query(sql, commit=commit)
+
+    def query_gen(
+        self, sql: str, params: Any = None
+    ) -> Generator[Dict[str, Any], None, None]:
+        """
+        SQLBackend protocol alias for query_generator.
+
+        Args:
+            sql: the SQL query to execute
+            params: ignored (not yet supported at the psycopg2 layer)
+
+        Yields:
+            one dict per row
+        """
+        return self.query_generator(sql)

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/query_cmd.py

@@ -136,6 +136,11 @@ class QueryCmd:
             xml = lod2xml.asXml()
             print(xml)
 
+        elif args.format is Format.raw:
+            for row in qlod:
+                values = list(row.values())
+                line = "\t".join(str(v) for v in values)
+                print(line)
         else:
             raise Exception(f"format {args.format} not supported yet")
 

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/querymain.py

@@ -96,7 +96,7 @@ class QueryMain(QueryCmd):
                 qlod = sparql.queryAsListOfDicts(self.queryCode)
             elif args.language == "sql":
                 backend = get_sql_backend(endpointConf, debug=args.debug)
-                qlod = backend.query(self.queryCode)
+                qlod = backend.query(self.queryCode, commit=args.commit)
             else:
                 raise Exception(f"language {args.language} not known/supported")
             self.format_output(qlod)
@@ -267,6 +267,12 @@ USAGE
             action="store_true",
             help="return the raw query result from the endpoint. (MIME type defined over -f or -m)",
         )
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="commit the SQL transaction after execution "
+            "(required for DDL/DML statements such as CREATE/INSERT/UPDATE)",
+        )
         parser.add_argument(
             "-V", "--version", action="version", version=program_version_message
         )

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/sql.py

@@ -256,6 +256,11 @@ class SQLDB(object):
             query = cur.execute(sqlQuery, params)
         else:
             query = cur.execute(sqlQuery)
+        # description is None for statements that do not return rows
+        # (e.g. CREATE TABLE, INSERT, UPDATE, DELETE)
+        if query.description is None:
+            cur.close()
+            return
         colname = [d[0] for d in query.description]
         try:
             # loop over all rows
@@ -281,7 +286,7 @@ class SQLDB(object):
         """
         return self.queryGen(sql, params)
 
-    def query(self, sql, params=None):
+    def query(self, sql, params=None, commit: bool = False):
         """
         run the given sql query and return a list of Dicts
 
@@ -289,6 +294,8 @@ class SQLDB(object):
 
             sql(string): the SQL query to be executed
             params(tuple): the query params, if any
+            commit(bool): if True, commit the connection after execution
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
 
         Returns:
             list: a list of Dicts
@@ -296,6 +303,8 @@ class SQLDB(object):
         resultList = []
         for record in self.queryGen(sql, params):
             resultList.append(record)
+        if commit:
+            self.c.commit()
         return resultList
 
     def queryAll(self, entityInfo, fixDates=True):

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/sql_backend.py

@@ -5,7 +5,7 @@ Unified SQL backend abstraction for pyLoDStorage.
 
 Defines the SQLBackend Protocol (duck-typed interface) and the
 get_sql_backend() factory that resolves the correct backend from
-an endpoint descriptor, hiding the jdbc:mysql and jdbc:duckdb
+an endpoint descriptor, hiding the jdbc:mysql, jdbc:duckdb, and jdbc:postgresql
 routing details from all call sites.
 
 Created on 2026-02-21
@@ -27,6 +27,11 @@ try:
 except ImportError:
     DuckDBQuery = None  # type: ignore
 
+try:
+    from lodstorage.postgresql import PostgreSqlQuery
+except ImportError:
+    PostgreSqlQuery = None  # type: ignore
+
 
 @runtime_checkable
 class SQLBackend(Protocol):
@@ -40,15 +45,18 @@ class SQLBackend(Protocol):
         - lodstorage.sql.SQLDB              (SQLite, always available)
         - lodstorage.mysql.MySqlQuery       (MySQL/MariaDB, optional: pip install pyLodStorage[mysql])
         - lodstorage.duckdb_query.DuckDBQuery (DuckDB, optional: pip install pyLodStorage[duckdb])
+        - lodstorage.postgresql.PostgreSqlQuery (PostgreSQL, optional: pip install pyLodStorage[postgresql])
     """
 
-    def query(self, sql: str, params: Any = None) -> list[dict]:
+    def query(self, sql: str, params: Any = None, commit: bool = False) -> list[dict]:
         """
         Execute an SQL query and return all results eagerly.
 
         Args:
             sql: the SQL query string to execute
             params: optional query parameters (support depends on backend)
+            commit: if True, commit the underlying connection after execution
+                (required for DDL/DML statements such as CREATE/INSERT/UPDATE)
 
         Returns:
             list of dicts, one per row
@@ -82,7 +90,7 @@ def get_sql_backend(endpoint, debug: bool = False) -> SQLBackend:
         debug: enable debug output on the returned backend
 
     Returns:
-        SQLBackend — SQLDB, MySqlQuery, or DuckDBQuery depending on endpoint
+        SQLBackend — SQLDB, MySqlQuery, DuckDBQuery, or PostgreSqlQuery depending on endpoint
 
     Raises:
         Exception: if the requested backend's optional dependency is not installed
@@ -112,5 +120,14 @@ def get_sql_backend(endpoint, debug: bool = False) -> SQLBackend:
         backend = DuckDBQuery(endpoint=endpoint, debug=debug)
         return backend
 
+    if url.startswith("jdbc:postgresql"):
+        if PostgreSqlQuery is None:
+            raise Exception(
+                "PostgreSQL backend requested but psycopg2 is not installed. "
+                "Install it with: pip install pyLodStorage[postgresql]"
+            )
+        backend = PostgreSqlQuery(endpoint=endpoint, debug=debug)
+        return backend
+
     backend = SQLDB(dbname=url, debug=debug)
     return backend

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/lodstorage/version.py

@@ -15,5 +15,5 @@ class Version(object):
     name = "pylodstorage"
     version = lodstorage.__version__
     date = "2020-09-10"
-    updated = "2026-02-28"
+    updated = "2026-05-17"
     description = "python List of Dict (Table) Storage library"

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/pyproject.toml

@@ -72,6 +72,10 @@ duckdb = [
   # https://pypi.org/project/duckdb/
   "duckdb>=1.1.0",
 ]
+postgresql = [
+  # https://pypi.org/project/psycopg2-binary/
+  "psycopg2-binary>=2.9.9",
+]
 test = [
   "green",
 ]

pylodstorage-0.19.4/tests/test_postgresql.py

@@ -0,0 +1,120 @@
+"""
+test_postgresql.py
+
+Tests for lodstorage.postgresql — PostgreSqlQuery with mocked psycopg2.
+
+Created on 2026-03-22
+
+@author: wf
+"""
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from lodstorage.query import Endpoint
+from lodstorage.sql_backend import PostgreSqlQuery, SQLBackend, get_sql_backend
+from tests.basetest import Basetest
+
+postgresql_available = PostgreSqlQuery is not None
+
+
+class TestPostgreSqlQuery(Basetest):
+    """
+    Verify PostgreSqlQuery with mocked psycopg2.
+    """
+
+    def setUp(self, debug=False, profile=True):
+        Basetest.setUp(self, debug=debug, profile=profile)
+
+    def postgresql_endpoint(self) -> Endpoint:
+        """Return an Endpoint configured for PostgreSQL."""
+        ep = Endpoint()
+        ep.endpoint = "jdbc:postgresql"
+        ep.host = "127.0.0.1"
+        ep.port = 5432
+        ep.user = "postgres"
+        ep.password = "testpass"
+        ep.database = "template1"
+        return ep
+
+    @unittest.skipUnless(postgresql_available, "psycopg2 not installed")
+    def testProtocolConformance(self):
+        """PostgreSqlQuery must satisfy SQLBackend at runtime."""
+        with patch("lodstorage.postgresql.psycopg2"):
+            pq = PostgreSqlQuery(endpoint=self.postgresql_endpoint())
+            self.assertIsInstance(pq, SQLBackend)
+
+    @unittest.skipUnless(postgresql_available, "psycopg2 not installed")
+    def testFactoryRouting(self):
+        """Factory must route jdbc:postgresql Endpoint -> PostgreSqlQuery."""
+        backend = get_sql_backend(self.postgresql_endpoint())
+        self.assertIsInstance(backend, PostgreSqlQuery)
+
+    @unittest.skipUnless(postgresql_available, "psycopg2 not installed")
+    @patch("lodstorage.postgresql.psycopg2")
+    def testQuery(self, mock_psycopg2):
+        """query() returns a list of dicts from mocked cursor."""
+        mock_cursor = MagicMock()
+        mock_cursor.fetchall.return_value = [
+            {"datname": "template1"},
+            {"datname": "n8n"},
+        ]
+        mock_connection = MagicMock()
+        mock_psycopg2.connect.return_value = mock_connection
+        mock_connection.cursor.return_value = mock_cursor
+
+        pq = PostgreSqlQuery(endpoint=self.postgresql_endpoint())
+        rows = pq.query("SELECT datname FROM pg_database")
+
+        self.assertEqual(2, len(rows))
+        self.assertEqual("template1", rows[0]["datname"])
+        self.assertEqual("n8n", rows[1]["datname"])
+        mock_cursor.execute.assert_called_once_with("SELECT datname FROM pg_database")
+        mock_connection.close.assert_called_once()
+
+    @unittest.skipUnless(postgresql_available, "psycopg2 not installed")
+    @patch("lodstorage.postgresql.psycopg2")
+    def testQueryGen(self, mock_psycopg2):
+        """query_gen() yields rows one by one from mocked cursor."""
+        mock_cursor = MagicMock()
+        mock_cursor.fetchone.side_effect = [
+            {"datname": "template1"},
+            {"datname": "n8n"},
+            None,
+        ]
+        mock_connection = MagicMock()
+        mock_psycopg2.connect.return_value = mock_connection
+        mock_connection.cursor.return_value = mock_cursor
+
+        pq = PostgreSqlQuery(endpoint=self.postgresql_endpoint())
+        names = [r["datname"] for r in pq.query_gen("SELECT datname FROM pg_database")]
+
+        self.assertEqual(["template1", "n8n"], names)
+        mock_cursor.close.assert_called_once()
+        mock_connection.close.assert_called_once()
+
+    @unittest.skipUnless(postgresql_available, "psycopg2 not installed")
+    @patch("lodstorage.postgresql.psycopg2")
+    def testDecodeBytes(self, mock_psycopg2):
+        """Binary values should be decoded to UTF-8 strings."""
+        mock_cursor = MagicMock()
+        mock_cursor.fetchall.return_value = [
+            {"name": b"caf\xc3\xa9"},
+        ]
+        mock_connection = MagicMock()
+        mock_psycopg2.connect.return_value = mock_connection
+        mock_connection.cursor.return_value = mock_cursor
+
+        pq = PostgreSqlQuery(endpoint=self.postgresql_endpoint())
+        rows = pq.query("SELECT name FROM test")
+
+        self.assertEqual(1, len(rows))
+        self.assertEqual("café", rows[0]["name"])
+
+    def testFactoryRaisesWithoutPsycopg2(self):
+        """Factory must raise if psycopg2 is not installed."""
+        ep = self.postgresql_endpoint()
+        with patch("lodstorage.sql_backend.PostgreSqlQuery", None):
+            with self.assertRaises(Exception) as ctx:
+                get_sql_backend(ep)
+            self.assertIn("psycopg2", str(ctx.exception))

{pylodstorage-0.19.2 → pylodstorage-0.19.4}/tests/test_queries.py

@@ -24,6 +24,7 @@ from lodstorage.query import (
     ValueFormatter,
     ValueFormatters,
 )
+from lodstorage.query_cmd import QueryCmd
 from lodstorage.querymain import QueryMain
 from lodstorage.querymain import main as queryMain
 from lodstorage.sparql import SPARQL
@@ -705,6 +706,36 @@ determines the number of instances available in the OpenStreetMap for the placeT
             except Exception as ex:
                 print(f"{query.title} at {endpointUrl} failed: {ex}")
 
+    def test_issue162_raw_format(self):
+        """
+        test raw format output
+
+        https://github.com/WolfgangFahl/pyLoDStorage/issues/162
+        """
+        debug = self.debug
+        qlod = [
+            {"grant": "GRANT USAGE ON *.* TO `test`@`%`"},
+            {"grant": "GRANT ALL PRIVILEGES ON `testdb`.* TO `test`@`%`"},
+        ]
+        args = Namespace(
+            debug=debug,
+            format=Format.raw,
+        )
+        query_cmd = QueryCmd.__new__(QueryCmd)
+        query_cmd.args = args
+        query_cmd.query = Query(name="test", query="SHOW GRANTS", lang="sql")
+        stdout = io.StringIO()
+        with redirect_stdout(stdout):
+            query_cmd.format_output(qlod)
+        result = stdout.getvalue()
+        if debug:
+            print(result)
+        self.assertIn("GRANT USAGE ON *.* TO `test`@`%`", result)
+        self.assertIn("GRANT ALL PRIVILEGES ON `testdb`.* TO `test`@`%`", result)
+        # raw format should not contain any table markup or quoting
+        self.assertNotIn("{|", result)
+        self.assertNotIn('"GRANT', result)
+
 
 class TestEndpoints(EndpointTest):
     """

pylodstorage-0.19.2/lodstorage/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.19.2"