ry-pg-utils 1.0.2__py3-none-any.whl

ry_pg_utils/updater.py

@@ -0,0 +1,181 @@
+import argparse
+import time
+import typing as T
+from dataclasses import dataclass
+
+from ry_redis_bus.helpers import RedisInfo, message_handler
+from ry_redis_bus.redis_client_base import RedisClientBase
+from ryutils import log
+from ryutils.verbose import Verbose
+
+from ry_pg_utils.connect import close_engine, init_database, is_database_initialized
+from ry_pg_utils.ipc import channels
+from ry_pg_utils.pb_types.database_pb2 import (  # pylint: disable=no-name-in-module
+    DatabaseSettingsPb,
+    PostgresMessagePb,
+    PostgresPb,
+)
+from ry_pg_utils.postgres_info import PostgresInfo
+
+DB_RETRY_TIME = 5.0
+
+
+def get_database_settings(
+    message_pb: DatabaseSettingsPb, backend_id: str, verbose: bool = False
+) -> T.Optional[PostgresInfo]:
+    raw_db_config_pb = message_pb.postgres.get(backend_id, None)
+
+    if raw_db_config_pb is None:
+        return None
+
+    if not message_pb.primaryDatabase:
+        return None
+
+    db_config_pb: PostgresPb = raw_db_config_pb
+
+    if verbose:
+        log.print_normal(f"Received database settings {backend_id}:\n{db_config_pb}")
+
+    database_settings_msg = PostgresInfo(
+        db_name=db_config_pb.database,
+        user=db_config_pb.user,
+        password=db_config_pb.password,
+        host=db_config_pb.host,
+        port=db_config_pb.port,
+    )
+
+    return database_settings_msg
+
+
+@dataclass
+class DbUpdater(RedisClientBase):
+    do_publish_db: bool
+    postgres_info: PostgresInfo
+    backend_id: str
+    logging_error_db_callback: T.Callable[[str, str], None] | None
+
+    def __init__(
+        self,
+        redis_info: RedisInfo,
+        args: argparse.Namespace,
+        backend_id: str,
+        verbose: Verbose,
+        logging_error_db_callback: T.Callable[[str, str], None] | None = None,
+    ):
+        super().__init__(
+            redis_info=redis_info,
+            verbose=verbose,
+        )
+
+        self.do_publish_db = args.do_publish_db if args else True
+
+        self.postgres_info = PostgresInfo(
+            db_name=args.postgres_db,
+            user=args.postgres_user,
+            password=args.postgres_password,
+            host=args.postgres_host,
+            port=args.postgres_port,
+        )
+
+        self.last_db_init_retry_time = 0.0
+        self.use_local_db_only = args.use_local_db_only if args else True
+        self.database_settings_msg: T.Optional[PostgresInfo] = None
+        self.backend_id = backend_id
+        self.logging_error_db_callback = logging_error_db_callback
+
+    @message_handler
+    def handle_database_config_message(self, message_pb: DatabaseSettingsPb) -> None:
+        database_settings = get_database_settings(message_pb, backend_id=self.backend_id)
+
+        if database_settings is not None:
+            self.database_settings_msg = database_settings
+
+    def init(self) -> None:
+        super().start()
+
+        if self.use_local_db_only:
+            init_database(
+                db_host=self.postgres_info.host,
+                db_port=self.postgres_info.port,
+                db_name=self.postgres_info.db_name,
+                db_user=self.postgres_info.user,
+                db_password=self.postgres_info.password,
+            )
+        else:
+            self.postgres_info = PostgresInfo.null()
+            log.print_fail(f"DbUpdater initialized with null database info: {self.postgres_info}")
+
+            self.subscribe(channels.DATABASE_CONFIG_CHANNEL, self.handle_database_config_message)
+
+    def update_db(self, postgres_info: PostgresInfo) -> None:
+        self.postgres_info = postgres_info
+
+    def get_info(self) -> T.Optional[PostgresInfo]:
+        return self.postgres_info
+
+    def step(self, force: bool = False) -> None:
+        now = time.time()
+
+        super().step()
+
+        if not force and now - self.last_db_init_retry_time < DB_RETRY_TIME:
+            return
+
+        self.last_db_init_retry_time = now
+
+        self.maybe_update_database()
+
+    def maybe_update_database(self) -> None:
+        new_postgres_info: T.Optional[PostgresInfo] = (
+            self.postgres_info if self.database_settings_msg is None else self.database_settings_msg
+        )
+
+        self.database_settings_msg = None
+
+        if new_postgres_info is None:
+            return
+
+        if new_postgres_info.is_null():
+            log.print_fail(f"Database info is null: {new_postgres_info}")
+            return
+
+        if new_postgres_info == self.postgres_info and is_database_initialized(
+            new_postgres_info.db_name
+        ):
+            return
+
+        log.print_bold(f"Updating database to {new_postgres_info}")
+
+        close_engine(self.postgres_info.db_name)
+        init_database(
+            db_host=new_postgres_info.host,
+            db_port=new_postgres_info.port,
+            db_name=new_postgres_info.db_name,
+            db_user=new_postgres_info.user,
+            db_password=new_postgres_info.password,
+        )
+
+        self.postgres_info = new_postgres_info
+
+        db_message_pb = PostgresMessagePb()
+        postgres_db_pb = PostgresPb()
+        postgres_db_pb.database = self.postgres_info.db_name
+        postgres_db_pb.user = self.postgres_info.user
+        postgres_db_pb.password = self.postgres_info.password
+        postgres_db_pb.host = self.postgres_info.host
+        postgres_db_pb.port = self.postgres_info.port
+        db_message_pb.postgres.CopyFrom(postgres_db_pb)
+        db_message_pb.utime.GetCurrentTime()
+        if self.do_publish_db:
+            self.publish(channels.DATABASE_CHANNEL, db_message_pb.SerializeToString())
+
+        def log_print_callback(message: str) -> None:
+            if self.logging_error_db_callback:
+                self.logging_error_db_callback(message, self.postgres_info.db_name)
+
+        log.update_callback(log_print_callback)
+
+    def run(self) -> None:
+        while True:
+            self.step()
+            time.sleep(1)

ry_pg_utils-1.0.2.dist-info/METADATA

@@ -0,0 +1,473 @@
+Metadata-Version: 2.4
+Name: ry-pg-utils
+Version: 1.0.2
+Summary: Utility functions for PostgreSQL
+Author: Ross Yeager
+Author-email: ryeager12@email.com
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ry_redis_bus
+Requires-Dist: ryutils
+Requires-Dist: sqlalchemy
+Requires-Dist: sqlalchemy_utils
+Requires-Dist: psycopg2-binary
+Requires-Dist: python-dotenv
+Requires-Dist: tenacity
+Requires-Dist: types-protobuf
+Requires-Dist: types-psycopg2
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
+# ry-pg-utils
+
+A Python utility library for PostgreSQL database operations with dynamic table creation, connection management, and Protocol Buffer integration.
+
+## Overview
+
+`ry-pg-utils` provides a robust framework for working with PostgreSQL databases in Python applications. It includes utilities for:
+
+- Database connection management with connection pooling
+- Dynamic table creation from Protocol Buffer message definitions
+- Thread-safe session management
+- Multi-backend support with automatic backend ID tracking
+- Database updater for dynamic configuration
+- Argument parsing for PostgreSQL connection parameters
+
+## Features
+
+- **Connection Management**: Thread-safe PostgreSQL connection pooling with automatic retry logic
+- **Dynamic Tables**: Automatically create and manage database tables from Protocol Buffer message schemas
+- **Multi-Backend Support**: Track data across multiple backend instances with automatic ID tagging
+- **Session Management**: Context managers for safe database session handling
+- **Configuration System**: Flexible configuration via environment variables and runtime settings
+- **Type Safety**: Full type hints and mypy support
+
+## Installation
+
+```bash
+pip install ry-pg-utils
+```
+
+### Dependencies
+
+- Python 3.12+
+- PostgreSQL database
+- SQLAlchemy
+- Protocol Buffer support
+
+## Configuration
+
+`ry-pg-utils` uses a flexible configuration system that can be customized in multiple ways:
+
+### 1. Environment Variables
+
+Create a `.env` file in your project root:
+
+```bash
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_DB=mydb
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=secret
+```
+
+### 2. Configuration Object
+
+```python
+from ry_pg_utils.config import pg_config
+
+# Access configuration
+print(pg_config.postgres_host)
+print(pg_config.postgres_port)
+
+# Modify at runtime
+pg_config.add_backend_to_all = False
+pg_config.backend_id = "custom_backend_id"
+```
+
+### Configuration Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `postgres_host` | str | From env | PostgreSQL server hostname |
+| `postgres_port` | int | From env | PostgreSQL server port |
+| `postgres_db` | str | From env | Database name |
+| `postgres_user` | str | From env | Database username |
+| `postgres_password` | str | From env | Database password |
+| `backend_id` | str | hostname_ip | Unique identifier for this backend instance |
+| `add_backend_to_all` | bool | True | Add backend_id column to all tables |
+| `add_backend_to_tables` | bool | True | Append backend_id to table names |
+| `raise_on_use_before_init` | bool | True | Raise exception if DB used before initialization |
+| `do_publish_db` | bool | False | Enable database publishing features |
+| `use_local_db_only` | bool | True | Use only local database connections |
+
+## Quick Start
+
+### 1. Initialize Database Connection
+
+```python
+from ry_pg_utils.connect import init_database, ManagedSession
+from ry_pg_utils.postgres_info import PostgresInfo
+
+# Create database connection info
+db_info = PostgresInfo(
+    db_name="myapp_db",
+    host="localhost",
+    port=5432,
+    user="postgres",
+    password="secret"
+)
+
+# Initialize the database connection
+init_database(db_info, db_name="myapp")
+```
+
+### 2. Use Dynamic Tables with Protocol Buffers
+
+```python
+from ry_pg_utils.dynamic_table import DynamicTableDb
+from your_app.proto import YourMessagePb
+
+# Create a message
+message = YourMessagePb()
+message.field1 = "value1"
+message.field2 = 42
+
+# Log message to database (table created automatically)
+DynamicTableDb.log_data_to_db(
+    msg=message,
+    db_name="myapp",
+    channel="my_channel"
+)
+
+# Check if data exists
+exists = DynamicTableDb.is_in_db(
+    msg=message,
+    db_name="myapp",
+    channel="my_channel",
+    attr="field1",
+    value="value1"
+)
+```
+
+### 3. Manual Session Management
+
+```python
+from ry_pg_utils.connect import ManagedSession
+
+# Use context manager for automatic session cleanup
+with ManagedSession(db="myapp") as session:
+    result = session.execute("SELECT * FROM my_table")
+    for row in result:
+        print(row)
+```
+
+## Core Components
+
+### `connect.py` - Connection Management
+
+The connection module provides thread-safe database connection and session management:
+
+```python
+from ry_pg_utils.connect import (
+    init_database,      # Initialize database connection
+    init_engine,        # Initialize SQLAlchemy engine
+    ManagedSession,     # Context manager for sessions
+    get_backend_id,     # Get current backend ID
+    set_backend_id,     # Set backend ID for thread
+    close_engine,       # Close database connection
+    clear_db,           # Clear all connections
+)
+```
+
+**Key Features:**
+- Thread-local backend ID tracking
+- Connection pooling with configurable parameters
+- Automatic connection recovery on failure
+- Session scoping for thread safety
+
+### `dynamic_table.py` - Dynamic Table Creation
+
+Automatically create and manage database tables from Protocol Buffer definitions:
+
+```python
+from ry_pg_utils.dynamic_table import DynamicTableDb
+
+# Create instance
+db = DynamicTableDb(db_name="myapp")
+
+# Add message to database
+db.add_message(
+    channel_name="events",
+    message_pb=my_protobuf_message,
+    log_print_failure=True,
+    verbose=True
+)
+
+# Check existence
+exists = db.inst_is_in_db(
+    message_pb=my_protobuf_message,
+    channel_name="events",
+    attr="event_id",
+    value=12345
+)
+```
+
+**Supported Protocol Buffer Types:**
+- `int32`, `int64`, `uint32`, `uint64` → PostgreSQL `Integer`
+- `float`, `double` → PostgreSQL `Float`
+- `bool` → PostgreSQL `Boolean`
+- `string` → PostgreSQL `String`
+- `bytes` → PostgreSQL `LargeBinary`
+- `Timestamp` (message) → PostgreSQL `DateTime`
+
+### `postgres_info.py` - Connection Information
+
+Data class for PostgreSQL connection parameters:
+
+```python
+from ry_pg_utils.postgres_info import PostgresInfo
+
+db_info = PostgresInfo(
+    db_name="mydb",
+    host="localhost",
+    port=5432,
+    user="postgres",
+    password="secret"
+)
+
+# Get connection URI
+uri = db_info.get_uri()  # postgresql://postgres:secret@localhost:5432/mydb
+```
+
+### `parse_args.py` - Argument Parsing
+
+Add PostgreSQL arguments to your argument parser:
+
+```python
+import argparse
+from ry_pg_utils.parse_args import add_postrgres_db_args
+
+parser = argparse.ArgumentParser()
+add_postrgres_db_args(parser)
+
+args = parser.parse_args()
+# Access: args.postgres_host, args.postgres_port, etc.
+```
+
+### `updater.py` - Database Configuration Updater
+
+Dynamically update database connections based on configuration messages:
+
+```python
+from ry_pg_utils.updater import PostgresDbUpdater
+
+updater = PostgresDbUpdater(
+    redis_info=redis_info,
+    verbose=verbose,
+    backend_id="my_backend"
+)
+
+# Start listening for configuration updates
+updater.run()
+```
+
+## Advanced Usage
+
+### Multi-Backend Support
+
+When `add_backend_to_all` is enabled, all tables automatically get a `backend_id` column:
+
+```python
+from ry_pg_utils.connect import set_backend_id, ManagedSession
+
+# Set backend ID for current thread
+set_backend_id("backend_1")
+
+# All subsequent operations will include this backend_id
+with ManagedSession(db="myapp") as session:
+    # Queries automatically filter by backend_id
+    result = session.execute("SELECT * FROM my_table")
+```
+
+### Custom Table Names
+
+When `add_backend_to_tables` is enabled, table names are automatically suffixed:
+
+```python
+from ry_pg_utils.connect import get_table_name
+
+# Returns "events_my_backend" if add_backend_to_tables=True
+table_name = get_table_name("events", backend_id="my_backend")
+```
+
+### ORM Base Class
+
+Use the pre-configured base class for SQLAlchemy models:
+
+```python
+from ry_pg_utils.connect import Base
+from sqlalchemy import Column, Integer, String
+
+class User(Base):
+    __tablename__ = 'users'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(100))
+    email = Column(String(200))
+
+# If add_backend_to_all=True, backend_id column is automatically added
+```
+
+## Error Handling
+
+The library includes robust error handling:
+
+```python
+from ry_pg_utils.connect import ManagedSession
+
+with ManagedSession(db="myapp") as session:
+    if session is None:
+        # Connection failed, handle gracefully
+        print("Failed to establish database connection")
+        return
+
+    try:
+        session.execute("SELECT * FROM my_table")
+    except Exception as e:
+        # Session will automatically rollback
+        print(f"Query failed: {e}")
+```
+
+## Type Safety
+
+The library is fully typed and includes a `py.typed` marker for mypy support:
+
+```bash
+# Run type checking
+mypy your_app.py
+```
+
+## Development
+
+### Setup Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/yourusername/ry-pg-utils.git
+cd ry-pg-utils
+
+# Create virtual environment
+python -m venv venv-dev
+source venv-dev/bin/activate  # On Windows: venv-dev\Scripts\activate
+
+# Install dependencies
+pip install -r packages/requirements-dev.txt
+```
+
+### Running Tests
+
+```bash
+# Activate virtual environment
+source venv-dev/bin/activate
+
+# Run tests
+python -m pytest test/
+```
+
+### Code Quality
+
+The project uses several tools for code quality:
+
+```bash
+# Format code
+black ry_pg_utils/
+
+# Type checking
+mypy ry_pg_utils/
+
+# Linting
+pylint ry_pg_utils/
+
+# Import sorting
+isort ry_pg_utils/
+```
+
+## Examples
+
+### Complete Application Example
+
+```python
+import argparse
+from ry_pg_utils.parse_args import add_postrgres_db_args
+from ry_pg_utils.connect import init_database, ManagedSession
+from ry_pg_utils.postgres_info import PostgresInfo
+from ry_pg_utils.dynamic_table import DynamicTableDb
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="My Database App")
+    add_postrgres_db_args(parser)
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    # Initialize database
+    db_info = PostgresInfo(
+        db_name=args.postgres_db,
+        host=args.postgres_host,
+        port=args.postgres_port,
+        user=args.postgres_user,
+        password=args.postgres_password
+    )
+
+    init_database(db_info, db_name="myapp")
+
+    # Use the database
+    with ManagedSession(db="myapp") as session:
+        if session:
+            result = session.execute("SELECT version()")
+            print(f"PostgreSQL version: {result.fetchone()[0]}")
+
+if __name__ == "__main__":
+    main()
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+## Author
+
+Ross Yeager - ryeager12@email.com
+
+## Changelog
+
+### Version 1.0.0
+- Initial release
+- Database connection management
+- Dynamic table creation
+- Multi-backend support
+- Configuration system
+- Protocol Buffer integration

ry_pg_utils-1.0.2.dist-info/RECORD

@@ -0,0 +1,20 @@
+ry_pg_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ry_pg_utils/config.py,sha256=eR61lRJo4O7N_p4IUwcSgHVf9kdXbaDobScD9Gh-MSA,1169
+ry_pg_utils/connect.py,sha256=jF-Sj9oySPDfTlOh4_C6c03t4KW5Sef_MLHx_sh52a4,10108
+ry_pg_utils/dynamic_table.py,sha256=VBkfbATRk_pW3EtAFHvMkGG0fLZMebw_LdWkFaDDugw,6866
+ry_pg_utils/notify_trigger.py,sha256=_dsuPkoqjnSQqWrsdZK3FqI6zG5UuygVXHwtERwN99Y,12069
+ry_pg_utils/parse_args.py,sha256=K5nzSfRhIBTazCqSi_FCCjLVdI_5a7G8MtqMwuGCjO4,752
+ry_pg_utils/postgres_info.py,sha256=mj9er830jvrJXUFuFKx1EmZMjuEpMDcd901kgYb_Cck,1495
+ry_pg_utils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ry_pg_utils/updater.py,sha256=TbUKJHQarzRrDjcbjbzIhy1sO61o-YhEG594gWnI0LA,5831
+ry_pg_utils/ipc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ry_pg_utils/ipc/channels.py,sha256=Qn1SUcxkl_vnM-ZejZalVRVu4sGPuNIWIbxmo8Q0DVI,612
+ry_pg_utils/pb_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ry_pg_utils/pb_types/database_pb2.py,sha256=yf__HycGrVoU8pzhYdnOhXhfAkgjeGl80ls_RsLemy4,2780
+ry_pg_utils/pb_types/database_pb2.pyi,sha256=svpmlEY_99dA_u3CJqn7CO5D7CH2pAGTOqlhVMZLXC4,5689
+ry_pg_utils/pb_types/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ry_pg_utils-1.0.2.dist-info/licenses/LICENSE,sha256=PYnig94SABlde939TWrqvOqQkkfjuHttf8KpWKlPFlA,1068
+ry_pg_utils-1.0.2.dist-info/METADATA,sha256=VGFQxqR7N686IzcfhxIJNFy0WZnPolV2DyzrGQESu8Q,11858
+ry_pg_utils-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ry_pg_utils-1.0.2.dist-info/top_level.txt,sha256=OruzbmsQHYyPnAw8RichQ5GK1Sxj-MQwWfJ93PEHXIM,12
+ry_pg_utils-1.0.2.dist-info/RECORD,,

ry_pg_utils-1.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ry_pg_utils-1.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Ross Yeager
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ry_pg_utils-1.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+ry_pg_utils