workercommon 0.4.1__tar.gz

workercommon-0.4.1/LICENSE.TXT

@@ -0,0 +1,9 @@
+Copyright 2020 Neil E. Hodges
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

workercommon-0.4.1/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: workercommon
+Version: 0.4.1
+Summary: Support code for various projects
+Author-email: "Neil E. Hodges" <47hasbegun@gmail.com>
+License-Expression: BSD-3-Clause
+Project-URL: Repository, https://gitgud.io/takeshitakenji/workercommon
+Project-URL: Issues, https://gitgud.io/takeshitakenji/workercommon/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 3 - Alpha
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.TXT
+Requires-Dist: psycopg2
+Requires-Dist: pika
+Dynamic: license-file
+
+# Requirements
+
+1. Python 3.8
+2. A Unix-like OS for `fcntl` support
+3. [psycopg2](https://www.psycopg.org/)
+4. [Pika](https://pika.readthedocs.io/en/stable/)

workercommon-0.4.1/README.md

@@ -0,0 +1,6 @@
+# Requirements
+
+1. Python 3.8
+2. A Unix-like OS for `fcntl` support
+3. [psycopg2](https://www.psycopg.org/)
+4. [Pika](https://pika.readthedocs.io/en/stable/)

workercommon-0.4.1/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+
+[project]
+name = "workercommon"
+authors = [
+  {name = "Neil E. Hodges", email = "47hasbegun@gmail.com"}
+]
+description = "Support code for various projects"
+readme = "README.md"
+requires-python = ">= 3.8"
+license = "BSD-3-Clause"
+dependencies = [
+  "psycopg2",
+  "pika",
+]
+version = "0.4.1"
+license-files = ["LICENSE.TXT"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Development Status :: 3 - Alpha",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Repository = "https://gitgud.io/takeshitakenji/workercommon"
+Issues = "https://gitgud.io/takeshitakenji/workercommon/issues"
+
+[tool.setuptools]
+zip-safe = false
+packages = [
+  "workercommon",
+  "workercommon.database",
+]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"workercommon" = ["py.typed"]
+"workercommon.database" = ["py.typed"]

workercommon-0.4.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

workercommon-0.4.1/workercommon/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+from . import config, locking, rabbitmqueue, worker, commands, connectionpool, background

workercommon-0.4.1/workercommon/background.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+import logging, os, asyncio
+from typing import Optional, Callable, Union, Dict
+from threading import Thread, Lock
+from time import sleep
+from uuid import uuid1, UUID
+from asyncio.events import AbstractEventLoop
+
+
+class BackgroundThread(Thread):
+    def __init__(self, niceness: int = 0):
+        super().__init__()
+        self.lock = Lock()
+        self.loop: Optional[AbstractEventLoop] = None
+        self.handles: Dict[UUID, asyncio.Handle] = {}
+        self.niceness = niceness
+
+    def submit(self, func: Callable[[], None], delay: Union[int, float]) -> None:
+        with self.lock:
+            if not self.loop:
+                raise RuntimeError("Loop has already been stopped")
+
+            while not self.loop.is_running() and not self.loop.is_closed():
+                sleep(0.25)
+            if self.loop.is_closed() or not self.loop.is_running():
+                raise RuntimeError("Loop has already been stopped")
+
+            self.loop.call_soon_threadsafe(self.submit_inner, func, delay)
+
+    def submit_inner(self, func: Callable[[], None], delay: Union[int, float]) -> UUID:
+        with self.lock:
+            if not self.loop:
+                raise RuntimeError("Loop has already been stopped")
+
+            handle_id = uuid1()
+
+            def task():
+                try:
+                    func()
+                finally:
+                    with self.lock:
+                        try:
+                            del self.handles[handle_id]
+                        except KeyError:
+                            pass
+
+            logging.debug(f"Submitting task {handle_id} to be run later ({delay})")
+            self.handles[handle_id] = self.loop.call_later(delay, task)
+            return handle_id
+
+    def stop(self) -> None:
+        with self.lock:
+            for task in self.handles.values():
+                task.cancel()
+            if self.loop is not None:
+                self.loop.stop()
+
+    def run(self) -> None:
+        os.nice(self.niceness)
+        with self.lock:
+            loop = self.loop = asyncio.new_event_loop()
+
+        try:
+            loop.run_forever()
+        finally:
+            with self.lock:
+                self.loop = None

workercommon-0.4.1/workercommon/commands.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 5):
+    raise RuntimeError("At least Python 3.5 is required")
+
+from functools import partial
+from typing import Generator, Tuple, Any
+
+
+class CommandElement(object):
+    def __init__(self, name: str, exclusive_lock: bool, function: Any):
+        self.name, self.function, self.exclusive_lock = name, function, exclusive_lock
+
+    def __call__(self, *args, **kwargs):
+        self.function(*args, **kwargs)
+
+
+def Command(name: str, exclusive_lock: bool = True):
+    return partial(CommandElement, name, exclusive_lock)
+
+
+class Commands(object):
+    @classmethod
+    def get_commands(cls) -> Generator[Tuple[str, CommandElement], None, None]:
+        for key in dir(cls):
+            value = getattr(cls, key)
+            if isinstance(value, CommandElement):
+                yield value.name, value
+
+    def load_command(self, name: str):
+        for cmd_name, value in self.get_commands():
+            if name == cmd_name:
+                return partial(value, self)
+        else:
+            raise ValueError("Unknown command: %s" % name)
+
+    def run_command(self, name: str, *args):
+        self.load_command(name)(*args)
+
+    def close(self):
+        pass

workercommon-0.4.1/workercommon/config.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+from configparser import ConfigParser
+import logging
+from logging.handlers import WatchedFileHandler
+from typing import Tuple, Optional, Callable
+from pathlib import Path
+
+
+class Configuration(object):
+    def __init__(self, filename: str):
+        self.parser = self.create_parser()
+        self.set_base_config(self.parser)
+        self.parser.read(filename)
+        self.check_all_values(self.parser)
+
+    def create_parser(self) -> ConfigParser:
+        return ConfigParser()
+
+    def set_base_config(self, parser: ConfigParser) -> None:
+        pass
+
+    def check_all_values(self, parser: ConfigParser) -> None:
+        pass
+
+    @staticmethod
+    def dir_exists(s: str) -> bool:
+        return bool(s and Path(s).is_dir())
+
+    def check_value(
+        self, section: str, key: str, validator: Callable[[str], bool]
+    ) -> None:
+        value: Optional[str] = self[section, key]
+        try:
+            if value is None or not validator(value):
+                raise ValueError
+        except:
+            raise ValueError(f"Invalid {section}/{key} value: {value}")
+
+    def __getitem__(self, parts: Tuple[str, str]) -> Optional[str]:
+        section, key = parts
+
+        try:
+            value = self.parser[section][key]
+            return value if value else None
+        except KeyError:
+            return None
+
+    def get_strict(self, section: str, key: str) -> str:
+        value = self[section, key]
+        if value is None:
+            raise ValueError(f"Missing {section}/{key} value")
+
+        return value
+
+    def get(self, section: str, key: str, default: str) -> str:
+        value = self[section, key]
+        return value if value is not None else default
+
+    def configure_logging(self, *handlers) -> None:
+        level = self["Logging", "level"]
+        if not level:
+            raise RuntimeError("Logging level should have already been validated")
+
+        log_format = self["Logging", "format"]
+        if not log_format:
+            log_format = "[%(asctime)s] [%(levelname)s] [%(module)s] [%(filename)s:%(lineno)d %(funcName)s] %(message)s"
+
+        if handlers:
+            logging.basicConfig(level=level, format=log_format, handlers=handlers)
+        else:
+            fname = self["Logging", "file"]
+            if fname:
+                handler = WatchedFileHandler(fname)
+                logging.basicConfig(level=level, format=log_format, handlers=[handler])
+            else:
+                logging.basicConfig(level=level, format=log_format, stream=sys.stderr)

workercommon-0.4.1/workercommon/connectionpool.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import sys, os
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+import logging
+from typing import Callable, Any, Dict, Optional
+from functools import lru_cache
+from threading import Lock
+
+
+class Closeable(object):
+    def close(self) -> None:
+        raise NotImplementedError
+
+
+class BaseConnectionPool(object):
+    def acquire(self) -> Closeable:
+        raise NotImplementedError
+
+    def release(self, connection: Closeable) -> None:
+        raise NotImplementedError
+
+    def error(self, connection: Closeable) -> None:
+        raise NotImplementedError
+
+
+class ConnectionHandle(object):
+    def __init__(self, pool: BaseConnectionPool):
+        self.pool = pool
+        self.connection: Optional[Closeable] = None
+
+    def __enter__(self):
+        if self.connection is not None:
+            raise RuntimeError("Already acquired a connection")
+
+        self.connection = self.pool.acquire()
+        return self.connection
+
+    def __exit__(self, type, value, traceback):
+        if self.connection is None:
+            raise RuntimeError("No connection has been acquired")
+        try:
+            if value is not None:
+                self.pool.error(self.connection)
+            else:
+                self.pool.release(self.connection)
+        finally:
+            self.connection = None
+
+
+class ConnectionPool(BaseConnectionPool):
+    def __init__(self, connection_src: Callable[[], Closeable]):
+        self.connection_src = connection_src
+        self.lock = Lock()
+        self.connections: Dict[Closeable, bool] = {}
+
+    def acquire(self) -> Closeable:
+        with self.lock:
+            for connection, in_use in self.connections.items():
+                if not in_use:
+                    break
+            else:
+                connection = self.connection_src()
+
+            self.connections[connection] = True
+            return connection
+
+    def release(self, connection: Closeable) -> None:
+        with self.lock:
+            if connection not in self.connections:
+                raise RuntimeError(f"Unknown connection: {connection}")
+            self.connections[connection] = False
+
+    def error(self, connection: Closeable) -> None:
+        with self.lock:
+            if connection not in self.connections:
+                raise RuntimeError(f"Unknown connection: {connection}")
+            try:
+                connection.close()
+            except:
+                logging.exception(f"Failed to close {connection}")
+            del self.connections[connection]
+
+    def close(self) -> None:
+        with self.lock:
+            for connection in self.connections.keys():
+                try:
+                    connection.close()
+                except:
+                    logging.exception(f"Failed to close {connection}")
+
+            self.connections.clear()
+
+    def __call__(self) -> ConnectionHandle:
+        return ConnectionHandle(self)

workercommon-0.4.1/workercommon/database/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+from .base import *
+
+try:
+    from .pg import (
+        Cursor as PgCursor,
+        Connection as PgConnection,
+        BaseTable as PgBaseTable,
+    )
+except ImportError:
+    pass

workercommon-0.4.1/workercommon/database/base.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 8):
+    raise RuntimeError("At least Python 3.8 is required")
+
+from typing import Any, Iterator, Dict, List, Sequence, Optional, Iterable, Union, Tuple
+import weakref
+
+
+class BaseConnection(object):
+    def get(self) -> Any:
+        raise NotImplementedError
+
+    def commit(self) -> None:
+        raise NotImplementedError
+
+    def rollback(self) -> None:
+        raise NotImplementedError
+
+    def close(self) -> None:
+        raise NotImplementedError
+
+
+class Cursor(object):
+    def get(self) -> Any:
+        raise NotImplementedError
+
+    def execute_direct(self, query, *args) -> Any:
+        raise NotImplementedError
+
+    def execute(self, query, *args) -> Iterator[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        raise NotImplementedError
+
+    @property
+    def rowcount(self) -> int:
+        raise NotImplementedError
+
+    @property
+    def columns(self) -> List[str]:
+        raise NotImplementedError
+
+    def __enter__(self) -> Any:
+        raise NotImplementedError
+
+    def __exit__(self, type, value, traceback) -> None:
+        raise NotImplementedError
+
+
+class Connection(BaseConnection):
+    def cursor(self) -> Cursor:
+        raise NotImplementedError
+
+
+class BaseTable(object):
+    ORDERINGS = frozenset(["ASC", "DESC"])
+
+    def __init__(
+        self,
+        name: str,
+        primary_key: str,
+        columns: Sequence[str],
+        ordering: Optional[str] = "ASC",
+    ):
+        if not name:
+            raise ValueError("Invalid name: {name}")
+
+        if not columns:
+            raise ValueError(f"Invalid column list: {columns}")
+
+        if not primary_key:
+            raise ValueError(f"Invalid column list: {primary_key}")
+
+        self.columns = frozenset(columns)
+        if primary_key not in columns:
+            raise ValueError(f"{primary_key} is not in columns: {self.columns}")
+
+        self.name, self.primary_key = name, primary_key
+
+        self.ordering = ordering if ordering in self.ORDERINGS else "ASC"
+
+    def validate(self, columns: Iterable[str]) -> None:
+        invalid_columns = {col for col in columns if col not in self.columns}
+        if invalid_columns:
+            raise ValueError(f"Invalid columns: {invalid_columns}")
+
+    @staticmethod
+    def unpack(
+        items: Union[Dict[str, Any], List[Tuple[str, Any]]]
+    ) -> Tuple[Tuple[str], Tuple[Any]]:
+        if isinstance(items, dict):
+            columns, values = zip(*items.items())
+        else:
+            columns, values = zip(*items)
+        return columns, values
+
+    def insert(self, base_cursor: Cursor, **input_values: Any) -> Any:
+        raise NotImplementedError
+
+    def upsert_on_column(
+        self,
+        base_cursor: Cursor,
+        column_name: str,
+        column_value: Any,
+        **input_values: Any,
+    ) -> Any:
+        raise NotImplementedError
+
+    def upsert(self, base_cursor: Cursor, column_value: Any, **input_values: Any) -> Any:
+        return self.upsert_on_column(
+            base_cursor, self.primary_key, column_value, **input_values
+        )
+
+    def select_by_columns(
+        self, base_cursor: Cursor, limit: Optional[int] = None, **input_values: Any
+    ) -> Iterator[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def select_custom_where(
+        self, base_cursor: Cursor, where: str, *values
+    ) -> Iterator[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def select_by_id(self, base_cursor: Cursor, pkey_value: Any) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def delete(self, base_cursor: Cursor, pkey_value: Any) -> bool:
+        raise NotImplementedError
+
+
+class TableCursor(object):
+    def __init__(self, cursor: Cursor, table: BaseTable):
+        self.cursor, self.table = weakref.ref(cursor), table
+
+    def get_cursor(self) -> Cursor:
+        cursor = self.cursor()
+        if cursor is None:
+            raise RuntimeError("Cursor has gone out of scope")
+        return cursor
+
+    def insert(self, **input_values: Any) -> Any:
+        return self.table.insert(self.get_cursor(), **input_values)
+
+    def upsert_on_column(
+        self, column_name: str, column_value: Any, **input_values: Any
+    ) -> Any:
+        return self.table.upsert_on_column(
+            self.get_cursor(), column_name, column_value, **input_values
+        )
+
+    def upsert(self, pkey_value: Any, **input_values: Any) -> Any:
+        return self.table.upsert(self.get_cursor(), pkey_value, **input_values)
+
+    def __setitem__(self, key: Any, value: Dict[str, Any]) -> Any:
+        self.upsert(key, **value)
+
+    def where(
+        self, limit: Optional[int], **input_values: Any
+    ) -> Iterator[Dict[str, Any]]:
+        return self.table.select_by_columns(self.get_cursor(), limit, **input_values)
+
+    def custom_where(self, where: str, *values) -> Iterator[Dict[str, Any]]:
+        return self.table.select_custom_where(self.get_cursor(), where, *values)
+
+    def __getitem__(self, key: Any) -> Dict[str, Any]:
+        return self.table.select_by_id(self.get_cursor(), key)
+
+    def __delitem__(self, key: Any) -> None:
+        if not self.table.delete(self.get_cursor(), key):
+            raise KeyError(key)