underscorepy 0.9.0__py3-none-any.whl

_/caches/memory.py

@@ -0,0 +1,77 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import os
+import json
+
+import _
+
+
+class Memory(_.caches.Cache):
+    async def init(self, component_name, **kwds):
+        if not hasattr(_.application, 'is_session_expired'):
+            raise _.error('Application does not have is_session_expired function defined')
+
+        self.sessions = {}
+
+        members = dict(
+            component = component_name,
+            sessions  = self.sessions,
+            )
+        subclass = type(component_name, (MemorySessions,), _.prefix(members))
+        _.application._record_handler('sessions', subclass)
+
+    def cookie_secret(self):
+        return os.urandom(32)
+
+    def save_session(self, session):
+        try:
+            session_id = session['session_id']
+        except KeyError:
+            raise _.error('No session_id defined in session')
+        self.sessions[session_id] = json.dumps(session)
+
+    async def load_session(self, session_id):
+        session = self.sessions.get(session_id, None)
+        if not session:
+            return None
+        session = json.loads(session)
+        if await _.wait(_.application.is_session_expired(session, self._expires)):
+            return None
+        return session
+
+    def clear_session(self, session_id):
+        self.sessions.pop(session_id, None)
+
+
+class MemorySessions(_.handlers.Protected):
+    @_.auth.protected
+    async def get(self, session_id=None):
+        if session_id:
+            session = self._sessions[session_id]
+            if not session:
+                raise _.HTTPError(404)
+            session = json.loads(session)
+            self.write(session)
+        else:
+            data = []
+            for session_id in self._sessions:
+                session = self._sessions[session_id]
+                data.append(json.loads(session))
+            self.write({'data':data})
+
+    @_.auth.protected
+    async def delete(self, session_id=None):
+        self.set_status(204)
+        if session_id:
+            del self._sessions[session_id]
+            callback = getattr(_.application, f'on_{self._component}_delete', None)
+            if callback is None:
+                callback = getattr(_.application, 'on_memory_delete', None)
+            if callback:
+                await _.wait(callback(self._component, session_id))

_/caches/redis.py

@@ -0,0 +1,104 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import base64
+import json
+import os
+
+import _
+
+try:
+    import redis.asyncio as redis
+except ImportError:
+    raise _.error('Missing redis module with asyncio support')
+
+
+class Redis(_.caches.Cache):
+    async def init(self, component_name, **kwds):
+        if 'socket_connect_timeout' not in kwds:
+            kwds['socket_connect_timeout'] = 3
+
+        if 'socket_timeout' not in kwds:
+            kwds['socket_timeout'] = 3
+
+        self.redis = redis.Redis(**kwds)
+        try:
+            await self.redis.ping()
+        except Exception as e:
+            raise _.error('Redis: %s: %s', component_name, e)
+
+        members = dict(
+            component = component_name,
+            redis     = self.redis,
+            )
+        subclass = type(component_name, (RedisSessions,), _.prefix(members))
+        _.application._record_handler('sessions', subclass)
+
+    async def close(self):
+        await self.redis.aclose()
+        self.redis = None
+
+    async def cookie_secret(self):
+        secret = await self.redis.get('cookie_secret')
+        if not secret:
+            secret = base64.b64encode(os.urandom(32))
+            await self.redis.set('cookie_secret', secret)
+        return secret
+
+    async def save_session(self, session):
+        try:
+            session_id = session['session_id']
+        except KeyError:
+            raise _.error('No session_id defined in session')
+        async with self.redis.pipeline(transaction=True) as pipe:
+            await pipe.set(f'session/{session_id}', json.dumps(session))
+            await pipe.expire(f'session/{session_id}', self._expires * 3600)
+            await pipe.execute()
+
+    async def load_session(self, session_id):
+        session = await self.redis.get(f'session/{session_id}')
+        if not session:
+            return None
+        return json.loads(session)
+
+    async def clear_session(self, session_id):
+        session = await self.redis.delete(f'session/{session_id}')
+
+    # fall through for calling redis functions directly
+    def __getattr__(self, attr):
+        return getattr(self.redis, attr)
+
+
+class RedisSessions(_.handlers.Protected):
+    @_.auth.protected
+    async def get(self, session_id=None):
+        if session_id:
+            session = await self._redis.get(f'session/{session_id}')
+            if not session:
+                raise _.HTTPError(404)
+            session = json.loads(session)
+            self.write(session)
+        else:
+            data = []
+            session_ids = await self._redis.keys('session/*')
+            for session_id in session_ids:
+                session = await self._redis.get(session_id)
+                data.append(json.loads(session))
+            data.sort(key=lambda d: d['time'])
+            self.write({'data':data})
+
+    @_.auth.protected
+    async def delete(self, session_id=None):
+        self.set_status(204)
+        if session_id:
+            await self._redis.delete(f'session/{session_id}')
+            callback = getattr(_.application, f'on_{self._component}_delete', None)
+            if callback is None:
+                callback = getattr(_.application, 'on_redis_delete', None)
+            if callback:
+                await _.wait(callback(self._component, session_id))

_/components.py

@@ -0,0 +1,107 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import importlib
+import logging
+import sys
+
+import _
+
+
+class Component(type(_)):
+    def __init__(self, module):
+        super().__init__(module.__name__)
+        self.module = module
+        self._components = {}
+
+    def __getattr__(self, name):
+        if name in self._components:
+            return self._components[name]
+        return getattr(self.module, name)
+
+    def __getitem__(self, name):
+        return self._components[name]
+
+    def __setitem__(self, name, value):
+        self._components[name] = value
+
+    def keys(self):
+        return self._components.keys()
+
+    def values(self):
+        return self._components.values()
+
+    def items(self):
+        return self._components.items()
+
+    def __str__(self):
+        return str(self._components)
+
+    def __len__(self):
+        return len(self._components)
+
+
+async def load(component_type):
+    # import the component submodule from _
+    module = importlib.import_module(f'_.{component_type}')
+    # wrap the submodule in a Component class, even when none are configured,
+    # since a later "import _.records" would otherwise leave the bare module
+    setattr(_, component_type, Component(module))
+
+    # skip components not specified in the config
+    if component_type not in _.config:
+        return
+
+    # iterate over the components specified in the config
+    for component_name in _.config[component_type]:
+        # check if the component is aliased
+        component = _.config[component_type][component_name]
+        if component is None:
+            component = component_name
+
+        # allow import 3rd-party components, specified using + prefix
+        if not component.startswith('+'):
+            import_path = f'_.{component_type}.{component}'
+            attr = None
+        else:
+            try:
+                component,attr = component.rsplit('.', 1)
+            except ValueError:
+                attr = None
+            import_path = component[1:]
+
+        # attempt to import the module
+        try:
+            module = importlib.import_module(import_path)
+        except ModuleNotFoundError as e:
+            raise _.error('Unknown module: %s: %s', import_path, e)
+
+        cls = None
+        if not attr:
+            for attr_name in dir(module):
+                attr = getattr(module, attr_name)
+                # _ components are types
+                if not isinstance(attr, type):
+                    continue
+                # _ components have _ member function
+                if not hasattr(attr, '_'):
+                    continue
+                cls = attr
+                break
+        else:
+            cls = getattr(module, attr)
+
+        if not cls:
+            raise _.error('%s: %s module not found', component, component_type)
+
+        try:
+            kwds = dict(_.config[component_name])
+        except KeyError:
+            kwds = {}
+
+        await cls._(component_name, **kwds)

_/databases/__init__.py

@@ -0,0 +1,282 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import base64
+import io
+import logging
+import typing
+
+import _
+
+try:
+    import sqlalchemy
+    import sqlalchemy.types
+    import sqlalchemy.orm
+    from sqlalchemy.ext.asyncio import AsyncSession,create_async_engine
+except ImportError:
+    raise _.error('Missing sqlalchemy module')
+
+logging.getLogger('alembic').setLevel(logging.WARNING)
+
+
+class UInt64(sqlalchemy.types.TypeDecorator):
+    '''unsigned 64-bit integer: NUMERIC(20) on Postgres, otherwise stored in
+       a BIGINT as its two's complement so values of 2**63 and up fit'''
+    impl     = sqlalchemy.BIGINT
+    cache_ok = True
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == 'postgresql':
+            return dialect.type_descriptor(sqlalchemy.NUMERIC(20, 0))
+        return dialect.type_descriptor(sqlalchemy.BIGINT())
+
+    def process_bind_param(self, value, dialect):
+        if value is None or dialect.name == 'postgresql':
+            return value
+        return value - (1 << 64) if value >= (1 << 63) else value
+
+    def process_result_value(self, value, dialect):
+        if value is None:
+            return value
+        if dialect.name == 'postgresql':
+            return int(value)
+        return value + (1 << 64) if value < 0 else value
+
+
+class JSONArray(sqlalchemy.types.TypeDecorator):
+    'list stored as JSON for databases without arrays, bytes as base64'
+    impl     = sqlalchemy.JSON
+    cache_ok = True
+
+    def __init__(self, binary=False):
+        super().__init__()
+        self.binary = binary
+
+    def process_bind_param(self, value, dialect):
+        if value is not None and self.binary:
+            value = [base64.b64encode(v).decode('ascii') for v in value]
+        return value
+
+    def process_result_value(self, value, dialect):
+        if value is not None and self.binary:
+            value = [base64.b64decode(v) for v in value]
+        return value
+
+
+class Database:
+    TEXT    = sqlalchemy.TEXT
+    INTEGER = sqlalchemy.INTEGER
+    BIGINT  = sqlalchemy.BIGINT
+    #DOUBLE  = sqlalchemy.DOUBLE_PRECISION
+    NUMERIC = sqlalchemy.NUMERIC
+    REAL    = sqlalchemy.REAL
+    BOOLEAN = sqlalchemy.BOOLEAN
+    JSON    = None
+    BYTES   = None
+    UUID    = None
+
+    TYPE_ANNOTATION_MAP = {}
+
+    @classmethod
+    async def _(cls, component_name, **kwds):
+        self = cls()
+        self.component_name = component_name
+        # each database gets its own models and metadata
+        self.Base = self._declarative_base()
+        self.check_schema = kwds.pop('check_schema', 'true').lower() not in ('false', 'no', 'off', '0')
+        _.databases[component_name] = self
+        setattr(_, component_name, self)
+
+        _.argparser.add_argument(f'--{component_name}-migrate',
+            action='store_true',
+            help=f'Apply schema changes to the {component_name} database'
+            )
+
+        await self.init(component_name, **kwds)
+
+    def _declarative_base(self):
+        class Base(
+                sqlalchemy.ext.asyncio.AsyncAttrs,
+                sqlalchemy.orm.MappedAsDataclass,
+                sqlalchemy.orm.DeclarativeBase,
+                ):
+            'base class for _.records'
+            type_annotation_map = self.TYPE_ANNOTATION_MAP
+        return Base
+
+    async def init(self, component_name, **kwds):
+        kwds['drivername'] = self.DRIVER
+        url = sqlalchemy.engine.URL.create(**kwds)
+        logging.debug('Database URL: %s: %s', component_name, url)
+        self.engine = create_async_engine(url, echo=False)
+        self.session = sqlalchemy.orm.sessionmaker(self.engine, class_=AsyncSession)
+
+    def array(self, item_type):
+        'column type for a list of item_type, overridden by databases with arrays'
+        return JSONArray(binary=isinstance(item_type, sqlalchemy.LargeBinary))
+
+    async def create_tables(self):
+        '''create the tables in an empty database, otherwise compare them to the
+           models and apply the changes if --<name>-migrate was given'''
+        try:
+            async with self.engine.begin() as conn:
+                await conn.run_sync(self._sync_schema)
+        except _.error:
+            raise
+        except Exception as e:
+            raise _.error('Database "%s": %s', self.component_name, e) from None
+
+    def _sync_schema(self, conn):
+        metadata = self.Base.metadata
+        existing = set(sqlalchemy.inspect(conn).get_table_names())
+        if not self.check_schema or not existing & set(metadata.tables):
+            metadata.create_all(conn)
+            return
+
+        try:
+            import alembic.migration
+            import alembic.operations
+        except ImportError:
+            logging.warning('Database "%s": install alembic to check the schema', self.component_name)
+            metadata.create_all(conn)
+            return
+
+        apply,manual = _plan_schema(conn, metadata)
+        migrate = getattr(_.args, f'{self.component_name}_migrate', False)
+
+        # apply nothing unless everything can be applied, SQLite can't roll back DDL
+        if apply and migrate and not manual:
+            operations = alembic.operations.Operations(alembic.migration.MigrationContext.configure(conn))
+            for op in apply:
+                logging.info('%s: %s', self.component_name, _render_sql(conn, op))
+                operations.invoke(op)
+            return
+
+        if not apply and not manual:
+            return
+
+        lines = [f'Database "{self.component_name}" differs from the schema:']
+        lines.extend(f'  {_render_sql(conn, op)}' for op in apply)
+        if manual:
+            lines.append('These changes may lose data and must be applied by hand:')
+            lines.extend(f'  {_render_sql(conn, op)}' for op in manual)
+            if apply:
+                lines.append(f'Then run with --{self.component_name}-migrate to apply the rest')
+        else:
+            lines.append(f'Run with --{self.component_name}-migrate to apply them')
+        raise _.error('%s', '\n'.join(lines))
+
+    async def close(self):
+        await self.engine.dispose()
+
+    # CREATE
+    async def insert(self, *args):
+        try:
+            async with self.session() as session:
+                async with session.begin():
+                    session.add_all(args)
+        except sqlalchemy.exc.IntegrityError as e:
+            raise _.error('%s', e) from None
+
+    # READ
+    async def find(self, cls, value=None, column=None):
+        statement = sqlalchemy.select(cls)
+        if value:
+            column = getattr(cls, column or cls.__primary_key__)
+            statement = statement.where(column == value)
+        async with self.session() as session:
+            results = await session.execute(statement)
+            return results.unique().scalars().all()
+
+    async def find_one(self, cls, value, column=None):
+        statement = sqlalchemy.select(cls)
+        column = getattr(cls, column or cls.__primary_key__)
+        statement = statement.where(column == value)
+        async with self.session() as session:
+            results = await session.execute(statement)
+            return results.unique().scalars().first()
+
+    # UPDATE
+    async def upsert(self, *args):
+        async with self.session() as session:
+            async with session.begin():
+                for obj in args:
+                    await session.merge(obj)
+
+    # DELETE
+    async def delete(self, obj, value=None):
+        async with self.session() as session:
+            async with session.begin():
+                if value is None:
+                    # delete an ORM instance
+                    await session.delete(obj)
+                else:
+                    # otherwise assume obj is the SQLalchemy table class
+                    column = getattr(obj, obj.__primary_key__)
+                    statement = sqlalchemy.delete(obj).where(column == value)
+                    results = await session.execute(statement)
+                    return results.rowcount
+
+
+def _plan_schema(conn, metadata):
+    '''compare the database with the models using alembic autogenerate and split
+       the changes into those that are safe to apply and those that may lose data'''
+    import alembic.autogenerate
+    import alembic.migration
+    import alembic.operations.ops as ops
+
+    # ignore tables in the database that these models don't define
+    def include_name(name, type_, parent_names):
+        return type_ != 'table' or name in metadata.tables
+
+    context = alembic.migration.MigrationContext.configure(conn, opts=dict(
+        include_name = include_name,
+        compare_type = True,
+        ))
+    script = alembic.autogenerate.produce_migrations(context, metadata)
+
+    apply,manual = [],[]
+    for op in script.upgrade_ops.ops:
+        for op in op.ops if isinstance(op, ops.ModifyTableOps) else [op]:
+            if _pkey_nullable_only(metadata, op):
+                continue
+            if isinstance(op, (ops.DropTableOp, ops.DropColumnOp, ops.AlterColumnOp)):
+                manual.append(op)
+            elif isinstance(op, ops.AddColumnOp) and not (op.column.nullable or op.column.server_default):
+                # existing rows would need a value
+                manual.append(op)
+            else:
+                apply.append(op)
+    return apply,manual
+
+
+def _pkey_nullable_only(metadata, op):
+    '''primary keys are never null, but models declare them Optional and
+       SQLite only adds NOT NULL when asked, so ignore that difference'''
+    import alembic.operations.ops as ops
+
+    if not isinstance(op, ops.AlterColumnOp) or op.modify_nullable is None:
+        return False
+    if op.modify_type is not None or op.modify_server_default is not False or op.modify_name is not None:
+        return False
+    table = metadata.tables.get(op.table_name)
+    return table is not None and op.column_name in table.c and table.c[op.column_name].primary_key
+
+
+def _render_sql(conn, op):
+    'the SQL for an alembic operation, for logging and error messages'
+    import alembic.migration
+    import alembic.operations
+
+    output = io.StringIO()
+    context = alembic.migration.MigrationContext.configure(
+        dialect_name = conn.dialect.name,
+        opts         = dict(as_sql=True, output_buffer=output),
+        )
+    alembic.operations.Operations(context).invoke(op)
+    return ' '.join(output.getvalue().split()).rstrip(';')

_/databases/postgres.py

@@ -0,0 +1,38 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import logging
+import typing
+
+import _
+
+try:
+    import asyncpg
+except ImportError:
+    raise _.error('Missing asyncpg module')
+
+import sqlalchemy
+import sqlalchemy.dialects.postgresql
+
+
+class Postgres(_.databases.Database):
+    DRIVER = 'postgresql+asyncpg'
+
+    ARRAY  = sqlalchemy.dialects.postgresql.ARRAY
+    JSON   = sqlalchemy.dialects.postgresql.JSONB
+    BYTES  = sqlalchemy.dialects.postgresql.BYTEA
+    UUID   = sqlalchemy.dialects.postgresql.UUID
+
+    TYPE_ANNOTATION_MAP = {
+        int:                         sqlalchemy.BIGINT,
+        dict[str, typing.Any]:       sqlalchemy.dialects.postgresql.JSONB,
+        list[dict[str, typing.Any]]: sqlalchemy.dialects.postgresql.JSONB,
+        }
+
+    def array(self, item_type):
+        return self.ARRAY(item_type)

_/databases/sqlite.py

@@ -0,0 +1,78 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import logging
+import typing
+import uuid
+
+import _
+
+try:
+    import aiosqlite
+except ImportError:
+    raise _.error('Missing aiosqlite module')
+
+logging.getLogger("aiosqlite").setLevel(logging.WARNING)
+
+import sqlalchemy
+import sqlalchemy.engine
+import sqlalchemy.dialects.sqlite
+
+#aiosqlite.register_adapter(bool, int)
+#aiosqlite.register_converter('BOOLEAN', lambda v: bool(int(v)))
+
+#aiosqlite.register_adapter(uuid.UUID, str)
+#aiosqlite.register_converter('UUID', lambda v: uuid.UUID(v))
+
+
+class UUID(sqlalchemy.types.TypeDecorator):
+    impl = sqlalchemy.types.CHAR
+
+    def load_dialect_impl(self, dialect):
+        return dialect.type_descriptor(sqlalchemy.types.CHAR(32))
+
+    def process_bind_param(self, value, dialect):
+        if value is None:
+            return value
+        if not isinstance(value, uuid.UUID):
+            return '%.32x' % uuid.UUID(value).int
+        return '%.32x' % value.int
+
+    def process_result_value(self, value, dialect):
+        if value is None:
+            return value
+        if not isinstance(value, uuid.UUID):
+            value = uuid.UUID(value)
+        return value
+
+
+class SQLite(_.databases.Database):
+    DRIVER = 'sqlite+aiosqlite'
+
+    ARRAY  = sqlalchemy.dialects.sqlite.JSON
+    JSON   = sqlalchemy.dialects.sqlite.JSON
+    BYTES  = sqlalchemy.dialects.sqlite.BLOB
+    BOOL   = sqlalchemy.dialects.sqlite.BOOLEAN
+    UUID   = UUID
+
+    TYPE_ANNOTATION_MAP = {
+        int:                         sqlalchemy.BIGINT,
+        dict[str, typing.Any]:       sqlalchemy.dialects.sqlite.JSON,
+        list[dict[str, typing.Any]]: sqlalchemy.dialects.sqlite.JSON,
+        }
+
+    async def init(self, component_name, **kwds):
+        logging.getLogger('aiosqlite').setLevel(logging.WARNING)
+        await super(SQLite, self).init(component_name, **kwds)
+
+        # SQLite enforces foreign keys per connection and only when asked
+        @sqlalchemy.event.listens_for(self.engine.sync_engine, 'connect')
+        def foreign_keys(dbapi_conn, record):
+            cursor = dbapi_conn.cursor()
+            cursor.execute('PRAGMA foreign_keys=ON')
+            cursor.close()