underscorepy 0.9.0__py3-none-any.whl

_/__init__.py

@@ -0,0 +1,46 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import collections
+import os
+
+from .utils import *
+
+from . import version
+from . import settings
+from . import auth
+from . import handlers
+from . import components
+
+from .application import Application,WebApplication
+
+
+class error(Exception):
+    '''generic error class for throwing exceptions'''
+
+    def __init__(self, fmt, *args):
+        self.message = fmt % args
+
+    def __str__(self):
+        return self.message
+
+
+class Container(collections.UserDict):
+    '''helper class to hold the loadable components'''
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            raise AttributeError(name) from None
+
+# placeholder for comonents
+caches    = Container()
+databases = Container()
+logins    = Container()
+records   = Container()
+supports  = Container()

_/application.py

@@ -0,0 +1,278 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import asyncio
+import logging
+import signal
+import traceback
+
+import tornado.web
+
+import _
+
+
+class Application:
+    @classmethod
+    def main(cls, ns=None):
+        _.application = self = cls()
+        try:
+            asyncio.run(self._async_main(ns))
+        except _.error as e:
+            if _.args.debug:
+                traceback.print_tb(e.__traceback__)
+            logging.error('%s', e)
+
+    async def _async_main(self, ns):
+        self.loop = asyncio.get_running_loop()
+        self._stop_event = asyncio.Event()
+
+        for signum in (signal.SIGINT, signal.SIGTERM):
+            try:
+                self.loop.add_signal_handler(signum, self.__signalHandler, signum, None)
+            except NotImplementedError:
+                # Windows event loops can't add signal handlers
+                signal.signal(signum, self.__signalHandler)
+
+        name = self.__class__.__name__.lower()
+        try:
+            await _.settings.load(ns=ns, name=name)
+            await self.logging()
+        except _.error as e:
+            if _.args.debug:
+                traceback.print_tb(e.__traceback__)
+            logging.error('%s', e)
+            self.stop()
+
+        if not self._stop_event.is_set():
+            try:
+                await self._async_init()
+            except _.error as e:
+                logging.error('%s', e)
+
+        for name,component in _.caches.items():
+            await component.close()
+
+        for name,component in _.databases.items():
+            await component.close()
+
+    async def _async_init(self, **kwds):
+        # call the underscore application's entry point
+        try:
+            await _.wait(self.initialize())
+        except NotImplementedError:
+            logging.warning('No "initialize" function defined')
+
+        # block until the stop event is set
+        await self._stop_event.wait()
+        # call clean-up code
+        await _.wait(self.on_stop())
+
+    async def initialize(self):
+        '''underscore app entry point'''
+        raise NotImplementedError
+
+    async def logging(self):
+        '''underscore apps can override or extend this function'''
+
+        # add the handlers to the logger
+        if _.config.getboolean(_.name, 'logging', fallback=False):
+            full_path = _.paths(f'{_.name}.log')
+            file_logger = logging.FileHandler(full_path)
+            file_logger.setLevel(logging.DEBUG if _.args.debug else logging.INFO)
+            file_logger.setFormatter(
+                logging.Formatter(
+                    fmt = '%(asctime)s %(levelname)-8s %(message)s',
+                    datefmt = '%Y-%m-%d %H:%M:%S',
+                    )
+                )
+            root_logger = logging.getLogger()
+            root_logger.addHandler(file_logger)
+
+    async def sleep(self, _timeout):
+        try:
+            await asyncio.wait_for(self._stop_event.wait(), timeout=_timeout)
+        except asyncio.TimeoutError:
+            return True
+        return False
+
+    def task(self, fn, *args, **kwds):
+        async def _task():
+            await _.wait(fn(*args, **kwds))
+        return asyncio.create_task(_task())
+
+    def periodic(self, _timeout, fn, *args, **kwds):
+        '''run a function or coroutine on a recurring basis'''
+        async def _periodic():
+            while True:
+                # bail if the stop event is set
+                # otherwise run the function after the timeout occurs
+                try:
+                    await asyncio.wait_for(self._stop_event.wait(), timeout=_timeout)
+                    break
+                except asyncio.TimeoutError:
+                    pass
+                try:
+                    await _.wait(fn(*args, **kwds))
+                except Exception as e:
+                    logging.exception(e)
+        return asyncio.create_task(_periodic())
+
+    def stop(self):
+        logging.debug('Setting stop event')
+        self._stop_event.set()
+
+    def on_stop(self):
+        pass
+
+    def _record_handler(self, component, cls):
+        '''records and caches register REST handlers here, only a WebApplication serves them'''
+        pass
+
+    def _login_handler(self, component, cls):
+        '''logins register their handlers here, only a WebApplication serves them'''
+        pass
+
+    def __signalHandler(self, signum, frame):
+        '''handle signals in a thread-safe way, frame is None unless on Windows'''
+        signame = signal.Signals(signum).name
+        handler = getattr(self, f'on_{signame.lower()}', None)
+        if handler:
+            if handler(signum, frame):
+                return
+        logging.info('Terminating %s on %s signal', _.name, signame)
+        self.loop.call_soon_threadsafe(self.stop)
+
+    # demonstrate a specific signal handler
+    # and print newline after ^C on terminals
+    def on_sigint(self, signum, frame):
+        print()
+
+
+class WebApplication(Application, tornado.web.Application):
+    async def _async_main(self, ns):
+        self._records_patterns = []
+        self._login_patterns   = []
+        self.patterns = []
+        self.settings = {}
+
+        _.argparser.add_argument('--address', '-a',
+            metavar='<address>',
+            help = 'Interface to bind to')
+
+        _.argparser.add_argument('--port', '-p',
+            metavar='<port>',
+            type=int,
+            help='Port to listen on')
+
+        await super(WebApplication, self)._async_main(ns)
+
+    async def _async_init(self, **kwds):
+        _.sessions = _.config.get(_.name, 'sessions', fallback=None)
+        if _.sessions:
+            logging.debug('Sessions cache: %s', _.sessions)
+            try:
+                _.sessions = _.caches[_.sessions]
+            except KeyError:
+                raise _.error('Unknown sessions cache instance: %s', _.sessions)
+
+        self.settings['static_path']   = _.paths('static')
+        self.settings['template_path'] = _.paths('templates')
+        self.settings['debug']         = _.args.debug
+        if 'cookie_secret' not in self.settings:
+            self.settings['cookie_secret'] = await self.cookie_secret()
+
+        # call the underscore application's entry point
+        try:
+            await _.wait(self.initialize())
+        except NotImplementedError:
+            logging.warning('No "initialize" function defined')
+
+        patterns = list(self._records_patterns)
+
+        if self._login_patterns:
+            patterns.extend(self._login_patterns)
+            patterns.extend([
+                ( r'/login',  _.logins.LoginPage ),
+                ( r'/logout', _.logins.Logout    ),
+                ])
+            self.settings['login_url'] = '/login'
+
+        patterns.extend(self.patterns)
+
+        # prepend prefix specified in the ini file, useful for nginx reverse proxy
+        prefix = _.config.get(_.name, 'prefix', fallback='')
+        if prefix:
+            if not prefix[0] == '/':
+                prefix = '/' + prefix
+            patterns = [(f'{prefix}{r[0]}',) + tuple(r[1:]) for r in patterns]
+        else:
+            patterns.append(
+                ( r'/(favicon.ico)', tornado.web.StaticFileHandler, {'path':''}),
+                )
+
+        if _.args.debug:
+            for (pattern,cls,*params) in patterns:
+                handler = f'{cls.__module__}.{cls.__name__}'
+                logging.debug('%-32s %s %s', pattern, handler, params[0] if params else '')
+
+        await self.__listen(patterns)
+        
+        # block until the stop event is set
+        await self._stop_event.wait()
+        # call clean-up code
+        await _.wait(self.on_stop())
+
+    async def __listen(self, patterns, **kwds):
+        '''call the Tornado Application init here to give children a chance
+           to initialize patterns and settings'''
+        super().__init__(patterns, **self.settings)
+
+        if not _.args.address:
+            _.args.address = _.config.get(_.name, 'address', fallback='127.0.0.1')
+
+        if not _.args.port:
+            _.args.port = _.config.getint(_.name, 'port', fallback=8080)
+
+        if 'xheaders' not in kwds:
+            kwds['xheaders'] = True
+
+        try:
+            self.listen(_.args.port, _.args.address, **kwds)
+            logging.info('Listening on %s:%d', _.args.address, _.args.port)
+        except Exception as e:
+            raise _.error('%s', e) from None
+
+    def _record_handler(self, component, cls):
+        '''called from _.records component to add handler'''
+        self._records_patterns.append(
+            (f'/{component}/{cls._component}(?:/(.*))?', cls)
+            )
+
+    def _login_handler(self, component, cls):
+        '''called from _.logins component to add handler'''
+        self._login_patterns.append(
+            (f'/{component}/{cls._component}', cls)
+            )
+
+    async def cookie_secret(self):
+        '''underscore apps can override this function'''
+        if _.sessions is not None:
+            return await _.wait(_.sessions.cookie_secret())
+
+    async def on_login_success(self, handler, user, *args, **kwds):
+        '''underscore apps should override this function if a login component is specified'''
+        raise NotImplementedError
+
+    async def on_login_failure(self, component, *args, **kwds):
+        '''underscore apps should override this function if a login component is specified'''
+        pass
+
+    async def on_logout(self, sessions_id):
+        '''underscore apps should override this function if a login component is specified'''
+        pass
+

_/auth.py

@@ -0,0 +1,163 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import base64
+import binascii
+import functools
+import hashlib
+import hmac
+import os
+import urllib.parse
+
+import _
+
+
+# based on tornado.web.authenticated
+def _next(handler):
+    if handler.request.method in ("GET","HEAD"):
+        url = handler.get_login_url()
+        if "?" not in url:
+            if urllib.parse.urlsplit(url).scheme:
+                next_url = handler.request.full_url()
+            else:
+                next_url = handler.request.uri
+            url += "?" + urllib.parse.urlencode(dict(next=next_url))
+        handler.redirect(url)
+    else:
+        raise _.HTTPError(403)
+
+
+# generic decorator to callback to a function
+# return true to allow the method to be called
+# return false to block the method
+def filter(filter_func):
+    def _filter(method):
+        @functools.wraps(method)
+        def wrapper(self, *args, **kwds):
+            if filter_func(self):
+                return method(self, *args, **kwds)
+            _next(self)
+        return wrapper
+    return _filter
+
+
+# same as above but pass the current_user session to the function
+# blocks when no session is associated with the request
+# return true to allow the method to be called
+# return false to block the method
+def filter_user(filter_func):
+    def _filter_user(method):
+        @functools.wraps(method)
+        def wrapper(self, *args, **kwds):
+            if self.current_user:
+                if filter_func(self.current_user):
+                    return method(self, *args, **kwds)
+            _next(self)
+        return wrapper
+    return _filter_user
+
+
+# inteded to be used by underscore apps to check that any user is logged in
+current_user = filter_user(lambda current_user: True)
+
+# intended to be over-ridden before components are loaded for more fine-grain
+# control over access to component handlers
+protected    = filter_user(lambda current_user: True)
+
+# default record access is unprotected, override before components are loaded
+# to require a session for the _.records handlers, for example:
+#
+#   _.auth.records = _.auth.filter_user(lambda session: session)
+#
+records      = lambda fn: fn
+
+
+# implement old school user/name password prompt
+# useful for command-line tools to pass as part of the URL
+def basic(realm='Authentication'):
+    def basic_auth(method):
+        @functools.wraps(method)
+        async def wrapper(self, *args, **kwds):
+            auth = self.request.headers.get('Authorization', '')
+            if auth.startswith('Basic '):
+                try:
+                    auth = binascii.a2b_base64(auth[6:]).decode('utf-8')
+                    username,password = auth.split(':', 1)
+                except (binascii.Error,UnicodeDecodeError,ValueError):
+                    raise _.HTTPError(400) from None
+                component = _.config.get(_.name, 'basic', fallback=None)
+                if not component:
+                    raise _.HTTPError(500, 'No component specified for basic auth')
+                try:
+                    login = _.logins[component]
+                except KeyError:
+                    raise _.HTTPError(500, 'Invalid component specified for basic auth')
+                success = await login.check(username, password)
+                if success:
+                    return await _.wait(method(self, *args, **kwds))
+            self.set_status(401)
+            self.set_header('WWW-Authenticate', f'Basic realm="{realm}"')
+            self.finish()
+        return wrapper
+    return basic_auth
+
+
+# scrypt parameters for new hashes, stored alongside each hash so they can be
+# raised later without invalidating existing passwords
+SCRYPT_N = 2 ** 15
+SCRYPT_R = 8
+SCRYPT_P = 1
+
+
+def _scrypt(password, salt, n, r, p):
+    return hashlib.scrypt(password.encode('utf-8'), salt=salt,
+        n=n, r=r, p=p, maxmem=128 * r * (n + p + 2), dklen=64)
+
+
+def hash_password(password):
+    '''return a salted scrypt hash formatted as scrypt$n$r$p$salt$hash'''
+    salt = os.urandom(16)
+    digest = _scrypt(password, salt, SCRYPT_N, SCRYPT_R, SCRYPT_P)
+    salt   = base64.b64encode(salt).decode('ascii')
+    digest = base64.b64encode(digest).decode('ascii')
+    return f'scrypt${SCRYPT_N}${SCRYPT_R}${SCRYPT_P}${salt}${digest}'
+
+
+def verify_password(password, stored, username=''):
+    '''check a password against a stored hash, username is only used by legacy hashes'''
+    if not password or not stored:
+        return False
+    if not stored.startswith('scrypt$'):
+        return hmac.compare_digest(simple_hash(username + password), stored)
+    try:
+        _scheme,n,r,p,salt,digest = stored.split('$')
+        n,r,p  = int(n), int(r), int(p)
+        salt   = base64.b64decode(salt)
+        digest = base64.b64decode(digest)
+    except (ValueError,binascii.Error):
+        return False
+    return hmac.compare_digest(_scrypt(password, salt, n, r, p), digest)
+
+
+def needs_rehash(stored):
+    '''true for legacy hashes or hashes made with weaker parameters'''
+    if not stored or not stored.startswith('scrypt$'):
+        return True
+    try:
+        n,r,p = (int(v) for v in stored.split('$')[1:4])
+    except ValueError:
+        return True
+    return (n,r,p) != (SCRYPT_N,SCRYPT_R,SCRYPT_P)
+
+
+def simple_hash(value):
+    '''legacy unsalted hash, only kept to verify existing passwords'''
+    value = value.encode('utf-8')
+    for i in range(99999):
+        value = hashlib.sha512(value).digest()
+    return base64.b64encode(value).decode('ascii')

_/caches/__init__.py

@@ -0,0 +1,50 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import _
+
+
+class Cache:
+    @classmethod
+    async def _(cls, component_name, **kwds):
+        # create a dynamic child class with kwds from the ini file
+        try:
+            members = dict(_.config['sessions'])
+        except KeyError:
+            members = {}
+
+        members['expires']  = int(members.get('expires',  24))
+        members['interval'] = int(members.get('interval', 60))
+        members.update(kwds)
+
+        # instantiate the derived class
+        self = type(cls.__name__, (cls,), _.prefix(members))()
+        _.caches[component_name] = self
+        await self.init(component_name, **kwds)
+
+    async def init(self, component_name, **kwds):
+        pass
+
+    async def close(self):
+        pass
+
+    async def cookie_secret(self):
+        raise NotImplementedError
+
+    def save_session(self, session):
+        raise NotImplementedError
+
+    def load_session(self, session_id):
+        raise NotImplementedError
+
+    def clear_session(self, session_id):
+        raise NotImplementedError
+
+
+class Handler(_.handlers.Protected):
+    pass

_/caches/dbcache.py

@@ -0,0 +1,140 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import base64
+import json
+import logging
+import os
+import typing
+
+import sqlalchemy
+
+import _
+import _.records
+
+class DbCache(_.caches.Cache):
+    _config  = 'config'
+    _key_col = 'key'
+    _val_col = 'value'
+    _key     = 'cookie'
+
+    _table       = 'sessions'
+    _session_col = 'session_id'
+
+    async def init(self, component_name, database=None, **kwds):
+        if not hasattr(_.application, 'is_session_expired'):
+            raise _.error('Application does not have is_session_expired function defined')
+
+        if database is None:
+            if 1 == len(_.databases):
+                database = list(_.databases.keys())[0]
+            else:
+                raise _.error('dbcache requires a database to be specified')
+        self.db = _.databases[database]
+
+        self.config_table = type(self._config, (_.records.RecordsInterface,self.db.Base,), {
+            '__tablename__'   : self._config,
+            '__annotations__' : {
+                self._key_col : str,
+                self._val_col : typing.Optional[str],
+                },
+            '__primary_key__' : self._key_col,
+            self._key_col : sqlalchemy.orm.mapped_column(primary_key=True, init=False),
+            self._val_col : sqlalchemy.orm.mapped_column(init=False),
+            })
+
+        annotations = {
+            self._session_col : str,
+            }
+        # create the session table
+        columns = {
+            '__tablename__'   : self._table,
+            '__annotations__' : annotations,
+            '__primary_key__' : self._session_col,
+            self._session_col : sqlalchemy.orm.mapped_column(primary_key=True, init=False),
+            }
+        for col,dbtype in kwds.items():
+            if not dbtype:
+                dbtype = 'str'
+            annotations[col] = typing.Optional[__builtins__.get(dbtype)]
+            columns[col] = sqlalchemy.orm.mapped_column(init=False)
+
+        self.session_table = type(self._table, (_.records.RecordsInterface, self.db.Base,), columns)
+
+        # interval comes from the [sessions] section of the ini
+        _.application.periodic(self._interval, self.clear_stale_sessions)
+
+        members = dict(
+            component  = component_name,
+            db         = self.db,
+            cls        = self.session_table,
+            table      = self._table,
+            session_id = self._session_col,
+            )
+        subclass = type('Sessions', (Sessions,), _.prefix(members))
+        _.application._record_handler('sessions', subclass)
+
+    async def cookie_secret(self):
+        # try to load exisiting cookie secret and return it
+        secret = await self.db.find_one(self.config_table, self._key)
+        if secret:
+            secret = secret[self._val_col]
+        else:
+            # otherwise generate and store the cookie secret
+            secret = base64.b64encode(os.urandom(32)).decode('ascii')
+            config = self.config_table()
+            config(**{self._key_col : self._key, self._val_col : secret})
+            await self.db.upsert(config)
+        return secret
+
+    async def save_session(self, session):
+        session = self.session_table._from_dict(**session)
+        await self.db.upsert(session)
+
+    async def load_session(self, session_id):
+        session = await self.db.find_one(self.session_table, session_id)
+        if not session:
+            return None
+        if await _.wait(_.application.is_session_expired(session, self._expires)):
+            return None
+        return session
+
+    async def clear_session(self, session_id):
+        session = await self.db.find_one(self.session_table, session_id)
+        if session:
+            await self.db.delete(session)
+
+    async def clear_stale_sessions(self):
+        for record in await self.db.find(self.session_table):
+            if await _.wait(_.application.is_session_expired(record, self._expires)):
+                logging.debug('Removing expired session: %s', record[self._session_col])
+                await self.db.delete(record)
+
+
+class Sessions(_.handlers.Protected):
+    @_.auth.protected
+    async def get(self, session_id=None):
+        if session_id:
+            record = await self._db.find_one(self._cls, session_id)
+            self.write(record._as_dict())
+        else:
+            records = await self._db.find(self._cls)
+            data = [r._as_dict() for r in records]
+            self.write({'data':data})
+
+    @_.auth.protected
+    async def delete(self, session_id=None):
+        if session_id:
+            await self._db.delete(self._cls, session_id)
+
+            callback = getattr(_.application, f'on_{self._component}_delete', None)
+            if callback is None:
+                callback = getattr(_.application, 'on_dbcache_delete', None)
+            if callback:
+                await _.wait(callback(self._component, session_id))
+        self.set_status(204)