underscorepy 0.6.3__tar.gz → 0.9.0__tar.gz

underscorepy-0.9.0/PKG-INFO

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: underscorepy
+Version: 0.9.0
+Summary: _.py library
+Author-email: Matthew Shaw <mshaw.cx@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://shaw.cx/_.py
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tornado>=6.3
+Requires-Dist: sqlalchemy[asyncio]>=2
+Provides-Extra: postgres
+Requires-Dist: asyncpg; extra == "postgres"
+Provides-Extra: sqlite
+Requires-Dist: aiosqlite; extra == "sqlite"
+Provides-Extra: redis
+Requires-Dist: redis>=5.0.1; extra == "redis"
+Provides-Extra: protobuf
+Requires-Dist: protobuf>=5.26; extra == "protobuf"
+Provides-Extra: ldap
+Requires-Dist: python-ldap; extra == "ldap"
+Provides-Extra: migrations
+Requires-Dist: alembic; extra == "migrations"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: aiosqlite; extra == "test"
+Requires-Dist: alembic; extra == "test"
+Provides-Extra: all
+Requires-Dist: underscorepy[ldap,migrations,postgres,protobuf,redis,sqlite]; extra == "all"
+Dynamic: license-file
+
+
+underscore.py
+-------------
+
+```INI
+[myapp]
+address = 0.0.0.0
+port    = 8080
+```
+
+```Python
+import _
+import _.websockets
+
+class Echo(_.websockets.WebSocket):
+    def on_message(self, msg):
+        print('Echo:', msg)
+        self.write_message(msg)
+
+class MyApp(_.Application):
+    def initialize(self):
+        self.patterns = [
+            ( r'/ws',       Echo                ),
+            ( r'/([a-z]*)', _.handlers.Template ),
+            ]
+
+MyApp.main()
+```
+
+Look at the skeleton directory for a more complete example application.
+
+Schema changes
+--------------
+
+Tables for records are created when the database is empty. After that, each
+start compares the models (for example your `.proto` messages) with the
+database and refuses to run if they differ:
+
+```
+ERROR    Database "store" differs from the schema:
+  ALTER TABLE "Reports_multiple" ADD COLUMN _position INTEGER
+Run with --store-migrate to apply them
+```
+
+`--<database>-migrate` applies changes that can't lose data. Drops and type
+changes are printed as SQL to run by hand. This needs `alembic`
+(`pip install underscorepy[migrations]`); set `check_schema = false` in the
+database's section to skip the check.

underscorepy-0.9.0/README.md

@@ -0,0 +1,48 @@
+
+underscore.py
+-------------
+
+```INI
+[myapp]
+address = 0.0.0.0
+port    = 8080
+```
+
+```Python
+import _
+import _.websockets
+
+class Echo(_.websockets.WebSocket):
+    def on_message(self, msg):
+        print('Echo:', msg)
+        self.write_message(msg)
+
+class MyApp(_.Application):
+    def initialize(self):
+        self.patterns = [
+            ( r'/ws',       Echo                ),
+            ( r'/([a-z]*)', _.handlers.Template ),
+            ]
+
+MyApp.main()
+```
+
+Look at the skeleton directory for a more complete example application.
+
+Schema changes
+--------------
+
+Tables for records are created when the database is empty. After that, each
+start compares the models (for example your `.proto` messages) with the
+database and refuses to run if they differ:
+
+```
+ERROR    Database "store" differs from the schema:
+  ALTER TABLE "Reports_multiple" ADD COLUMN _position INTEGER
+Run with --store-migrate to apply them
+```
+
+`--<database>-migrate` applies changes that can't lose data. Drops and type
+changes are printed as SQL to run by hand. This needs `alembic`
+(`pip install underscorepy[migrations]`); set `check_schema = false` in the
+database's section to skip the check.

underscorepy-0.9.0/_/__init__.py

@@ -0,0 +1,46 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import collections
+import os
+
+from .utils import *
+
+from . import version
+from . import settings
+from . import auth
+from . import handlers
+from . import components
+
+from .application import Application,WebApplication
+
+
+class error(Exception):
+    '''generic error class for throwing exceptions'''
+
+    def __init__(self, fmt, *args):
+        self.message = fmt % args
+
+    def __str__(self):
+        return self.message
+
+
+class Container(collections.UserDict):
+    '''helper class to hold the loadable components'''
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            raise AttributeError(name) from None
+
+# placeholder for comonents
+caches    = Container()
+databases = Container()
+logins    = Container()
+records   = Container()
+supports  = Container()

{underscorepy-0.6.3 → underscorepy-0.9.0}/_/application.py

@@ -8,10 +8,7 @@
 
 import asyncio
 import logging
-import os
 import signal
-import socket
-import sys
 import traceback
 
 import tornado.web
@@ -19,29 +16,27 @@ import tornado.web
 import _
 
 
-class Application(tornado.web.Application):
+class Application:
     @classmethod
     def main(cls, ns=None):
         _.application = self = cls()
         try:
-            asyncio.run(self.__async_main(ns))
+            asyncio.run(self._async_main(ns))
         except _.error as e:
             if _.args.debug:
                 traceback.print_tb(e.__traceback__)
             logging.error('%s', e)
 
-    async def __async_main(self, ns):
-        self._records_patterns = []
-        self._login_patterns   = []
-
-        self.patterns = []
-        self.settings = {}
-
-        self.loop = asyncio.get_event_loop()
+    async def _async_main(self, ns):
+        self.loop = asyncio.get_running_loop()
         self._stop_event = asyncio.Event()
 
-        signal.signal(signal.SIGINT,  self.__signalHandler)
-        signal.signal(signal.SIGTERM, self.__signalHandler)
+        for signum in (signal.SIGINT, signal.SIGTERM):
+            try:
+                self.loop.add_signal_handler(signum, self.__signalHandler, signum, None)
+            except NotImplementedError:
+                # Windows event loops can't add signal handlers
+                signal.signal(signum, self.__signalHandler)
 
         name = self.__class__.__name__.lower()
         try:
@@ -53,18 +48,9 @@ class Application(tornado.web.Application):
             logging.error('%s', e)
             self.stop()
 
-        if not self._stop_event.is_set():
-            for name,component in _.logins.items():
-                try:
-                    await _.wait(component.args(name))
-                except _.error as e:
-                    logging.error('%s', e)
-                    self.stop()
-                    break
-
         if not self._stop_event.is_set():
             try:
-                await self.__async_init()
+                await self._async_init()
             except _.error as e:
                 logging.error('%s', e)
 
@@ -74,84 +60,24 @@ class Application(tornado.web.Application):
         for name,component in _.databases.items():
             await component.close()
 
-    async def __async_init(self, **kwds):
-        # check if a sessions cache was specified
-        _.sessions = _.config.get(_.name, 'sessions', fallback=None)
-        if _.sessions:
-            logging.debug('Sessions cache: %s', _.sessions)
-            try:
-                _.sessions = _.caches[_.sessions]
-            except KeyError:
-                raise _.error('Unknown sessions cache instance: %s', _.sessions)
-
-        self.settings['static_path']   = _.paths('static')
-        self.settings['template_path'] = _.paths('templates')
-        self.settings['debug']         = _.args.debug
-
+    async def _async_init(self, **kwds):
         # call the underscore application's entry point
         try:
             await _.wait(self.initialize())
         except NotImplementedError:
             logging.warning('No "initialize" function defined')
 
-        if 'cookie_secret' not in self.settings:
-            self.settings['cookie_secret'] = await self.cookie_secret()
-
-        patterns = list(self._records_patterns)
-
-        if self._login_patterns:
-            self._login_patterns += [
-                ( r'/login',  _.logins.LoginPage ),
-                ( r'/logout', _.logins.Logout    ),
-                ]
-            self.settings['login_url'] = '/login'
-        patterns += self._login_patterns
-        patterns += self.patterns
-
-        patterns.append(
-            ( r'/(favicon.ico)', tornado.web.StaticFileHandler, {'path':''}),
-            )
-
-        if _.args.debug:
-            for (pattern,cls,*params) in patterns:
-                handler = f'{cls.__module__}.{cls.__name__}'
-                logging.debug('%-32s %s %s', pattern, handler, params[0] if params else '')
-
-        await self.__listen(patterns)
+        # block until the stop event is set
         await self._stop_event.wait()
+        # call clean-up code
         await _.wait(self.on_stop())
 
-    async def __listen(self, patterns, **kwds):
-        # call the Tornado Application init here to give children a chance
-        # to initialize patterns and settings
-        super().__init__(patterns, **self.settings)
-
-        if 'xheaders' not in kwds:
-            kwds['xheaders'] = True
-
-        try:
-            self.listen(_.args.port, _.args.address, **kwds)
-        except Exception as e:
-            raise _.error('%s', e) from None
-
-        logging.info('Listening on %s:%d', _.args.address, _.args.port)
-
-    def _record_handler(self, name, cls):
-        self._records_patterns.append(
-            (f'/{name}/{cls._name}(?:/(.*))?', cls)
-            )
-
-    def _login_handler(self, name, cls):
-        self._login_patterns.append(
-            (f'/{name}/{cls._name}', cls)
-            )
-
     async def initialize(self):
-        'underscore apps should override this function'
+        '''underscore app entry point'''
         raise NotImplementedError
 
     async def logging(self):
-        'underscore apps can override or extend this function'
+        '''underscore apps can override or extend this function'''
 
         # add the handlers to the logger
         if _.config.getboolean(_.name, 'logging', fallback=False):
@@ -167,14 +93,12 @@ class Application(tornado.web.Application):
             root_logger = logging.getLogger()
             root_logger.addHandler(file_logger)
 
-    async def cookie_secret(self):
-        'underscore apps can override this function'
-        if _.sessions is not None:
-            return await _.wait(_.sessions.cookie_secret())
-
-    async def on_login(self, handler, user, *args, **kwds):
-        'underscore apps should override this function if a login is specified'
-        raise NotImplementedError
+    async def sleep(self, _timeout):
+        try:
+            await asyncio.wait_for(self._stop_event.wait(), timeout=_timeout)
+        except asyncio.TimeoutError:
+            return True
+        return False
 
     def task(self, fn, *args, **kwds):
         async def _task():
@@ -182,7 +106,7 @@ class Application(tornado.web.Application):
         return asyncio.create_task(_task())
 
     def periodic(self, _timeout, fn, *args, **kwds):
-        'run a function or coroutine on a recurring basis'
+        '''run a function or coroutine on a recurring basis'''
         async def _periodic():
             while True:
                 # bail if the stop event is set
@@ -190,10 +114,10 @@ class Application(tornado.web.Application):
                 try:
                     await asyncio.wait_for(self._stop_event.wait(), timeout=_timeout)
                     break
-                except asyncio.TimeoutError as e:
+                except asyncio.TimeoutError:
                     pass
                 try:
-                    result = await _.wait(fn(*args, **kwds))
+                    await _.wait(fn(*args, **kwds))
                 except Exception as e:
                     logging.exception(e)
         return asyncio.create_task(_periodic())
@@ -205,8 +129,16 @@ class Application(tornado.web.Application):
     def on_stop(self):
         pass
 
+    def _record_handler(self, component, cls):
+        '''records and caches register REST handlers here, only a WebApplication serves them'''
+        pass
+
+    def _login_handler(self, component, cls):
+        '''logins register their handlers here, only a WebApplication serves them'''
+        pass
+
     def __signalHandler(self, signum, frame):
-        'handle signals in a thread-safe way'
+        '''handle signals in a thread-safe way, frame is None unless on Windows'''
         signame = signal.Signals(signum).name
         handler = getattr(self, f'on_{signame.lower()}', None)
         if handler:
@@ -219,3 +151,128 @@ class Application(tornado.web.Application):
     # and print newline after ^C on terminals
     def on_sigint(self, signum, frame):
         print()
+
+
+class WebApplication(Application, tornado.web.Application):
+    async def _async_main(self, ns):
+        self._records_patterns = []
+        self._login_patterns   = []
+        self.patterns = []
+        self.settings = {}
+
+        _.argparser.add_argument('--address', '-a',
+            metavar='<address>',
+            help = 'Interface to bind to')
+
+        _.argparser.add_argument('--port', '-p',
+            metavar='<port>',
+            type=int,
+            help='Port to listen on')
+
+        await super(WebApplication, self)._async_main(ns)
+
+    async def _async_init(self, **kwds):
+        _.sessions = _.config.get(_.name, 'sessions', fallback=None)
+        if _.sessions:
+            logging.debug('Sessions cache: %s', _.sessions)
+            try:
+                _.sessions = _.caches[_.sessions]
+            except KeyError:
+                raise _.error('Unknown sessions cache instance: %s', _.sessions)
+
+        self.settings['static_path']   = _.paths('static')
+        self.settings['template_path'] = _.paths('templates')
+        self.settings['debug']         = _.args.debug
+        if 'cookie_secret' not in self.settings:
+            self.settings['cookie_secret'] = await self.cookie_secret()
+
+        # call the underscore application's entry point
+        try:
+            await _.wait(self.initialize())
+        except NotImplementedError:
+            logging.warning('No "initialize" function defined')
+
+        patterns = list(self._records_patterns)
+
+        if self._login_patterns:
+            patterns.extend(self._login_patterns)
+            patterns.extend([
+                ( r'/login',  _.logins.LoginPage ),
+                ( r'/logout', _.logins.Logout    ),
+                ])
+            self.settings['login_url'] = '/login'
+
+        patterns.extend(self.patterns)
+
+        # prepend prefix specified in the ini file, useful for nginx reverse proxy
+        prefix = _.config.get(_.name, 'prefix', fallback='')
+        if prefix:
+            if not prefix[0] == '/':
+                prefix = '/' + prefix
+            patterns = [(f'{prefix}{r[0]}',) + tuple(r[1:]) for r in patterns]
+        else:
+            patterns.append(
+                ( r'/(favicon.ico)', tornado.web.StaticFileHandler, {'path':''}),
+                )
+
+        if _.args.debug:
+            for (pattern,cls,*params) in patterns:
+                handler = f'{cls.__module__}.{cls.__name__}'
+                logging.debug('%-32s %s %s', pattern, handler, params[0] if params else '')
+
+        await self.__listen(patterns)
+        
+        # block until the stop event is set
+        await self._stop_event.wait()
+        # call clean-up code
+        await _.wait(self.on_stop())
+
+    async def __listen(self, patterns, **kwds):
+        '''call the Tornado Application init here to give children a chance
+           to initialize patterns and settings'''
+        super().__init__(patterns, **self.settings)
+
+        if not _.args.address:
+            _.args.address = _.config.get(_.name, 'address', fallback='127.0.0.1')
+
+        if not _.args.port:
+            _.args.port = _.config.getint(_.name, 'port', fallback=8080)
+
+        if 'xheaders' not in kwds:
+            kwds['xheaders'] = True
+
+        try:
+            self.listen(_.args.port, _.args.address, **kwds)
+            logging.info('Listening on %s:%d', _.args.address, _.args.port)
+        except Exception as e:
+            raise _.error('%s', e) from None
+
+    def _record_handler(self, component, cls):
+        '''called from _.records component to add handler'''
+        self._records_patterns.append(
+            (f'/{component}/{cls._component}(?:/(.*))?', cls)
+            )
+
+    def _login_handler(self, component, cls):
+        '''called from _.logins component to add handler'''
+        self._login_patterns.append(
+            (f'/{component}/{cls._component}', cls)
+            )
+
+    async def cookie_secret(self):
+        '''underscore apps can override this function'''
+        if _.sessions is not None:
+            return await _.wait(_.sessions.cookie_secret())
+
+    async def on_login_success(self, handler, user, *args, **kwds):
+        '''underscore apps should override this function if a login component is specified'''
+        raise NotImplementedError
+
+    async def on_login_failure(self, component, *args, **kwds):
+        '''underscore apps should override this function if a login component is specified'''
+        pass
+
+    async def on_logout(self, sessions_id):
+        '''underscore apps should override this function if a login component is specified'''
+        pass
+

{underscorepy-0.6.3 → underscorepy-0.9.0}/_/auth.py

@@ -10,6 +10,8 @@ import base64
 import binascii
 import functools
 import hashlib
+import hmac
+import os
 import urllib.parse
 
 import _
@@ -67,6 +69,13 @@ current_user = filter_user(lambda current_user: True)
 # control over access to component handlers
 protected    = filter_user(lambda current_user: True)
 
+# default record access is unprotected, override before components are loaded
+# to require a session for the _.records handlers, for example:
+#
+#   _.auth.records = _.auth.filter_user(lambda session: session)
+#
+records      = lambda fn: fn
+
 
 # implement old school user/name password prompt
 # useful for command-line tools to pass as part of the URL
@@ -76,8 +85,11 @@ def basic(realm='Authentication'):
         async def wrapper(self, *args, **kwds):
             auth = self.request.headers.get('Authorization', '')
             if auth.startswith('Basic '):
-                auth = binascii.a2b_base64(auth[6:]).decode('utf-8')
-                username,password = auth.split(':', 1)
+                try:
+                    auth = binascii.a2b_base64(auth[6:]).decode('utf-8')
+                    username,password = auth.split(':', 1)
+                except (binascii.Error,UnicodeDecodeError,ValueError):
+                    raise _.HTTPError(400) from None
                 component = _.config.get(_.name, 'basic', fallback=None)
                 if not component:
                     raise _.HTTPError(500, 'No component specified for basic auth')
@@ -87,19 +99,65 @@ def basic(realm='Authentication'):
                     raise _.HTTPError(500, 'Invalid component specified for basic auth')
                 success = await login.check(username, password)
                 if success:
-                    return method(self, *args, **kwds)
+                    return await _.wait(method(self, *args, **kwds))
             self.set_status(401)
-            self.set_header('WWW-Authenticate', f'Basic realm={realm}')
+            self.set_header('WWW-Authenticate', f'Basic realm="{realm}"')
             self.finish()
         return wrapper
     return basic_auth
 
 
+# scrypt parameters for new hashes, stored alongside each hash so they can be
+# raised later without invalidating existing passwords
+SCRYPT_N = 2 ** 15
+SCRYPT_R = 8
+SCRYPT_P = 1
+
+
+def _scrypt(password, salt, n, r, p):
+    return hashlib.scrypt(password.encode('utf-8'), salt=salt,
+        n=n, r=r, p=p, maxmem=128 * r * (n + p + 2), dklen=64)
+
+
+def hash_password(password):
+    '''return a salted scrypt hash formatted as scrypt$n$r$p$salt$hash'''
+    salt = os.urandom(16)
+    digest = _scrypt(password, salt, SCRYPT_N, SCRYPT_R, SCRYPT_P)
+    salt   = base64.b64encode(salt).decode('ascii')
+    digest = base64.b64encode(digest).decode('ascii')
+    return f'scrypt${SCRYPT_N}${SCRYPT_R}${SCRYPT_P}${salt}${digest}'
+
+
+def verify_password(password, stored, username=''):
+    '''check a password against a stored hash, username is only used by legacy hashes'''
+    if not password or not stored:
+        return False
+    if not stored.startswith('scrypt$'):
+        return hmac.compare_digest(simple_hash(username + password), stored)
+    try:
+        _scheme,n,r,p,salt,digest = stored.split('$')
+        n,r,p  = int(n), int(r), int(p)
+        salt   = base64.b64decode(salt)
+        digest = base64.b64decode(digest)
+    except (ValueError,binascii.Error):
+        return False
+    return hmac.compare_digest(_scrypt(password, salt, n, r, p), digest)
+
+
+def needs_rehash(stored):
+    '''true for legacy hashes or hashes made with weaker parameters'''
+    if not stored or not stored.startswith('scrypt$'):
+        return True
+    try:
+        n,r,p = (int(v) for v in stored.split('$')[1:4])
+    except ValueError:
+        return True
+    return (n,r,p) != (SCRYPT_N,SCRYPT_R,SCRYPT_P)
+
+
 def simple_hash(value):
+    '''legacy unsalted hash, only kept to verify existing passwords'''
     value = value.encode('utf-8')
     for i in range(99999):
         value = hashlib.sha512(value).digest()
     return base64.b64encode(value).decode('ascii')
-
-
-__all__ = _.all()

{underscorepy-0.6.3 → underscorepy-0.9.0}/_/caches/__init__.py

@@ -11,7 +11,7 @@ import _
 
 class Cache:
     @classmethod
-    async def _(cls, name, **kwds):
+    async def _(cls, component_name, **kwds):
         # create a dynamic child class with kwds from the ini file
         try:
             members = dict(_.config['sessions'])
@@ -24,10 +24,10 @@ class Cache:
 
         # instantiate the derived class
         self = type(cls.__name__, (cls,), _.prefix(members))()
-        _.caches[name] = self
-        await self.init(name, **kwds)
+        _.caches[component_name] = self
+        await self.init(component_name, **kwds)
 
-    async def init(self, **kwds):
+    async def init(self, component_name, **kwds):
         pass
 
     async def close(self):
@@ -37,12 +37,12 @@ class Cache:
         raise NotImplementedError
 
     def save_session(self, session):
-        try:
-            return session['session_id']
-        except KeyError:
-            raise _.error('No session_id defined in session')
+        raise NotImplementedError
+
+    def load_session(self, session_id):
+        raise NotImplementedError
 
-    async def load_session(self, session_id):
+    def clear_session(self, session_id):
         raise NotImplementedError