underscorepy 0.9.0__py3-none-any.whl

_/records/protobuf.py

@@ -0,0 +1,496 @@
+#
+# (c) 2015-2024 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import configparser
+import dataclasses
+import json
+import keyword
+import logging
+import os
+import types
+import typing
+
+import sqlalchemy
+import sqlalchemy.ext.orderinglist
+import tornado.web
+import google.protobuf.descriptor
+import google.protobuf.message
+import google.protobuf.json_format
+
+import _
+
+
+def _repeated(field):
+    # FieldDescriptor.label was removed in protobuf 7
+    try:
+        return field.is_repeated
+    except AttributeError:
+        return field.label == field.LABEL_REPEATED
+
+
+def _check_names(descriptor):
+    # record fields are Python attributes so keywords can't be used
+    for field in descriptor.fields:
+        if keyword.iskeyword(field.name):
+            raise _.error('Field "%s" in %s is a Python keyword', field.name, descriptor.full_name)
+
+
+def _is_map(field):
+    return field.message_type is not None and field.message_type.GetOptions().map_entry
+
+
+def _is_message(field):
+    return field.type == field.TYPE_MESSAGE
+
+
+def _json_form(pb):
+    'the proto3 JSON form of a message, keeping the field names from the .proto'
+    return google.protobuf.json_format.MessageToDict(pb,
+        always_print_fields_with_no_presence = True,
+        preserving_proto_field_name          = True,
+        )
+
+
+class Protobuf(_.records.Record):
+    async def init(self, module, **kwds):
+        # setup the container beforehand so the data module can use data decorators
+        if hasattr(_, self.component_name):
+            raise _.error('Record name "%s" for "%s" conflicts in _ root', self.component_name, module.__name__)
+
+        _.argparser.add_argument(f'--{self.component_name}',
+            action='store_true',
+            help='output Protobuf options template'
+            )
+
+        # options come from Protobuf.proto but need to be compiled in-line to avoid headaches
+        self._options_name = f"{kwds.get('options', 'Protobuf')}_pb2"
+
+        # container class for derived Python types
+        self._container = ProtobufContainer()
+
+        # access the types via the name or alias of the component
+        setattr(_, self.component_name, self._container)
+
+        # default to JSON style tables instead of relational
+        try:
+            self.relational = _.config.getboolean(self.component_name, 'relational')
+        except configparser.NoOptionError:
+            self.relational = False
+
+        # default to creating handlers unless specified otherwise in the ini
+        try:
+            self.handlers = _.config.getboolean(self.component_name, 'handlers')
+        except configparser.NoOptionError:
+            self.handlers = True
+
+        await super().init(module, **kwds)
+
+    @classmethod
+    async def args(cls, component_name):
+        if getattr(_.args, component_name, False):
+            path = os.path.join(os.path.dirname(__file__), 'Protobuf.proto')
+            protobuf = open(path, 'r').read()
+            print(protobuf)
+            _.application.stop()
+
+    def load(self, module):
+        try:
+            self._options = getattr(module, self._options_name)
+        except AttributeError:
+            raise _.error('Cannot find protobuf module "%s"', self._options_name)
+        self._package = module.__name__
+        # container path of the subpackage being loaded, e.g. ['sub']
+        self._path = []
+        self.load_module(module)
+        del self._module_name
+        del self._path
+
+    def load_module(self, module):
+        self._module_name = module.__name__
+        for member_name in dir(module):
+            # ignore the options file
+            if member_name == self._options_name:
+                continue
+
+            # get a handle to the pb2 module descriptor
+            member = getattr(module, member_name)
+
+            # ignore anything that isn't a module from this package
+            if not isinstance(member, types.ModuleType):
+                continue
+            if not member.__name__.startswith(self._package + '.'):
+                continue
+
+            # all proto messages end with _pb2
+            if not member_name.endswith('_pb2'):
+                # ignore any module starting with _
+                if not member_name.startswith('_'):
+                    # load submodules for nested protobufs
+                    self._container[member_name] = ProtobufContainer(self._container.handlers)
+                    stash = (self._container,self._module_name)
+                    self._container = self._container[member_name]
+                    self._path.append(member_name)
+                    self.load_module(member)
+                    self._path.pop()
+                    (self._container,self._module_name) = stash
+                continue
+
+            # iterate over all the message definitions
+            for name in member.DESCRIPTOR.message_types_by_name:
+                message = getattr(member, name)
+                options = message.DESCRIPTOR.GetOptions()
+
+                # subpackage messages get their own tables and routes,
+                # pbt.sub.Tags is table sub_Tags served at /<component>/sub/Tags
+                table_name = '_'.join(self._path + [name])
+                route      = '/'.join(self._path + [name])
+
+                _check_names(message.DESCRIPTOR)
+                if self.db and not options.Extensions[self._options.no_table]:
+                    record_type = self._proto_table(table_name, message=message)
+                else:
+                    # declare the message fields so _as_dict has something to return
+                    fields = message.DESCRIPTOR.fields
+                    members = {f.name:None for f in fields}
+                    members['__annotations__'] = {f.name:typing.Any for f in fields}
+                    members['_ProtoInterface__pb'] = message
+                    members['_proto_children'] = {}
+                    record_type = type(name, (ProtoInterface,), members)
+                    record_type = dataclasses.dataclass(init=False, kw_only=True)(record_type)
+
+                self._container[name] = record_type
+
+                if self.handlers and not options.Extensions[self._options.no_handler]:
+                    self._proto_handler(route, record_type, module.__name__, message.DESCRIPTOR.full_name)
+
+    def _proto_table(self, name, message=None, descriptor=None, parent=None, parent_key=None, parent_col=None, ordered=False):
+        child_tables = {}
+        annotations  = {}
+
+        members = {
+            '__tablename__'   : name,
+            '__annotations__' : annotations,
+            '__module__'      : self._module_name,
+            }
+
+        if message:
+            members['_ProtoInterface__pb'] = message
+
+        if descriptor is None:
+            descriptor = message.DESCRIPTOR
+        _check_names(descriptor)
+
+        primary_key = None
+        # iterate over message to determine columns
+        for field in descriptor.fields:
+            col_options = field.GetOptions()
+
+            # check if column should be primary key
+            is_primary_key = col_options.Extensions[self._options.pkey]
+            if is_primary_key:
+                if primary_key:
+                    raise _.error('Only one primary key can be specified')
+                primary_key = field.name
+
+            # check if column should be unique
+            unique = col_options.Extensions[self._options.uniq]
+
+            if _is_message(field) and is_primary_key:
+                raise _.error('Message type cannot be primary key')
+            if _repeated(field) and is_primary_key:
+                raise _.error('Repeated field cannot be primary key')
+
+            # check for foreign key
+            if col_options.HasExtension(self._options.ref):
+                print('TODO: FKEY:', col_options.Extensions[self._options.ref])
+
+            if _is_message(field) and self.relational and not _is_map(field):
+                # messages become child tables, maps stay JSON
+                ref_table_name = f'{name}_{field.name}'
+                ordering = {}
+                if _repeated(field):
+                    annotations[field.name] = sqlalchemy.orm.Mapped[list[ref_table_name]]
+                    # keep repeated children in the order they were given
+                    ordering = dict(
+                        order_by         = f'{ref_table_name}._position',
+                        collection_class = sqlalchemy.ext.orderinglist.ordering_list('_position'),
+                        )
+                else:
+                    annotations[field.name] = sqlalchemy.orm.Mapped[typing.Optional[ref_table_name]]
+                members[field.name] = sqlalchemy.orm.relationship(
+                    back_populates=name,
+                    lazy='joined',
+                    cascade="all, delete-orphan",
+                    passive_deletes=True,
+                    init=False,
+                    **ordering,
+                    )
+                child_tables[ref_table_name] = field
+                continue
+
+            if _is_message(field):
+                # messages and maps are stored in their proto3 JSON form
+                column_type = (self.db.JSON or sqlalchemy.JSON)(none_as_null=True)
+            else:
+                column_type = self._column_type(field)
+                if _repeated(field):
+                    # TEXT[] rather than VARCHAR[] to match existing tables
+                    if field.type == field.TYPE_STRING:
+                        column_type = sqlalchemy.TEXT()
+                    column_type = self.db.array(column_type)
+
+            annotations[field.name] = sqlalchemy.orm.Mapped[typing.Optional[typing.Any]]
+            members[field.name] = sqlalchemy.orm.mapped_column(
+                column_type,
+                primary_key=is_primary_key,
+                unique=unique,
+                init=False,
+                )
+
+        table_options = descriptor.GetOptions()
+
+        if table_options.HasExtension(self._options.id):
+            primary_key = table_options.Extensions[self._options.id]
+            members[primary_key] = sqlalchemy.orm.mapped_column(
+                sqlalchemy.INTEGER,
+                primary_key=True,
+                autoincrement=True,
+                init=False,
+                )
+        elif primary_key is None:
+            primary_key = f'{name}_id'
+            members[primary_key] = sqlalchemy.orm.mapped_column(
+                sqlalchemy.INTEGER,
+                primary_key=True,
+                autoincrement=True,
+                init=False,
+                )
+
+        if ordered:
+            # unannotated so it stays out of _as_dict, like the foreign key
+            members['_position'] = sqlalchemy.orm.mapped_column(
+                sqlalchemy.INTEGER,
+                init=False,
+                default=None,
+                )
+
+        if parent and parent_key and parent_col:
+            members[f'{parent}_{parent_key}'] = sqlalchemy.orm.mapped_column(
+                sqlalchemy.ForeignKey(f'{parent}.{parent_key}', ondelete='CASCADE'),
+                init=False,
+                )
+            members[f'{parent}'] = sqlalchemy.orm.relationship(
+                parent,
+                back_populates=parent_col,
+                lazy='joined',
+                init=False,
+                )
+
+        members['__primary_key__'] = primary_key
+        members['_proto_children'] = {}
+
+        table_type = type(name, (ProtoInterface,self.db.Base,), members)
+
+        if message:
+            # refuse to store a record that its message can't represent
+            sqlalchemy.event.listen(table_type, 'before_insert', _validate)
+            sqlalchemy.event.listen(table_type, 'before_update', _validate)
+
+        for child_table,field in child_tables.items():
+            child_type = self._proto_table(child_table, descriptor=field.message_type, parent=name, parent_key=primary_key, parent_col=field.name, ordered=_repeated(field))
+            table_type._proto_children[field.name] = child_type
+            setattr(table_type, f'_{field.name}', child_type)
+
+        return table_type
+
+    def _proto_handler(self, route, record_type, module_name, full_name):
+        ## check if a custom handler was defined
+        proto_handler = self._container.handlers.get(full_name)
+        # the default handler needs a table
+        if not proto_handler and not hasattr(record_type, '__table__'):
+            return
+
+        handler_name = route.replace('/', '_')
+        if proto_handler:
+            handler_name = proto_handler.__name__
+            module_name = proto_handler.__module__
+        types = [proto_handler] if proto_handler else [_.records.HandlerInterface]
+        types.append(tornado.web.RequestHandler)
+
+        # served at the message's route, whichever handler class is used
+        record_handler = type(handler_name, tuple(types), {
+            '_component' : route,
+            '_db'        : self.db,
+            '_record'    : record_type,
+            '__module__' : module_name,
+            })
+
+        _.application._record_handler(self.component_name, record_handler)
+
+    def _column_type(self, field):
+        F = google.protobuf.descriptor.FieldDescriptor
+        if field.type in (F.TYPE_DOUBLE, F.TYPE_FLOAT):
+            return sqlalchemy.Float()
+        if field.type == F.TYPE_BOOL:
+            return sqlalchemy.Boolean()
+        if field.type == F.TYPE_STRING:
+            return sqlalchemy.String()
+        if field.type == F.TYPE_BYTES:
+            return (self.db.BYTES or sqlalchemy.LargeBinary)()
+        if field.type in (F.TYPE_UINT64, F.TYPE_FIXED64):
+            return _.databases.UInt64()
+        # the remaining integers and enums all fit in a BIGINT
+        return sqlalchemy.BIGINT()
+
+
+class ProtoInterface(_.records.RecordsInterface):
+    'converts between records and protobuf messages'
+
+    # child table classes by field name, only used in relational mode
+    _proto_children = {}
+
+    def _assign_pb(self, pb):
+        'set the record fields from a protobuf message'
+        json_form = None
+        for field in pb.DESCRIPTOR.fields:
+            child_type = self._proto_children.get(field.name)
+            value = getattr(pb, field.name)
+            if _is_message(field) and not child_type:
+                if json_form is None:
+                    json_form = _json_form(pb)
+                # unset singular messages are left out of the JSON form
+                value = json_form.get(field.name)
+            elif child_type:
+                if _repeated(field):
+                    value = [child_type()._assign_pb(item) for item in value]
+                elif pb.HasField(field.name):
+                    value = child_type()._assign_pb(value)
+                else:
+                    value = None
+            elif _repeated(field):
+                value = list(value)
+            elif field.has_presence and not pb.HasField(field.name):
+                # optional and oneof fields that were not set
+                value = None
+            setattr(self, field.name, value)
+        return self
+
+    def _fill_pb(self, pb):
+        'set the fields of a protobuf message from the record'
+        json_form = {}
+        for field in pb.DESCRIPTOR.fields:
+            value = getattr(self, field.name, None)
+            if value is None:
+                continue
+            if _is_message(field) and not self._proto_children.get(field.name):
+                json_form[field.name] = value
+            elif _is_message(field):
+                if _repeated(field):
+                    for item in value:
+                        item._fill_pb(getattr(pb, field.name).add())
+                else:
+                    getattr(pb, field.name).SetInParent()
+                    value._fill_pb(getattr(pb, field.name))
+            elif _repeated(field):
+                getattr(pb, field.name).extend(_scalar(field, v) for v in value)
+            else:
+                setattr(pb, field.name, _scalar(field, value))
+        if json_form:
+            google.protobuf.json_format.ParseDict(json_form, pb)
+        return pb
+
+    @classmethod
+    def _parse(cls, parse, *args):
+        pb = cls.__pb()
+        try:
+            parse(*args, pb)
+        except (google.protobuf.json_format.ParseError, google.protobuf.message.DecodeError) as e:
+            raise _.error('%s', e) from None
+        return pb
+
+    def __call__(self, *args, **kwds):
+        msg = args[0] if args else kwds
+        return self._assign_pb(self._parse(google.protobuf.json_format.ParseDict, msg))
+
+    @classmethod
+    def _from_dict(cls, *args, **kwds):
+        msg = args[0] if args else kwds
+        return cls()._assign_pb(cls._parse(google.protobuf.json_format.ParseDict, msg))
+
+    @classmethod
+    def _from_json(cls, msg):
+        return cls()._assign_pb(cls._parse(google.protobuf.json_format.Parse, msg))
+
+    @classmethod
+    def _from_binary(cls, packed):
+        pb = cls.__pb()
+        try:
+            pb.ParseFromString(packed)
+        except google.protobuf.message.DecodeError as e:
+            raise _.error('%s', e) from None
+        return cls()._assign_pb(pb)
+
+    @classmethod
+    def _from_pb(cls, pb):
+        return cls()._assign_pb(pb)
+
+    def _as_binary(self):
+        return self._as_pb().SerializeToString()
+
+    def _as_pb(self):
+        try:
+            return self._fill_pb(self.__pb())
+        except (google.protobuf.json_format.ParseError, TypeError, ValueError, KeyError) as e:
+            raise _.error('%s', e) from None
+
+
+def _validate(mapper, connection, record):
+    # raises _.error when a field doesn't fit the message, e.g. an unknown nested key
+    record._as_pb()
+
+
+def _scalar(field, value):
+    # databases can return Decimal for integers and float columns
+    F = google.protobuf.descriptor.FieldDescriptor
+    if field.type in (F.TYPE_DOUBLE, F.TYPE_FLOAT):
+        return float(value)
+    if field.type == F.TYPE_ENUM and isinstance(value, str):
+        return field.enum_type.values_by_name[value].number
+    if field.type not in (F.TYPE_STRING, F.TYPE_BYTES, F.TYPE_BOOL):
+        return int(value)
+    return value
+
+
+class ProtobufContainer(_.Container):
+    def __init__(self, handlers=None):
+        super().__init__()
+        # custom handlers by message full name, shared with subpackage containers
+        self.handlers = {} if handlers is None else handlers
+
+    # decorator for adding custom handlers for message types
+    def handles(self, _message):
+        def wrap(_handler):
+            self.handlers[_message.DESCRIPTOR.full_name] = _handler
+            return _handler
+        return wrap
+
+MessageToDict = _json_form
+
+class ProtoJSONEncoder(_.JSONEncoder):
+    def default(self, obj):
+        if hasattr(obj, 'DESCRIPTOR'):
+            return MessageToDict(obj)
+        return _.JSONEncoder.default(self, obj)
+
+json._default_encoder = ProtoJSONEncoder()
+
+
+if '__main__' == __name__:
+    path = os.path.join(os.path.dirname(__file__), 'Protobuf.proto')
+    protobuf = open(path, 'r').read()
+    print(protobuf)

_/settings.py

@@ -0,0 +1,159 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import argparse
+import collections
+import configparser
+import inspect
+import logging
+import os
+import sys
+import time
+import traceback
+
+import _
+
+
+# override exit to set the stop event
+class ArgParser(argparse.ArgumentParser):
+    def exit(self, status=0, message=None):
+        if message:
+            self._print_message(message, sys.stderr)
+        _.application.stop()
+
+
+_.argparser = ArgParser(add_help=False)
+
+_.argparser.add_argument('--ini', '-I',
+    metavar='<path>',
+    help='Specify additional ini file')
+
+_.config = configparser.ConfigParser(
+    allow_no_value = True,
+    interpolation  = None,
+    )
+_.config.optionxform = str
+
+
+async def load(**kwds):
+    # get the path of the caller
+    try:
+        caller = inspect.getfile(_.application.__class__)
+    except (TypeError,OSError):
+        caller = '.'
+
+    # allow apps to define a name or derive from caller
+    _.name = kwds.get('name', None)
+    if _.name is None:
+        # get the name of the script
+        _.name = os.path.basename(caller)
+        if _.name.endswith('.py'):
+            _.name = _.name[:-3]
+
+    # allow apps to define a namespace
+    _.ns = kwds.get('ns', _.name)
+    if _.ns is None:
+        _.ns = ''
+
+    # get the directory of the script
+    root = kwds.get('root', None)
+    if root is None:
+        root = os.path.dirname(caller)
+    root = os.path.abspath(root)
+
+    _.paths = _.Paths(root=root, ns=_.ns)
+
+    # if ns is not passed in use the supplied or derived ns
+    ini_name = _.name or _.ns
+    ini_dir  = _.ns or _.name
+
+    # allow several locations for ini files
+    ini_files = [
+        _.paths(f'{ini_name}.ini'),
+        _.paths(f'{ini_name}.ini.local'),
+        os.path.join(os.path.sep, 'etc', f'{ini_dir}', f'{ini_name}.ini'),
+        os.path.join(os.path.expanduser('~'), f'.{ini_name}.ini'),
+        ]
+
+    # first pass at parsing args to get additional ini files
+    _.args,remainder = _.argparser.parse_known_args()
+
+    # append a custom ini file if specified
+    if _.args.ini:
+        ini_files.append(_.args.ini)
+
+    # little hack because I like the generic options at the bottom of the help message
+    _.args.debug = '--debug' in remainder or '-D' in remainder
+    logging.basicConfig(
+        format  = '%(asctime)s %(levelname)-8s %(message)s',
+        datefmt = '%Y-%m-%d %H:%M:%S',
+        level   = logging.DEBUG if _.args.debug else logging.INFO,
+        force   = True
+        )
+
+    # load the ini files, at least one file must exist
+    try:
+        if _.args.debug:
+            for ini_file in ini_files:
+                logging.debug('Loading ini file: %s', ini_file)
+        ok = _.config.read(ini_files)
+    except configparser.ParsingError as e:
+        raise _.error('Unable to parse file: %s', e)
+
+    if not ok:
+        logging.warning('Unable to read config file(s):\n  %s', '\n  '.join(ini_files))
+
+    # load the components specified in the ini file
+    try:
+        await _.components.load('databases')
+        await _.components.load('records')
+        await _.components.load('caches')
+        await _.components.load('logins')
+        await _.components.load('supports')
+    except _.error as e:
+        logging.error('%s', e)
+        _.application.stop()
+        return
+
+    # add the generic arguments after any components
+    _.argparser.add_argument('--debug', '-D',
+        action='store_true',
+        help='Log verbose debugging information')
+
+    _.argparser.add_argument('--version', '-V',
+        action='store_true',
+        help='Show version and exit'
+        )
+
+    _.argparser.add_argument('--help', '-h',
+        action='help', default=argparse.SUPPRESS,
+        help='Show help message')
+
+    _.args = _.argparser.parse_args()
+
+    try:
+        # tables are created once every component has defined its models
+        for name,component in _.databases.items():
+            if not _.application._stop_event.is_set():
+                await component.create_tables()
+
+        for name,component in _.records.items():
+            if not _.application._stop_event.is_set():
+                await _.wait(component.args(name))
+
+        for name,component in _.supports.items():
+            if not _.application._stop_event.is_set():
+                await _.wait(component.args(name))
+
+        for name,component in _.logins.items():
+            if not _.application._stop_event.is_set():
+                await _.wait(component.args(name))
+    except _.error as e:
+        logging.error('%s', e)
+        _.application.stop()
+        return

_/supports/__init__.py

@@ -0,0 +1,28 @@
+#
+# (c) 2015-2023 Matthew Shaw
+#
+# Authors
+# =======
+# Matthew Shaw <mshaw.cx@gmail.com>
+#
+
+import os
+
+import _
+
+
+class Support:
+    @classmethod
+    async def _(cls, component_name, **kwds):
+        self = cls()
+        _.supports[component_name] = self
+        self.root = os.path.dirname(__file__)
+        self.root = os.path.abspath(self.root)
+        await self.init(component_name, **kwds)
+
+    async def init(self, component_name, **kwds):
+        pass
+
+    @classmethod
+    async def args(cls, component_name):
+        pass