crossplane-function-pythonic 0.0.7a0__py3-none-any.whl → 0.0.7b0__py3-none-any.whl

crossplane/pythonic/function.py

@@ -0,0 +1,260 @@
+"""A Crossplane composition function."""
+
+import asyncio
+import base64
+import builtins
+import importlib
+import inspect
+import logging
+import sys
+
+import grpc
+import crossplane.function.response
+from crossplane.function.proto.v1 import run_function_pb2 as fnv1
+from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1
+from .. import pythonic
+
+builtins.BaseComposite = pythonic.BaseComposite
+builtins.append = pythonic.append
+builtins.Map = pythonic.Map
+builtins.List = pythonic.List
+builtins.Unknown = pythonic.Unknown
+builtins.Yaml = pythonic.Yaml
+builtins.Json = pythonic.Json
+builtins.B64Encode = pythonic.B64Encode
+builtins.B64Decode = pythonic.B64Decode
+
+logger = logging.getLogger(__name__)
+
+
+class FunctionRunner(grpcv1.FunctionRunnerService):
+    """A FunctionRunner handles gRPC RunFunctionRequests."""
+
+    def __init__(self, debug=False):
+        """Create a new FunctionRunner."""
+        self.debug = debug
+        self.clazzes = {}
+
+    def invalidate_module(self, module):
+        self.clazzes.clear()
+        if module in sys.modules:
+            del sys.modules[module]
+        importlib.invalidate_caches()
+
+    async def RunFunction(
+        self, request: fnv1.RunFunctionRequest, _: grpc.aio.ServicerContext
+    ) -> fnv1.RunFunctionResponse:
+        try:
+            return await self.run_function(request)
+        except:
+            logger.exception('Exception thrown in run fuction')
+            raise
+
+    async def run_function(self, request):
+        composite = request.observed.composite.resource
+        name = list(reversed(composite['apiVersion'].split('/')[0].split('.')))
+        name.append(composite['kind'])
+        name.append(composite['metadata']['name'])
+        logger = logging.getLogger('.'.join(name))
+        if 'iteration' in request.context:
+            request.context['iteration'] = request.context['iteration'] + 1
+        else:
+            request.context['iteration'] = 1
+        logger.debug(f"Starting compose, {ordinal(request.context['iteration'])} pass")
+
+        response = crossplane.function.response.to(request)
+
+        if composite['apiVersion'] == 'pythonic.fortra.com/v1alpha1' and composite['kind'] == 'Composite':
+            if 'composite' not in composite['spec']:
+                logger.error('Missing spec "composite"')
+                crossplane.function.response.fatal(response, 'Missing spec "composite"')
+                return response
+            composite = composite['spec']['composite']
+        else:
+            if 'composite' not in request.input:
+                logger.error('Missing input "composite"')
+                crossplane.function.response.fatal(response, 'Missing input "composite"')
+                return response
+            composite = request.input['composite']
+
+        clazz = self.clazzes.get(composite)
+        if not clazz:
+            if '\n' in composite:
+                module = Module()
+                try:
+                    exec(composite, module.__dict__)
+                except Exception as e:
+                    logger.exception('Exec exception')
+                    crossplane.function.response.fatal(response, f"Exec exception: {e}")
+                    return response
+                composite = ['<script>', 'Composite']
+            else:
+                composite = composite.rsplit('.', 1)
+                if len(composite) == 1:
+                    logger.error(f"Composite class name does not include module: {composite[0]}")
+                    crossplane.function.response.fatal(response, f"Composite class name does not include module: {composite[0]}")
+                    return response
+                try:
+                    module = importlib.import_module(composite[0])
+                except Exception as e:
+                    logger.error(str(e))
+                    crossplane.function.response.fatal(response, f"Import module exception: {e}")
+                    return response
+            clazz = getattr(module, composite[1], None)
+            if not clazz:
+                logger.error(f"{composite[0]} did not define: {composite[1]}")
+                crossplane.function.response.fatal(response, f"{composite[0]} did not define: {composite[1]}")
+                return response
+            composite = '.'.join(composite)
+            if not inspect.isclass(clazz):
+                logger.error(f"{composite} is not a class")
+                crossplane.function.response.fatal(response, f"{composite} is not a class")
+                return response
+            if not issubclass(clazz, BaseComposite):
+                logger.error(f"{composite} is not a subclass of BaseComposite")
+                crossplane.function.response.fatal(response, f"{composite} is not a subclass of BaseComposite")
+                return response
+            self.clazzes[composite] = clazz
+
+        try:
+            composite = clazz(request, response, logger)
+        except Exception as e:
+            logger.exception('Instatiate exception')
+            crossplane.function.response.fatal(response, f"Instatiate exception: {e}")
+            return response
+
+        try:
+            result = composite.compose()
+            if asyncio.iscoroutine(result):
+                await result
+        except Exception as e:
+            logger.exception('Compose exception')
+            crossplane.function.response.fatal(response, f"Compose exception: {e}")
+            return response
+
+        requested = []
+        for name, required in composite.requireds:
+            if required.apiVersion and required.kind:
+                r = Map(apiVersion=required.apiVersion, kind=required.kind)
+                if required.namespace:
+                    r.namespace = required.namespace
+                if required.matchName:
+                    r.matchName = required.matchName
+                for key, value in required.matchLabels:
+                    r.matchLabels[key] = value
+                if r != composite.context._requireds[name]:
+                    composite.context._requireds[name] = r
+                    requested.append(name)
+        if requested:
+            logger.info(f"Requireds requested: {','.join(requested)}")
+            return response
+
+        unknownResources = []
+        warningResources = []
+        fatalResources = []
+        for name, resource in sorted(entry for entry in composite.resources):
+            unknowns = resource.desired._getUnknowns
+            if unknowns:
+                unknownResources.append(name)
+                warning = False
+                fatal = False
+                if resource.observed:
+                    warningResources.append(name)
+                    warning = True
+                    if resource.unknownsFatal or (resource.unknownsFatal is None and composite.unknownsFatal):
+                        fatalResources.append(name)
+                        fatal = True
+                if self.debug:
+                    for destination, source in sorted(unknowns.items()):
+                        destination = self.trimFullName(destination)
+                        source = self.trimFullName(source)
+                        if fatal:
+                            logger.error(f'Observed unknown: {destination} = {source}')
+                        elif warning:
+                            logger.warning(f'Observed unknown: {destination} = {source}')
+                        else:
+                            logger.debug(f'Desired unknown: {destination} = {source}')
+                if resource.observed:
+                    resource.desired._patchUnknowns(resource.observed)
+                else:
+                    del composite.resources[name]
+
+        if fatalResources:
+            level = logger.error
+            reason = 'FatalUnknowns'
+            message = f"Observed resources with unknowns: {','.join(fatalResources)}"
+            status = False
+            event = composite.events.fatal
+        elif warningResources:
+            level = logger.warning
+            reason = 'ObservedUnknowns'
+            message = f"Observed resources with unknowns: {','.join(warningResources)}"
+            status = False
+            event = composite.events.warning
+        elif unknownResources:
+            level = logger.info
+            reason = 'DesiredUnknowns'
+            message = f"Desired resources with unknowns: {','.join(unknownResources)}"
+            status = False
+            event = composite.events.info
+        else:
+            level = None
+            reason = 'AllComposed'
+            message = 'All resources are composed'
+            status = True
+            event = None
+        if not self.debug and level:
+            level(message)
+        composite.conditions.ResourcesComposed(reason, message, status)
+        if event:
+            event(reason, message)
+
+        for name, resource in composite.resources:
+            if resource.autoReady or (resource.autoReady is None and composite.autoReady):
+                if resource.ready is None:
+                    if resource.conditions.Ready.status:
+                        resource.ready = True
+
+        logger.info('Completed compose')
+        return response
+
+    def trimFullName(self, name):
+        name = name.split('.')
+        for values in (
+                ('request', 'observed', 'resources', None, 'resource'),
+                ('request', 'extra_resources', None, 'items', 'resource'),
+                ('response', 'desired', 'resources', None, 'resource'),
+        ):
+            if len(values) <= len(name):
+                for ix, value in enumerate(values):
+                    if value and value != name[ix] and not name[ix].startswith(f"{value}["):
+                       break
+                else:
+                    ix = 0
+                    for value in values:
+                        if value:
+                            if value == name[ix]:
+                                del name[ix]
+                            elif ix:
+                                name[ix-1] += name[ix][len(value):]
+                                del name[ix]
+                            else:
+                                name[ix] = name[ix][len(value):]
+                                ix += 1
+                        else:
+                            ix += 1
+                    break
+        return '.'.join(name)
+
+
+def ordinal(ix):
+    ix = int(ix)
+    if 11 <= (ix % 100) <= 13:
+        suffix = 'th'
+    else:
+        suffix = ['th', 'st', 'nd', 'rd', 'th'][min(ix % 10, 4)]
+    return str(ix) + suffix
+
+
+class Module:
+    pass

crossplane/pythonic/main.py

@@ -0,0 +1,187 @@
+"""The composition function's main CLI."""
+
+import argparse
+import asyncio
+import logging
+import os
+import pathlib
+import shlex
+import signal
+import sys
+import traceback
+
+import crossplane.function.logging
+import crossplane.function.proto.v1.run_function_pb2_grpc as grpcv1
+import grpc
+import pip._internal.cli.main
+
+from . import function
+
+
+def main():
+    try:
+        asyncio.run(Main().main())
+    except:
+        traceback.print_exc()
+        sys.exit(1)
+
+
+class Main:
+    async def main(self):
+        parser = argparse.ArgumentParser('Forta Crossplane Function')
+        parser.add_argument(
+            '--debug', '-d',
+            action='store_true',
+            help='Emit debug logs.',
+        )
+        parser.add_argument(
+            '--log-name-width',
+            type=int,
+            default=40,
+            help='Width of the logger name in the log output, default 40',
+        )
+        parser.add_argument(
+            '--address',
+            default='0.0.0.0:9443',
+            help='Address at which to listen for gRPC connections, default: 0.0.0.0:9443',
+        )
+        parser.add_argument(
+            '--tls-certs-dir',
+            default=os.getenv('TLS_SERVER_CERTS_DIR'),
+            help='Serve using mTLS certificates.',
+        )
+        parser.add_argument(
+            '--insecure',
+            action='store_true',
+            help='Run without mTLS credentials. If you supply this flag --tls-certs-dir will be ignored.',
+        )
+        parser.add_argument(
+            '--packages',
+            action='store_true',
+            help='Discover python packages from function-pythonic ConfigMaps.'
+        )
+        parser.add_argument(
+            '--packages-secrets',
+            action='store_true',
+            help='Also Discover python packages from function-pythonic Secrets.'
+        )
+        parser.add_argument(
+            '--packages-namespace',
+            action='append',
+            default=[],
+            help='Namespaces to discover function-pythonic ConfigMaps and Secrets in, default is cluster wide.',
+        )
+        parser.add_argument(
+            '--packages-dir',
+            default='./pythonic-packages',
+            help='Directory to store discovered function-pythonic ConfigMaps and Secrets to, defaults "<cwd>/pythonic-packages"'
+        )
+        parser.add_argument(
+            '--pip-install',
+            help='Pip install command to install additional Python packages.'
+        )
+        parser.add_argument(
+            '--python-path',
+            action='append',
+            default=[],
+            help='Filing system directories to add to the python path',
+        )
+        parser.add_argument(
+            '--allow-oversize-protos',
+            action='store_true',
+            help='Allow oversized protobuf messages'
+        )
+        args = parser.parse_args()
+
+        self.configure_logging(args)
+
+        if args.pip_install:
+            pip._internal.cli.main.main(['install', *shlex.split(args.pip_install)])
+
+        # enables read only volumes or mismatched uid volumes
+        sys.dont_write_bytecode = True
+        for path in reversed(args.python_path):
+            sys.path.insert(0, str(pathlib.Path(path).resolve()))
+
+        if args.allow_oversize_protos:
+            from google.protobuf.internal import api_implementation
+            if api_implementation._c_module:
+                api_implementation._c_module.SetAllowOversizeProtos(True)
+
+        grpc.aio.init_grpc_aio()
+        grpc_runner = function.FunctionRunner(args.debug)
+        grpc_server = grpc.aio.server()
+        grpcv1.add_FunctionRunnerServiceServicer_to_server(grpc_runner, grpc_server)
+        if args.tls_certs_dir:
+            certs = pathlib.Path(args.tls_certs_dir)
+            grpc_server.add_secure_port(
+                args.address,
+                grpc.ssl_server_credentials(
+                    private_key_certificate_chain_pairs=[(
+                        (certs / 'tls.key').read_bytes(),
+                        (certs / 'tls.crt').read_bytes(),
+                    )],
+                    root_certificates=(certs / 'ca.crt').read_bytes(),
+                    require_client_auth=True,
+                ),
+            )
+        else:
+            if not args.insecure:
+                raise ValueError('Either --tls-certs-dir or --insecure must be specified')
+            grpc_server.add_insecure_port(args.address)
+        await grpc_server.start()
+
+        if args.packages:
+            from . import packages
+            async with asyncio.TaskGroup() as tasks:
+                tasks.create_task(grpc_server.wait_for_termination())
+                tasks.create_task(packages.operator(
+                    args.packages_secrets,
+                    args.packages_namespace,
+                    args.packages_dir,
+                    grpc_server,
+                    grpc_runner,
+                ))
+        else:
+            def stop():
+                asyncio.ensure_future(grpc_server.stop(5))
+            loop = asyncio.get_event_loop()
+            loop.add_signal_handler(signal.SIGINT, stop)
+            loop.add_signal_handler(signal.SIGTERM, stop)
+            await grpc_server.wait_for_termination()
+
+    def configure_logging(self, args):
+        formatter = Formatter(args.log_name_width)
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        logger = logging.getLogger()
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+
+
+class Formatter(logging.Formatter):
+    def __init__(self, name_width):
+        super(Formatter, self).__init__(
+            f"[{{asctime}}.{{msecs:03.0f}}] {{sname:{name_width}.{name_width}}} [{{levelname:8.8}}] {{message}}",
+            '%Y-%m-%d %H:%M:%S',
+            '{',
+        )
+        self.name_width = name_width
+
+    def format(self, record):
+        record.sname = record.name
+        extra = len(record.sname) - self.name_width
+        if extra > 0:
+            names = record.sname.split('.')
+            for ix, name in enumerate(names):
+                if len(name) > extra:
+                    names[ix] = name[extra:]
+                    break
+                names[ix] = name[:1]
+                extra -= len(name) - 1
+            record.sname = '.'.join(names)
+        return super(Formatter, self).format(record)
+
+
+if __name__ == '__main__':
+    main()

crossplane/pythonic/packages.py

@@ -0,0 +1,158 @@
+
+import base64
+import importlib
+import logging
+import pathlib
+import sys
+
+import kopf
+
+
+PACKAGE_LABEL = {'function-pythonic.package': kopf.PRESENT}
+PACKAGES_DIR = None
+GRPC_SERVER = None
+GRPC_RUNNER = None
+
+
+def operator(packages_secrets, packages_namespace, packages_dir, grpc_server, grpc_runner):
+    logging.getLogger('kopf.objects').setLevel(logging.INFO)
+    global PACKAGES_DIR, GRPC_SERVER, GRPC_RUNNER
+    PACKAGES_DIR = packages_dir
+    GRPC_SERVER = grpc_server
+    GRPC_RUNNER = grpc_runner
+    PACKAGES_DIR = pathlib.Path(packages_dir).expanduser().resolve()
+    sys.path.insert(0, str(PACKAGES_DIR))
+    if packages_secrets:
+        kopf.on.create('', 'v1', 'secrets', labels=PACKAGE_LABEL)(create)
+        kopf.on.resume('', 'v1', 'secrets', labels=PACKAGE_LABEL)(create)
+        kopf.on.update('', 'v1', 'secrets', labels=PACKAGE_LABEL)(update)
+        kopf.on.delete('', 'v1', 'secrets', labels=PACKAGE_LABEL)(delete)
+    return kopf.operator(
+        standalone=True,
+        clusterwide=not packages_namespace,
+        namespaces=packages_namespace,
+    )
+
+
+@kopf.on.startup()
+async def startup(settings, **_):
+    settings.scanning.disabled = True
+
+
+@kopf.on.cleanup()
+async def cleanup(**_):
+    await GRPC_SERVER.stop(5)
+
+
+@kopf.on.create('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
+@kopf.on.resume('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
+async def create(body, logger, **_):
+    package_dir, package = get_package_dir(body)
+    if package_dir:
+        package_dir.mkdir(parents=True, exist_ok=True)
+        secret = body['kind'] == 'Secret'
+        invalidate = False
+        for name, text in body.get('data', {}).items():
+            package_file = package_dir / name
+            if secret:
+                package_file.write_bytes(base64.b64decode(text.encode('utf-8')))
+            else:
+                package_file.write_text(text)
+            if package_file.suffixes == ['.py']:
+                module = '.'.join(package + [package_file.stem])
+                GRPC_RUNNER.invalidate_module(module)
+                logger.info(f"Created module: {module}")
+            else:
+                logger.info(f"Created file: {'/'.join(package + [name])}")
+
+
+@kopf.on.update('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
+async def update(body, old, logger, **_):
+    old_package_dir, old_package = get_package_dir(old)
+    if old_package_dir:
+        old_data = old.get('data', {})
+    else:
+        old_data = {}
+    old_names = set(old_data.keys())
+    package_dir, package = get_package_dir(body, logger)
+    if package_dir:
+        package_dir.mkdir(parents=True, exist_ok=True)
+        secret = body['kind'] == 'Secret'
+        for name, text in body.get('data', {}).items():
+            package_file = package_dir / name
+            if package_dir == old_package_dir and text == old_data.get(name, None):
+                action = 'Unchanged'
+            else:
+                if secret:
+                    package_file.write_bytes(base64.b64decode(text.encode('utf-8')))
+                else:
+                    package_file.write_text(text)
+                action = 'Updated' if package_dir == old_package_dir and name in old_names else 'Created'
+            if package_file.suffixes == ['.py']:
+                module = '.'.join(package + [package_file.stem])
+                if action != 'Unchanged':
+                    GRPC_RUNNER.invalidate_module(module)
+                logger.info(f"{action} module: {module}")
+            else:
+                logger.info(f"{action} file: {'/'.join(package + [name])}")
+            if package_dir == old_package_dir:
+                old_names.discard(name)
+    if old_package_dir:
+        for name in old_names:
+            package_file = old_package_dir / name
+            package_file.unlink(missing_ok=True)
+            if package_file.suffixes == ['.py']:
+                module = '.'.join(old_package + [package_file.stem])
+                GRPC_RUNNER.invalidate_module(module)
+                logger.info(f"Removed module: {module}")
+            else:
+                logger.info(f"Removed file: {'/'.join(old_package + [name])}")
+        while old_package and old_package_dir.is_dir() and not list(old_package_dir.iterdir()):
+            old_package_dir.rmdir()
+            module = '.'.join(old_package)
+            GRPC_RUNNER.invalidate_module(module)
+            logger.info(f"Removed package: {module}")
+            old_package_dir = old_package_dir.parent
+            old_package.pop()
+
+
+@kopf.on.delete('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
+async def delete(old, logger, **_):
+    package_dir, package = get_package_dir(old)
+    if package_dir:
+        for name in old.get('data', {}).keys():
+            package_file = package_dir / name
+            package_file.unlink(missing_ok=True)
+            if package_file.suffixes == ['.py']:
+                module = '.'.join(package + [package_file.stem])
+                GRPC_RUNNER.invalidate_module(module)
+                logger.info(f"Deleted module: {module}")
+            else:
+                logger.info(f"Deleted file: {'/'.join(package + [name])}")
+        while package and package_dir.is_dir() and not list(package_dir.iterdir()):
+            package_dir.rmdir()
+            module = '.'.join(package)
+            GRPC_RUNNER.invalidate_module(module)
+            logger.info(f"Deleted package: {module}")
+            package_dir = package_dir.parent
+            package.pop()
+
+
+def get_package_dir(body, logger=None):
+    package = body.get('metadata', {}).get('labels', {}).get('function-pythonic.package', None)
+    if package is None:
+        if logger:
+            logger.error('function-pythonic.package label is missing')
+        return None, None
+    package_dir = PACKAGES_DIR
+    if package == '':
+        package = []
+    else:
+        package = package.split('.')
+        for segment in package:
+            if not segment.isidentifier():
+                if logger:
+                    logger.error('Package has invalid package name: %s', package)
+                return None, None
+            package_dir = package_dir / segment
+    return package_dir, package