crossplane-function-pythonic 0.5.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

crossplane/pythonic/__about__.py

@@ -1,2 +1,2 @@
 # This is set at build time, using "hatch version"
-__version__ = "0.5.0"
+__version__ = "0.6.1"

crossplane/pythonic/composite.py

@@ -1,10 +1,12 @@
 
 import datetime
 from google.protobuf.duration_pb2 import Duration
-from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 
-from . import auto_ready
-from . import protobuf
+from . import (
+    auto_ready,
+    protobuf,
+)
+from .proto.v1 import run_function_pb2 as fnv1
 
 
 _notset = object()
@@ -104,11 +106,13 @@ class BaseComposite:
         )
         self.response = protobuf.Message(None, 'response', response.DESCRIPTOR, response)
         self.logger = logger
+        self.capabilities = Capabilities(self.request.meta.capabilities)
         self.parameters = self.request.input.parameters
         self.credentials = Credentials(self.request)
         self.context = self.response.context
         self.environment = self.context['apiextensions.crossplane.io/environment']
         self.requireds = Requireds(self)
+        self.schemas = Schemas(self)
         self.resources = Resources(self)
         self.autoReady = True
         self.usages = False
@@ -123,6 +127,7 @@ class BaseComposite:
         self.metadata = self.observed.metadata
         self.spec = self.observed.spec
         self.status = self.desired.status
+        self.output = self.response.output
         self.conditions = Conditions(observed, self.response)
         self.results = Results(self.response)
         self.events = Results(self.response) # Deprecated, use self.results
@@ -136,6 +141,30 @@ class BaseComposite:
         raise NotImplementedError()
 
 
+class Capabilities:
+    def __init__(self, capabilities):
+        self._capabilities = capabilities
+
+    def __bool__(self):
+        return fnv1.CAPABILITY_CAPABILITIES in self._capabilities
+
+    @property
+    def requireds(self):
+        return fnv1.CAPABILITY_REQUIRED_RESOURCES in self._capabilities if self else None
+
+    @property
+    def credentials(self):
+        return fnv1.CAPABILITY_CREDENTIALS in self._capabilities if self else None
+
+    @property
+    def conditions(self):
+        return fnv1.CAPABILITY_CONDITIONS in self._capabilities if self else None
+
+    @property
+    def schemas(self):
+        return fnv1.CAPABILITY_REQUIRED_SCHEMAS in self._capabilities if self else None
+
+
 class Credentials:
     def __init__(self, request):
         self.__dict__['_request'] = request
@@ -558,6 +587,131 @@ class RequiredResource:
         return bool(self.observed)
 
 
+class Schemas:
+    def __init__(self, composite):
+        self._composite = composite
+        self._cache = {}
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        schema = self._cache.get(key)
+        if not schema:
+            schema = Schema(self._composite, key)
+            self._cache[key] = schema
+        return schema
+
+    def __bool__(self):
+        return bool(len(self))
+
+    def __len__(self):
+        names = set()
+        for name, schema in self._composite.request.required_schemas:
+            names.add(name)
+        for name, selector in self._composite.response.requirements.schemas:
+            names.add(name)
+        return len(names)
+
+    def __contains__(self, key):
+        if key in self._composite.request.required_schemas:
+            return True
+        if key in self._composite.response.requirements.schemas:
+            return True
+        return False
+
+    def __iter__(self):
+        names = set()
+        for name, schema in self._composite.request.required_schemas:
+            names.add(name)
+        for name, selector in self._composite.response.requirements.schemas:
+            names.add(name)
+        for name in sorted(names):
+            yield name, self[name]
+
+
+class Schema:
+    def __init__(self, composite, name):
+        self.name = name
+        self._selector = composite.response.requirements.schemas[name]
+        self._schema = composite.request.required_schemas[name].openapi_v3
+
+    def __call__(self, kind=_notset, apiVersion=_notset):
+        self._selector()
+        if kind != _notset:
+            # Allow for apiVersion in the first arg and kind in the second arg
+            if '/' in kind or kind == 'v1':
+                if apiVersion != _notset:
+                    self.kind = apiVersion
+                apiVersion = kind
+            else:
+                self.kind = kind
+        if apiVersion != _notset:
+            self.apiVersion = apiVersion
+        return self
+
+    @property
+    def apiVersion(self):
+        return self._selector.api_version
+
+    @apiVersion.setter
+    def apiVersion(self, apiVersion):
+        self._selector.api_version = apiVersion
+
+    @property
+    def kind(self):
+        return self._selector.kind
+
+    @kind.setter
+    def kind(self, kind):
+        self._selector.kind = kind
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __aenter__(self):
+        return self
+
+    def __aexit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        return self._schema[key]
+
+    def __bool__(self):
+        return bool(self._schema)
+
+    def __len__(self):
+        return len(self._schema)
+
+    def __contains__(self, item):
+        return item in self._schema
+
+    def __iter__(self):
+        for key, value in self._schema:
+            yield key, value
+
+    def __hash__(self):
+        return hash(self._schema)
+
+    def __eq__(self, other):
+        if instance(other, Schema):
+            other = other._schema
+        return self._schema == other
+
+    def __str__(self):
+        return str(self._schema)
+
+    def __format__(self, spec='yaml'):
+        return format(self,_schema, spec)
+
+
 class Conditions:
     def __init__(self, observed, response=None):
         self._observed = observed

crossplane/pythonic/function.py

@@ -7,8 +7,8 @@ import logging
 import sys
 
 import grpc
-from crossplane.function.proto.v1 import run_function_pb2 as fnv1
-from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1
+from .proto.v1 import run_function_pb2 as fnv1
+from .proto.v1 import run_function_pb2_grpc as grpcv1
 from .. import pythonic
 
 logger = logging.getLogger(__name__)
@@ -121,8 +121,13 @@ class FunctionRunner(grpcv1.FunctionRunnerService):
         except Exception as e:
             return self.fatal(request, logger, 'Compose', e)
 
-        if requireds := self.get_requireds(step, composite):
-            logger.debug(f"Requireds requested: {','.join(requireds)}")
+        schemas = self.get_schemas(step, composite)
+        requireds = self.get_requireds(step, composite)
+        if schemas or requireds:
+            if schemas:
+                logger.debug(f"Required schemas: {','.join(schemas)}")
+            if requireds:
+                logger.debug(f"Required resources: {','.join(requireds)}")
         else:
             self.process_usages(composite)
             self.process_unknowns(composite)
@@ -155,11 +160,21 @@ class FunctionRunner(grpcv1.FunctionRunnerService):
             ]
         )
 
+    def get_schemas(self, step, composite):
+        schemas = []
+        for name, schema in composite.schemas:
+            if len(schema.kind) and len(schema.apiVersion):
+                s = pythonic.Map(kind=schema.kind, apiVersion=schema.apiVersion)
+                if s != step.schemas[name]:
+                    step.schemas[name] = s
+                    schemas.append(name)
+        return schemas
+
     def get_requireds(self, step, composite):
         requireds = []
         for name, required in composite.requireds:
-            if len(required.apiVersion) and len(required.kind):
-                r = pythonic.Map(apiVersion=required.apiVersion, kind=required.kind)
+            if len(required.kind) and len(required.apiVersion):
+                r = pythonic.Map(kind=required.kind, apiVersion=required.apiVersion)
                 if len(required.namespace):
                     r.namespace = required.namespace
                 if len(required.matchName):

crossplane/pythonic/grpc.py

@@ -7,7 +7,6 @@ import shlex
 import signal
 import sys
 
-import crossplane.function.proto.v1.run_function_pb2_grpc as grpcv1
 import grpc
 
 from . import (
@@ -15,6 +14,8 @@ from . import (
     command,
     function,
 )
+from .proto.v1 import run_function_pb2_grpc as grpcv1
+
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +46,12 @@ class Command(command.Command):
         parser.add_argument(
             '--packages',
             action='store_true',
+            dest='packages_configmaps',
+            help='Discover python packages from function-pythonic ConfigMaps, deprecated use --packages-configmaps'
+        )
+        parser.add_argument(
+            '--packages-configmaps',
+            action='store_true',
             help='Discover python packages from function-pythonic ConfigMaps.'
         )
         parser.add_argument(
@@ -57,7 +64,17 @@ class Command(command.Command):
             action='append',
             default=[],
             metavar='NAMESPACE',
-            help='Namespaces to discover function-pythonic ConfigMaps in, default is cluster wide.',
+            help='Namespaces to discover function-pythonic ConfigMaps and Secrets in, default is cluster wide.',
+        )
+        parser.add_argument(
+            '--packages-environmentconfigs',
+            action='store_true',
+            help='Also Discover python packages from function-pythonic EnvironmentConfigs.'
+        )
+        parser.add_argument(
+            '--packages-compositions',
+            action='store_true',
+            help='Also Discover python packages from function-pythonic Compositions.'
         )
         parser.add_argument(
             '--packages-dir',
@@ -75,7 +92,9 @@ class Command(command.Command):
         if not self.args.tls_certs_dir and not self.args.insecure:
             print('Either --tls-certs-dir or --insecure must be specified', file=sys.stderr)
             sys.exit(1)
-
+        if (self.args.packages_environmentconfigs or self.args.packages_compositions) and self.args.packages_namespace:
+            print('--packages-namespace cannot be used with --packages-environment-configs or --packages-compositions', file=sys.stderr)
+            sys.exit(1)
         if self.args.pip_install:
             import pip._internal.cli.main
             pip._internal.cli.main.main(['install', '--user', *shlex.split(self.args.pip_install)])
@@ -108,15 +127,18 @@ class Command(command.Command):
             )
         await grpc_server.start()
 
-        if self.args.packages:
+        if self.args.packages_configmaps or self.args.packages_secrets or self.args.packages_environmentconfigs or self.args.packages_compositions:
             from . import packages
             async with asyncio.TaskGroup() as tasks:
                 tasks.create_task(grpc_server.wait_for_termination())
                 tasks.create_task(packages.operator(
                     grpc_server,
                     grpc_runner,
+                    self.args.packages_configmaps,
                     self.args.packages_secrets,
                     self.args.packages_namespace,
+                    self.args.packages_environmentconfigs,
+                    self.args.packages_compositions,
                     self.args.packages_dir,
                 ))
         else:

crossplane/pythonic/packages.py

@@ -10,27 +10,38 @@ import kopf
 GRPC_SERVER = None
 GRPC_RUNNER = None
 PACKAGES_DIR = None
-PACKAGE_LABEL = {'function-pythonic.package': kopf.PRESENT}
+PACKAGE_LABEL = 'function-pythonic.package'
+PACKAGE_LABELS = {PACKAGE_LABEL: kopf.PRESENT}
 
 
-def operator(grpc_server, grpc_runner, packages_secrets, packages_namespaces, packages_dir):
+def operator(grpc_server, grpc_runner, packages_configmaps, packages_secrets, packages_namespaces, packages_environmentconfigs, packages_compositions, packages_dir):
     logging.getLogger('kopf.objects').setLevel(logging.INFO)
     global GRPC_SERVER, GRPC_RUNNER, PACKAGES_DIR
     GRPC_SERVER = grpc_server
     GRPC_RUNNER = grpc_runner
     PACKAGES_DIR = pathlib.Path(packages_dir).expanduser().resolve()
     sys.path.insert(0, str(PACKAGES_DIR))
+    if packages_configmaps:
+        on_resource('', 'v1', 'configmaps')
     if packages_secrets:
-        kopf.on.create('', 'v1', 'secrets', labels=PACKAGE_LABEL)(create)
-        kopf.on.resume('', 'v1', 'secrets', labels=PACKAGE_LABEL)(create)
-        kopf.on.update('', 'v1', 'secrets', labels=PACKAGE_LABEL)(update)
-        kopf.on.delete('', 'v1', 'secrets', labels=PACKAGE_LABEL)(delete)
+        on_resource('', 'v1', 'secrets')
+    if not packages_namespaces:
+        if packages_environmentconfigs:
+            on_resource('apiextensions.crossplane.io', 'v1beta1', 'environmentconfigs')
+        if packages_compositions:
+            on_resource('apiextensions.crossplane.io', 'v1', 'compositions')
     return kopf.operator(
         standalone=True,
         clusterwide=not packages_namespaces,
         namespaces=packages_namespaces,
     )
 
+def on_resource(group, version, plural):
+    kopf.on.create(group, version, plural, labels=PACKAGE_LABELS)(create)
+    kopf.on.resume(group, version, plural, labels=PACKAGE_LABELS)(create)
+    kopf.on.update(group, version, plural, labels=PACKAGE_LABELS)(update)
+    kopf.on.delete(group, version, plural, labels=PACKAGE_LABELS)(delete)
+
 
 @kopf.on.startup()
 async def startup(settings, **_):
@@ -42,107 +53,128 @@ async def cleanup(**_):
     await GRPC_SERVER.stop(5)
 
 
-@kopf.on.create('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
-@kopf.on.resume('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
-async def create(body, logger, **_):
-    package_dir = get_package_dir(body, logger)
-    if package_dir:
-        secret = body['kind'] == 'Secret'
-        for name, text in body.get('data', {}).items():
-            package_file_write(package_dir, name, secret, text, 'Created', logger)
-
-
-@kopf.on.update('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
-async def update(body, old, logger, **_):
-    old_package_dir = get_package_dir(old)
-    if old_package_dir:
-        old_data = old.get('data', {})
-    else:
-        old_data = {}
-    old_names = set(old_data.keys())
-    package_dir = get_package_dir(body, logger)
-    if package_dir:
-        secret = body['kind'] == 'Secret'
-        for name, text in body.get('data', {}).items():
-            if package_dir == old_package_dir and text == old_data.get(name, None):
-                action = 'Unchanged'
-            else:
-                action = 'Updated' if package_dir == old_package_dir and name in old_names else 'Created'
-            package_file_write(package_dir, name, secret, text, action, logger)
-            if package_dir == old_package_dir:
-                old_names.discard(name)
-    if old_package_dir:
-        for name in old_names:
-            package_file_unlink(old_package_dir, name, 'Removed', logger)
-
-
-@kopf.on.delete('', 'v1', 'configmaps', labels=PACKAGE_LABEL)
-async def delete(old, logger, **_):
-    package_dir = get_package_dir(old)
-    if package_dir:
-        for name in old.get('data', {}).keys():
-            package_file_unlink(package_dir, name, 'Deleted', logger)
-
-
-def get_package_dir(body, logger=None):
-    package = body.get('metadata', {}).get('labels', {}).get('function-pythonic.package', None)
+async def create(resource, labels, body, logger, **_):
+    resource_create(resource, labels, 'Created', body, logger)
+
+
+async def update(resource, labels, body, old, logger, **_):
+    resource_delete(resource, labels, 'Removed', old, logger)
+    resource_create(resource, labels, 'Added', body, logger)
+
+
+async def delete(resource, labels, body, logger, **_):
+    resource_delete(resource, labels, 'Deleted', body, logger)
+
+
+def resource_create(resource, labels, action, body, logger):
+    package_dir = resource_package_dir(resource, labels, logger)
+    if not package_dir:
+        return
+    if resource.plural in ('configmaps', 'secrets', 'environmentconfigs'):
+        package_create(resource, action, package_dir, body.get('data', {}), logger)
+    elif resource.plural == 'compositions':
+        for step in body.get('spec', {}).get('pipeline', []):
+            input = step.get('input')
+            if input and input.get('apiVersion') == 'pythonic.fn.crossplane.io/v1alpha1':
+                package_create(resource, action, package_dir, input.get('packages', {}), logger)
+
+
+def resource_delete(resource, labels, action, body, logger):
+    package_dir = resource_package_dir(resource, labels, logger)
+    if not package_dir:
+        return
+    if resource.plural in ('configmaps', 'secrets', 'environmentconfigs'):
+        package_delete(action, package_dir, body.get('data', {}), logger)
+    elif resource.plural == 'compositions':
+        for step in body.get('spec', {}).get('pipeline', []):
+            input = step.get('input')
+            if input and input.get('apiVersion') == 'pythonic.fn.crossplane.io/v1alpha1':
+                package_delete(action, package_dir, step.get('input', {}).get('packages', {}), logger)
+
+
+def resource_package_dir(resource, labels, logger):
+    package = labels.get(PACKAGE_LABEL)
     if package is None:
         if logger:
-            logger.error('function-pythonic.package label is missing')
+            logger.error(f"{PACKAGE_LABEL} label is missing")
         return None
     package_dir = PACKAGES_DIR
-    if package:
+    if resource.plural in ('configmaps', 'secrets') and package:
         for segment in package.split('.'):
             if not segment.isidentifier():
-                if logger:
-                    logger.error('Package has invalid package name: %s', package)
+                logger.error('Package has invalid package name: %s', package)
                 return None
             package_dir = package_dir / segment
     return package_dir
 
 
-def package_file_write(package_dir, name, secret, text, action, logger):
-    package_file = package_dir / name
-    if action != 'Unchanged':
-        package_file.parent.mkdir(parents=True, exist_ok=True)
-        if secret:
-            package_file.write_bytes(base64.b64decode(text.encode('utf-8')))
-        else:
-            package_file.write_text(text)
-    module, name = package_file_name(package_file)
-    if module:
-        if action != 'Unchanged':
-            GRPC_RUNNER.invalidate_module(name)
-        logger.info(f"{action} module: {name}")
-    else:
-        logger.info(f"{action} file: {name}")
-
-
-def package_file_unlink(package_dir, name, action, logger):
-    package_file = package_dir / name
-    package_file.unlink(missing_ok=True)
-    module, name = package_file_name(package_file)
-    if module:
-        GRPC_RUNNER.invalidate_module(name)
-        logger.info(f"{action} module: {name}")
-    else:
-        logger.info(f"{action} file: {name}")
-    package_dir = package_file.parent
-    while (
-            package_dir.is_relative_to(PACKAGES_DIR)
-            and package_dir.is_dir()
-            and not list(package_dir.iterdir())
-    ):
-        package_dir.rmdir()
-        module = str(package_dir.relative_to(PACKAGES_DIR)).replace('/', '.')
-        if module != '.':
-            GRPC_RUNNER.invalidate_module(module)
-            logger.info(f"{action} package: {module}")
-        package_dir = package_dir.parent
-
-
-def package_file_name(package_file):
-    name = str(package_file.relative_to(PACKAGES_DIR))
+def package_create(resource, action, package_dir, package, logger):
+    for name, value in package.items():
+        if validate_entry(name, value, logger):
+            package_name = package_dir / name
+            if isinstance(value, str):
+                package_name.parent.mkdir(parents=True, exist_ok=True)
+                if resource.plural == 'secrets':
+                    package_name.write_bytes(base64.b64decode(value.encode('utf-8')))
+                else:
+                    package_name.write_text(value)
+                module, name = package_file_name(package_name)
+                if module:
+                    GRPC_RUNNER.invalidate_module(name)
+                    logger.info(f"{action} module: {name}")
+                else:
+                    logger.info(f"{action} file: {name}")
+            elif isinstance(value, dict):
+                package_create(resource, action, package_name, value, logger)
+
+
+def package_delete(action, package_dir, package, logger):
+    for name, value in package.items():
+        if validate_entry(name, value, logger):
+            package_name = package_dir / name
+            if isinstance(value, str):
+                package_name.unlink(missing_ok=True)
+                module, name = package_file_name(package_name)
+                if module:
+                    GRPC_RUNNER.invalidate_module(name)
+                    logger.info(f"{action} module: {name}")
+                else:
+                    logger.info(f"{action} file: {name}")
+                parent = package_name.parent
+                while (
+                        parent.is_relative_to(PACKAGES_DIR)
+                        and parent.is_dir()
+                        and not list(parent.iterdir())
+                ):
+                    parent.rmdir()
+                    module = str(parent.relative_to(PACKAGES_DIR)).replace('/', '.')
+                    if module != '.':
+                        GRPC_RUNNER.invalidate_module(module)
+                        logger.info(f"{action} package: {module}")
+                    parent = parent.parent
+            elif isinstance(value, dict):
+                package_delete(action, package_name, value, logger)
+
+
+def validate_entry(name, value, logger):
+    if isinstance(value, str):
+        if not name.endswith('.py'):
+            if '.' in name or '/' in name:
+                logger.error(f"Python package file name is not valid: {name}")
+                return False
+            return True
+        name = name[:-3]
+    elif not isinstance(value, dict):
+        logger.error(f"Python package \"{name}\" value is not a valid type: {value.__class__}")
+        return False
+    if name.isidentifier():
+        return True
+    logger.error(f"Python package name is not an identifier: {name}")
+    return False
+
+
+def package_file_name(package_name):
+    name = str(package_name.relative_to(PACKAGES_DIR))
     if name.endswith('.py'):
         return True, name[:-3].replace('/', '.')
     return False, name