openldap-opencensus-stats 0.0.21__py3-none-any.whl

openldap_opencensus_stats/__init__.py

@@ -0,0 +1,3 @@
+from openldap_opencensus_stats.configuration import Configuration   # noqa: F401
+from openldap_opencensus_stats.ldap_server import LdapServer        # noqa: F401
+from openldap_opencensus_stats.ldap_statistic import LdapStatistic  # noqa: F401

openldap_opencensus_stats/config_transformers/__init__.py

@@ -0,0 +1,20 @@
+
+# The transformations process in the order listed here.
+from openldap_opencensus_stats.config_transformers.base \
+    import ConfigurationTransformationChainSingleton
+from openldap_opencensus_stats.config_transformers.snake_case \
+    import SnakeCaseConfigurationTransformer                                   # noqa: F401
+from openldap_opencensus_stats.config_transformers.metric_dn \
+    import MetricDnConfigurationTransformer                                    # noqa: F401
+from openldap_opencensus_stats.config_transformers.copy_for_each_database \
+    import CopyForEachDatabaseConfigurationTransformer                         # noqa: F401
+from openldap_opencensus_stats.config_transformers.child_object \
+    import ChildObjectConfigurationTransformer                                 # noqa: F401
+from openldap_opencensus_stats.config_transformers.metric_name_interpolation \
+    import MetricNameInterpolationConfigurationTransformer                     # noqa: F401
+from openldap_opencensus_stats.config_transformers.metric_name \
+    import MetricNameConfigurationTransformer                                  # noqa: F401
+from openldap_opencensus_stats.config_transformers.metric_query_dn \
+    import MetricQueryDnConfigurationTransformer                               # noqa: F401
+
+ConfigurationTransformationChainSingleton().register(MetricDnConfigurationTransformer)

openldap_opencensus_stats/config_transformers/base.py

@@ -0,0 +1,47 @@
+import logging
+import re
+
+from openldap_opencensus_stats.ldap_server import LdapServerPool
+
+
+class ConfigurationTransformer:
+    transformers = []
+
+    def __init_subclass__(cls, **kwargs):
+        ConfigurationTransformationChainSingleton().register(cls)
+
+    @staticmethod
+    def process(configuration):
+        return configuration
+
+    @staticmethod
+    def get_ldap_server(server_config):
+        args = dict([
+            (re.sub(r'(?<!^)(?=[A-Z])', '_', name).lower(), value)
+            for name, value in server_config.get('connection', {}).items()
+        ])
+        args['database'] = server_config.get('database')
+        ldap_server = LdapServerPool().get_ldap_server(**args)
+        return ldap_server
+
+
+class ConfigurationTransformationChainSingleton:
+    transformation_chain = []
+
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(cls, 'instance'):
+            cls.instance = super(ConfigurationTransformationChainSingleton, cls).__new__(cls)
+        return cls.instance
+
+    def register(self, cls):
+        self.transformation_chain.append(cls)
+        logging.info(f"Registered transformer: {cls.__name__}")
+
+    def transform_configuration(self, configuration):
+        configuration_result = {}
+        configuration_input = configuration
+        for link in self.transformation_chain:
+            print(f"Calling configuration chain link {link.__name__}")
+            configuration_result = link.process(configuration_input)
+            configuration_input = configuration_result
+        return configuration_result

openldap_opencensus_stats/config_transformers/child_object.py

@@ -0,0 +1,75 @@
+import copy
+import logging
+import re
+
+import ldap
+
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class ChildObjectConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        if not isinstance(configuration, dict):
+            raise ValueError('The top level of the configuration must be a mapping.')
+
+        config = copy.deepcopy(configuration)
+        for server_config in config.get('ldap_servers'):
+            ldap_server = ChildObjectConfigurationTransformer.get_ldap_server(server_config)
+            # A server entry with no object tree defines no cn=Monitor
+            # metrics (e.g. statsLogPipe only) and has nothing to transform
+            if not server_config.get('sync_only', False) and server_config.get('object'):
+                server_config['object'] = ChildObjectConfigurationTransformer.process_objects_for_ldap_server(
+                    configuration=server_config.get('object'),
+                    dn='',
+                    ldap_server=ldap_server
+                )
+
+        return config
+
+    @staticmethod
+    def process_objects_for_ldap_server(configuration, dn, ldap_server):
+        if not configuration or not ldap_server:
+            raise ValueError(
+                'ChildObjectConfigurationTransformer.process_objects_for_ldap_server: All arguments must exist'
+            )
+
+        if isinstance(configuration, dict):
+            config = {}
+            for key, value in configuration.items():
+                proper_value = ChildObjectConfigurationTransformer.process_objects_for_ldap_server(
+                    value,
+                    configuration.get('computed_dn', dn),
+                    ldap_server
+                )
+                if key == 'children':
+                    if not isinstance(proper_value, dict):
+                        raise TypeError("A configuration entry of 'children' must contain a mapping.")
+
+                    for child_dn in ChildObjectConfigurationTransformer.get_child_keys(dn, ldap_server):
+                        child_value = copy.deepcopy(proper_value)
+                        child_value['computed_dn'] = child_dn
+                        child_value['computed_rdn'] = re.sub(r',.*', '', child_dn)
+                        new_key = re.sub(r',.*', '', re.sub(r'^[^=]*=', '', child_dn))
+                        if re.match(child_value.get('rdn'), child_value.get('computed_rdn')):
+                            config[new_key] = child_value
+                else:
+                    config[key] = proper_value
+            return config
+        elif isinstance(configuration, list):
+            return [
+                ChildObjectConfigurationTransformer.process_objects_for_ldap_server(item, dn, ldap_server)
+                for item in configuration
+            ]
+        else:
+            return configuration
+
+    @staticmethod
+    def get_child_keys(dn, ldap_server):
+        ldap_result = ldap_server.query(dn=dn, scope=ldap.SCOPE_ONELEVEL)
+        if not ldap_result:
+            logging.warning(f"Found no child objects for {dn} on LDAP database {ldap_server.database}!")
+        return [
+            result_item[0]
+            for result_item in ldap_result
+        ]

openldap_opencensus_stats/config_transformers/copy_for_each_database.py

@@ -0,0 +1,13 @@
+import copy
+
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class CopyForEachDatabaseConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        conf = copy.deepcopy(configuration)
+        ldap_servers = conf.get('ldap_servers', [])
+        for server_conf in ldap_servers:
+            server_conf['object'] = copy.deepcopy(conf.get('object'))
+        return conf

openldap_opencensus_stats/config_transformers/metric_dn.py

@@ -0,0 +1,46 @@
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class MetricDnConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        return MetricDnConfigurationTransformer.process_object_config(configuration)
+
+    @staticmethod
+    def process_object_config(configuration, suffix_dn='', separator=','):
+        if isinstance(configuration, dict):
+            config = {}
+            if suffix_dn:
+                full_suffix = separator + suffix_dn
+            else:
+                full_suffix = suffix_dn
+            computed_dn = suffix_dn
+            rdn = configuration.get('computed_rdn', configuration.get('rdn'))
+            if rdn:
+                computed_dn = rdn + full_suffix
+
+            for key, value in configuration.items():
+                config[key] = MetricDnConfigurationTransformer.process_object_config(
+                    configuration=value,
+                    suffix_dn=computed_dn,
+                    separator=separator
+                )
+
+            # True if either 'rdn' or 'attribute' is a key
+            if configuration.get('rdn', configuration.get('attribute')):
+                config['computed_dn'] = computed_dn
+            else:
+                computed_dn = suffix_dn
+
+            return config
+        elif isinstance(configuration, list):
+            return [
+                MetricDnConfigurationTransformer.process_object_config(
+                    configuration=item,
+                    suffix_dn=suffix_dn,
+                    separator=separator
+                )
+                for item in configuration
+            ]
+        else:
+            return configuration

openldap_opencensus_stats/config_transformers/metric_name.py

@@ -0,0 +1,37 @@
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class MetricNameConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        return MetricNameConfigurationTransformer.process_object_config(configuration)
+
+    @staticmethod
+    def process_object_config(configuration, prefix='', default_name='', separator='/'):
+        full_prefix = prefix + separator if prefix else ''
+        if isinstance(configuration, dict):
+            config = {}
+            metric_name = prefix
+            if configuration.get('rdn') or configuration.get('attribute'):
+                metric_name = full_prefix + configuration.get('computed_name', configuration.get('name', default_name))
+                config['metric_name'] = metric_name
+            for key, value in configuration.items():
+                config[key] = MetricNameConfigurationTransformer.process_object_config(
+                    configuration=value,
+                    prefix=metric_name,
+                    default_name=key,
+                    separator=separator
+                )
+            return config
+        elif isinstance(configuration, list):
+            return [
+                MetricNameConfigurationTransformer.process_object_config(
+                    configuration=item,
+                    prefix=prefix,
+                    default_name=default_name,
+                    separator=separator
+                )
+                for item in configuration
+            ]
+        else:
+            return configuration

openldap_opencensus_stats/config_transformers/metric_name_interpolation.py

@@ -0,0 +1,84 @@
+import copy
+import logging
+import re
+
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class MetricNameInterpolationConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        if not isinstance(configuration, dict):
+            raise ValueError(
+                'NameInterpolationConfigurationTransformer expect a configuration that is a dict '
+                'but received one that isn\'t.')
+
+        config = copy.deepcopy(configuration)
+        for server_config in config.get('ldap_servers'):
+            if not server_config.get('sync_only', False):
+                ldap_server = ConfigurationTransformer.get_ldap_server(server_config=server_config)
+                server_config['object'] = MetricNameInterpolationConfigurationTransformer.process_objects_for_ldap_server(
+                    configuration=server_config.get('object', {}),
+                    dn='',
+                    ldap_server=ldap_server
+                )
+        return config
+
+    @staticmethod
+    def process_objects_for_ldap_server(configuration, dn, ldap_server):
+
+        if isinstance(configuration, list):
+            return [
+                MetricNameInterpolationConfigurationTransformer.process_objects_for_ldap_server(item, dn, ldap_server)
+                for item in configuration
+            ]
+        elif isinstance(configuration, dict):
+            config = {}
+
+            for key, value in configuration.items():
+                proper_value = MetricNameInterpolationConfigurationTransformer.process_objects_for_ldap_server(
+                    value, dn, ldap_server)
+                if key in ['name']:
+                    config['computed_name'] = MetricNameInterpolationConfigurationTransformer.create_metric_name(
+                        configuration,
+                        ldap_server
+                    )
+                config[key] = proper_value
+
+            return config
+
+        else:
+            return configuration
+
+    @staticmethod
+    def create_metric_name(config, ldap_server):
+        def repl_func(matches):
+            groups = matches.groups()
+            if 'rdn' == groups[0]:
+                index = int(groups[1])
+                rdn_matches = re.match(
+                    config.get('rdn'),            # Pattern to match - the 'rdn' entry
+                    config.get('computed_rdn')    # String for matching - the 'computed_rdn' entry
+                )
+                if rdn_matches:
+                    return rdn_matches.group(index)
+            elif 'attr' == groups[0]:
+                attribute_name = groups[1]
+                ldap_result = ldap_server.query_dn_and_attribute(
+                    config.get('computed_dn', ''),
+                    attribute_name
+                )
+                value = ldap_result[0]
+                if isinstance(value, bytes):
+                    value = value.decode()
+                return str(value)
+            else:
+                logging.error(f"Unknown interpolation requested in name: {groups[0]}.{groups[1]}")
+
+        name = config.get('computed_name', config.get('name', ''))
+        name = re.sub(
+            '{([^\\.}]+)\\.([^}]+)}',
+            repl_func,
+            name
+        ).lower()
+        return name

openldap_opencensus_stats/config_transformers/metric_query_dn.py

@@ -0,0 +1,37 @@
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class MetricQueryDnConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        return MetricQueryDnConfigurationTransformer.process_object_config(configuration)
+
+    @staticmethod
+    def process_object_config(configuration, query_dn=None):
+        if isinstance(configuration, dict):
+            config = {}
+            configuration_query_dn = configuration.get('query_dn', configuration.get('rdn'))
+            if not query_dn and configuration_query_dn:
+                query_dn = configuration_query_dn
+
+            for key, value in configuration.items():
+                config[key] = MetricQueryDnConfigurationTransformer.process_object_config(
+                    configuration=value,
+                    query_dn=query_dn,
+                )
+
+            # True if either 'rdn' or 'attribute' is a key
+            if query_dn:
+                config['query_dn'] = query_dn
+
+            return config
+        elif isinstance(configuration, list):
+            return [
+                MetricQueryDnConfigurationTransformer.process_object_config(
+                    configuration=item,
+                    query_dn=query_dn,
+                )
+                for item in configuration
+            ]
+        else:
+            return configuration

openldap_opencensus_stats/config_transformers/snake_case.py

@@ -0,0 +1,22 @@
+import re
+
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformer
+
+
+class SnakeCaseConfigurationTransformer(ConfigurationTransformer):
+    @staticmethod
+    def process(configuration):
+        config = {}
+        for key, value in configuration.items():
+            snake_case_key = re.sub(r'(?<!^)(?=[A-Z])', '_', key).lower()
+            if isinstance(value, dict):
+                snake_cased_value = SnakeCaseConfigurationTransformer.process(value)
+            elif isinstance(value, list):
+                snake_cased_value = [
+                    SnakeCaseConfigurationTransformer.process(x) if isinstance(x, dict) else x
+                    for x in value
+                ]
+            else:
+                snake_cased_value = value
+            config[snake_case_key] = snake_cased_value
+        return config

openldap_opencensus_stats/configuration.py

@@ -0,0 +1,172 @@
+import copy
+import logging
+import logging.config
+
+import yaml
+from time import sleep
+
+from openldap_opencensus_stats.config_transformers.base import ConfigurationTransformationChainSingleton
+from openldap_opencensus_stats.ldap_metric_set import MetricSet
+from openldap_opencensus_stats.sync_metric_set import SyncMetricSet
+from openldap_opencensus_stats.ldap_server import LdapServerPool
+from openldap_opencensus_stats.ldap_statistic import LdapStatistic
+from openldap_opencensus_stats.ldap_stats_log_pipe import LdapStatsLogPipeReader, METRICS_NAMESPACE_DEFAULT
+
+from opencensus.stats import stats
+from opencensus.ext.prometheus import stats_exporter
+
+# Make up for broken code in the Prometheus exporter
+import opencensus.stats.aggregation_data
+opencensus.stats.aggregation_data.SumAggregationDataFloat = opencensus.stats.aggregation_data.SumAggregationData
+
+SUPPORTED_EXPORTERS = ['Prometheus', 'Stackdriver']
+
+
+class Configuration:
+    def __init__(self, config_file_name):
+        if config_file_name is None:
+            raise ValueError("Config file name must be supplied")
+        self._config_file_name = config_file_name
+        self._configuration_dict = {}
+        self._sleep_time = 5
+        self._metric_sets = []
+        self._ldap_metrics = {}
+        self._stats_log_pipe_readers = []
+
+        self.reconfigure()
+
+    def reconfigure(self):
+        self._configuration_dict = read_yaml_file(self._config_file_name)
+        normalized_configuration = ConfigurationTransformationChainSingleton().transform_configuration(
+            self._configuration_dict
+        )
+
+        self._sleep_time = normalized_configuration.get('period', 5)
+        log_config = normalized_configuration.get('log_config')
+        if log_config and isinstance(log_config, dict):
+            log_config['version'] = log_config.get('version', 1)
+            logging.config.dictConfig(log_config)
+        prometheus_namespace = METRICS_NAMESPACE_DEFAULT
+        for exporter_config in normalized_configuration.get('exporters', []):
+            exporter = create_exporter(exporter_config)
+            stats.stats.view_manager.register_exporter(exporter)
+            if exporter_config.get('name') == 'Prometheus':
+                prometheus_namespace = exporter.options.namespace
+
+        for ldap_server_config in normalized_configuration.get('ldap_servers', []):
+            if not ldap_server_config.get('sync_only', False):
+                metric_set = self.generate_metric_set(ldap_server_config)
+                self._metric_sets.append(metric_set)
+            stats_log_pipe = ldap_server_config.get('stats_log_pipe')
+            if stats_log_pipe:
+                if isinstance(stats_log_pipe, str):
+                    stats_log_pipe = {'pipe': stats_log_pipe}
+                stats_log_pipe_reader = LdapStatsLogPipeReader(
+                    database=ldap_server_config.get('database'),
+                    pipe_path=stats_log_pipe.get('pipe'),
+                    suffixes=stats_log_pipe.get('suffixes'),
+                    namespace=prometheus_namespace
+                )
+                stats_log_pipe_reader.start()
+                self._stats_log_pipe_readers.append(stats_log_pipe_reader)
+
+        for base_dn, sync_config in normalized_configuration.get('sync', {}).items():
+            ldap_server_names = sync_config.get('cluster_servers', [])
+            ldap_servers = [
+                LdapServerPool().get_ldap_server(**server)
+                for server in normalized_configuration.get('ldap_servers', [])
+                if server['database'] in ldap_server_names
+            ]
+            report_servers = sync_config.get('report_servers', [])
+            sync_metric_set = SyncMetricSet(
+                base_dn=base_dn,
+                ldap_servers=ldap_servers,
+                report_servers=report_servers
+            )
+            self._metric_sets.append(sync_metric_set)
+
+    def generate_metric_set(self, ldap_server_config):
+        args = copy.deepcopy(ldap_server_config.get('connection', {}))
+        args['database'] = ldap_server_config.get('database')
+        ldap_server = LdapServerPool().get_ldap_server(**args)
+        metric_set = MetricSet(ldap_server=ldap_server)
+        configs = [ldap_server_config.get('object')]
+        while configs:
+            config = configs.pop()
+            if isinstance(config, list):
+                for item in config:
+                    configs.insert(0, item)
+            elif isinstance(config, dict):
+                dn = config.get('computed_dn')
+                name = config.get('metric_name')
+                attribute = config.get('attribute')
+                unit = config.get('unit')
+                description = config.get('description', '')
+                value_function = config.get('func', 'value')
+                query_dn = config.get('query_dn')
+                if dn and name and attribute and unit and query_dn:
+                    stat = self._ldap_metrics.get(name)
+                    if not stat:
+                        stat = LdapStatistic(
+                            dn=dn,
+                            name=name,
+                            attribute=attribute,
+                            description=description,
+                            unit=unit,
+                            value_function=value_function,
+                            query_dn=query_dn,
+                            tag_keys=['database']
+                        )
+                        self._ldap_metrics[name] = stat
+                    metric_set.add_statistic(stat)
+                for key, value in config.items():
+                    configs.insert(0, value)
+        return metric_set
+
+    def metric_sets(self):
+        return self._metric_sets
+
+    def sleep(self):
+        sleep(self._sleep_time)
+
+
+def read_yaml_file(file_name):
+    with open(file_name, 'r') as file:
+        ret_val = yaml.safe_load(file)
+    return ret_val
+
+
+def create_exporter(exporter_configuration=None):
+    if exporter_configuration is None:
+        raise ValueError("Cannot create an exporter with no configuration!")
+
+    name = exporter_configuration.get('name')
+    if name not in SUPPORTED_EXPORTERS:
+        logging.error(
+            f"Requested exporter named {name}, which is not supported.  Choose from:{', '.join(SUPPORTED_EXPORTERS)}"
+        )
+        raise ValueError(
+            f"Requested exporter named {name}, which is not supported.  Choose from:{', '.join(SUPPORTED_EXPORTERS)}"
+        )
+
+    options = exporter_configuration.get('options', {})
+    exporter = None
+    if "Prometheus" == name:
+        if 'options' not in exporter_configuration:
+            logging.error("The Prometheus exporter requires options configuration.")
+            raise ValueError("The Prometheus exporter requires options configuration.")
+        final_options = {'namespace': METRICS_NAMESPACE_DEFAULT, 'port': 8000, 'address': '0.0.0.0'}
+        final_options.update(options)
+        exporter = stats_exporter.new_stats_exporter(
+            stats_exporter.Options(**final_options)
+        )
+
+    elif "Stackdriver" == name:
+        # Imported here, not at module level: opencensus-ext-stackdriver and its
+        # grpcio dependency are optional (see pyproject.toml's "stackdriver" extra),
+        # so this module must not import them unconditionally.
+        import opencensus.ext.stackdriver.stats_exporter
+        exporter = opencensus.ext.stackdriver.stats_exporter.new_stats_exporter(interval=5)
+        print(f"Exporting stats to this project {exporter.options.project_id}")
+
+    return exporter

openldap_opencensus_stats/ldap_metric_set.py

@@ -0,0 +1,45 @@
+import copy
+
+from opencensus.stats import stats
+from opencensus.tags import tag_map, tag_value, tag_key
+
+
+class MetricSet:
+    def __init__(self, ldap_server=None, ldap_statistics=None):
+        self._ldap_server = ldap_server
+        if not ldap_statistics or not isinstance(ldap_statistics, list):
+            ldap_statistics = []
+        self._ldap_statistics = copy.deepcopy(ldap_statistics)
+        self._query_dns = set()
+
+    def set_ldap_server(self, ldap_server):
+        self._ldap_server = ldap_server
+
+    def add_statistic(self, ldap_statistic):
+        self._ldap_statistics.append(ldap_statistic)
+        self._query_dns.add(ldap_statistic.query_dn)
+
+    def collect(self):
+        tag_keys = [tag_key.TagKey('database')]
+        results = {}
+        for query_dn in self._query_dns:
+            results[query_dn] = dict([
+                (result_dn, result_attributes)
+                for result_dn, result_attributes in self._ldap_server.query(dn=query_dn)
+            ])
+        mmap = stats.stats.stats_recorder.new_measurement_map()
+        for server_statistic in self._ldap_statistics:
+            ldap_value = results.get(
+                server_statistic.query_dn, {}
+            ).get(
+                server_statistic.dn, {}
+            ).get(
+                server_statistic.attribute
+            )
+            server_statistic.collect(ldap_server=self._ldap_server, measurement_map=mmap, ldap_value=ldap_value)
+        tmap = tag_map.TagMap()
+        tmap.insert(
+            tag_keys[0],
+            tag_value.TagValue(self._ldap_server.database)
+        )
+        mmap.record(tmap)