ducted 2.0__py3-none-any.whl

docs/conf.py

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+#
+# Duct documentation build configuration file, created by
+# sphinx-quickstart on Fri Oct 17 21:58:31 2014.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Duct'
+copyright = u'2024, Colin Alston'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '2.0'
+# The full version, including alpha/beta/rc tags.
+release = '2.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Ductdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('index', 'Duct.tex', u'Duct Documentation',
+   u'Colin Alston', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'duct', u'Duct Documentation',
+     [u'Colin Alston'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'Duct', u'Duct Documentation',
+   u'Colin Alston', 'Duct', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

duct/__init__.py

@@ -0,0 +1,13 @@
+"""Duct - A monitoring agent
+
+.. moduleauthor:: Colin Alston <colin@imcol.in>
+
+"""
+
+from duct import service
+
+def makeService(config):
+    """
+    Create DuctService object
+    """
+    return service.DuctService(config)

duct/__main__.py

@@ -0,0 +1,66 @@
+"""
+Duct — asyncio entry point
+
+Usage:
+    ductd -c duct.yml
+    python -m duct -c duct.yml
+"""
+
+import argparse
+import asyncio
+import logging
+import signal
+
+from duct.configuration import ConfigFile
+from duct.service import DuctService
+
+
+def main():
+    """Entry point for the duct daemon."""
+    parser = argparse.ArgumentParser(
+        description='Duct — A monitoring agent and event processor'
+    )
+    parser.add_argument(
+        '-c', '--config',
+        default='duct.yml',
+        help='Configuration file (default: duct.yml)',
+    )
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='Enable verbose/debug logging',
+    )
+    args = parser.parse_args()
+
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(
+        level=log_level,
+        format='%(asctime)s %(name)s %(levelname)s %(message)s',
+    )
+
+    asyncio.run(_run(args.config))
+
+
+async def _run(config_path):
+    config = ConfigFile(config_path)
+    svc = DuctService(config)
+
+    loop = asyncio.get_running_loop()
+
+    stop_event = asyncio.Event()
+
+    def _handle_signal():
+        stop_event.set()
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(sig, _handle_signal)
+
+    await svc.startService()
+
+    await stop_event.wait()
+
+    await svc.stopService()
+
+
+if __name__ == '__main__':
+    main()

duct/aggregators.py

@@ -0,0 +1,33 @@
+"""
+.. module:: aggregators
+   :synopsis: Aggregation functions
+
+.. moduleauthor:: Colin Alston <colin@imcol.in>
+"""
+
+
+def Counter32(al, br, delta):
+    """32bit counter aggregator with wrapping
+    """
+    if br < al:
+        ch = 4294967295 - al
+        return (ch + br) / float(delta)
+
+    return (br - al) / float(delta)
+
+def Counter64(al, br, delta):
+    """64bit counter aggregator with wrapping
+    """
+    if br < al:
+        ch = 18446744073709551615 - al
+        return (ch + br) / float(delta)
+
+    return (br - al) / float(delta)
+
+def Counter(al, br, delta):
+    """Counter derivative
+    """
+    if br < al:
+        return None
+
+    return (br - al) / float(delta)

duct/configuration.py

@@ -0,0 +1,177 @@
+"""
+.. module:: configuration
+   :synopsis: Configuration file parser
+
+.. moduleauthor:: Colin Alston <colin@imcol.in>
+"""
+
+import os
+import itertools
+import copy
+import logging
+
+import yaml
+
+log = logging.getLogger(__name__)
+
+class ConfigurationError(Exception):
+    """General exception class for Duct configuration issues
+    """
+
+class ConfigFile(object):
+    """Duct configuration file parser and accessor
+    """
+    def __init__(self, path):
+        if os.path.exists(path):
+            with open(path, 'rt', encoding='utf-8') as conf:
+                self.raw_config = yaml.load(conf, Loader=yaml.SafeLoader)
+
+            if not self.raw_config:
+                self.raw_config = {}
+                log.warning("Warning: No configuration content")
+        else:
+            raise ConfigurationError(f"Configuration file '{path}' not found")
+
+        self.known_items = {
+            'sources': list,
+            'outputs': list,
+            'interval': (float, int,),
+            'ttl': (float, int,),
+            'blueprint': list,
+            'ssh_key': str,
+            'ssh_user': str,
+            'toolbox': dict
+        }
+
+        self._parse_config()
+
+    def _validate_type(self, item, vtype):
+        if not isinstance(vtype, tuple):
+            vtype = (vtype, )
+
+        parse = False
+        for vt in vtype:
+            try:
+                assert isinstance(self.raw_config.get(item, vt()), vtype)
+                parse = True
+            except AssertionError:
+                pass
+
+        if not parse:
+            raise ConfigurationError(
+                f"{item} must be one of type {repr(vtype)}")
+
+    def _validate_config(self):
+        for key, val in self.known_items.items():
+            self._validate_type(key, val)
+
+    def _parse_config(self):
+        self._merge_includes()
+
+        self._validate_config()
+
+        self._build_blueprints()
+
+    def _merge_includes(self):
+        def both(i1, i2, t):
+            return isinstance(i1, t) and isinstance(i2, t)
+
+        paths = self.raw_config.get('include_path', [])
+        if not isinstance(paths, list):
+            paths = [paths]
+
+        paths2 = self.raw_config.get('include', [])
+        if not isinstance(paths2, list):
+            paths2 = [paths2]
+
+        paths.extend(paths2)
+
+
+        for ipath in paths:
+            if os.path.exists(ipath):
+                files = [os.path.join(ipath, fi) for fi in os.listdir(ipath)
+                         if fi.endswith('.yml') or fi.endswith('.yaml')]
+
+                for conf_file in files:
+                    with open(conf_file, 'rt', encoding='utf-8') as yaml_path:
+                        conf = yaml.load(yaml_path, Loader=yaml.SafeLoader)
+                        for key, val in conf.items():
+                            if key in self.raw_config:
+                                if both(val, self.raw_config[key], dict):
+                                    # Merge dicts
+                                    for k2, v2 in val.items():
+                                        self.raw_config[key][k2] = v2
+
+                                elif both(val, self.raw_config[key], list):
+                                    # Extend lists
+                                    self.raw_config[key].extend(val)
+                                else:
+                                    # Overwrite
+                                    self.raw_config[key] = val
+                            else:
+                                self.raw_config[key] = val
+                        log.warning('Loadded additional configuration from %s',
+                                    conf_file)
+            else:
+                log.warning('Config Error: include_path %s does not exist',
+                            ipath)
+
+    def _build_blueprints(self):
+        # Turn toolboxes into a dict
+        toolboxes = self.raw_config.get('toolbox', {})
+        blueprints = self.raw_config.get('blueprint', [])
+
+        if blueprints:
+            # Make sure raw config stubs exist if we have any blueprints
+            if 'sources' not in self.raw_config:
+                self.raw_config['sources'] = []
+
+        for blueprint in blueprints:
+            tbs = blueprint['toolbox']
+            # Listify it so we can inherit multiple toolboxes
+            if not isinstance(toolboxes, list):
+                tbs = [tbs]
+
+            tbs = [toolboxes[tool] for tool in tbs]
+
+            # Compute a dot product of all the config settings vectors
+            inversions = []
+            for key, val in blueprint.get('sets', {}).items():
+                inversions.append([(key, jay) for jay in val])
+
+            for options in itertools.product(*inversions):
+                for toolbox in tbs:
+                    for source in toolbox.get('sources', []):
+                        # Make a copy of the source dict
+                        mysource = copy.copy(source)
+
+                        # Merge toolbox defaults
+                        for key, val in toolbox.get('defaults', {}).items():
+                            mysource[key] = val
+
+                        # Merge blueprint defaults
+                        for key, val in blueprint.get('defaults', {}).items():
+                            mysource[key] = val
+
+                        # Merge set permutation
+                        for key, val in options:
+                            mysource[key] = val
+
+                        # Bolt it into our real config file
+                        self.raw_config['sources'].append(mysource)
+
+        # Free a tiny bit of memory
+        if 'toolbox' in self.raw_config:
+            del self.raw_config['toolbox']
+
+        if 'blueprint' in self.raw_config:
+            del self.raw_config['blueprint']
+
+    def get(self, item, default=None):
+        """Returns `item` from configuration if it exists, otherwise returns
+           `default`
+        """
+        return self.raw_config.get(item, default)
+
+    def __getitem__(self, item):
+        return self.raw_config[item]

duct/ihateprotobuf/proto_pb2.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: proto.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bproto.proto\"\x81\x01\n\x05State\x12\x0c\n\x04time\x18\x01 \x01(\x03\x12\r\n\x05state\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\x0c\n\x04once\x18\x06 \x01(\x08\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x0b\n\x03ttl\x18\x08 \x01(\x02\"\xce\x01\n\x05\x45vent\x12\x0c\n\x04time\x18\x01 \x01(\x03\x12\r\n\x05state\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x0b\n\x03ttl\x18\x08 \x01(\x02\x12\x1e\n\nattributes\x18\t \x03(\x0b\x32\n.Attribute\x12\x15\n\rmetric_sint64\x18\r \x01(\x12\x12\x10\n\x08metric_d\x18\x0e \x01(\x01\x12\x10\n\x08metric_f\x18\x0f \x01(\x02\"\x17\n\x05Query\x12\x0e\n\x06string\x18\x01 \x01(\t\"g\n\x03Msg\x12\n\n\x02ok\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x16\n\x06states\x18\x04 \x03(\x0b\x32\x06.State\x12\x15\n\x05query\x18\x05 \x01(\x0b\x32\x06.Query\x12\x16\n\x06\x65vents\x18\x06 \x03(\x0b\x32\x06.Event\"\'\n\tAttribute\x12\x0b\n\x03key\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x01(\tB\x1a\n\x11\x63om.aphyr.riemannB\x05Proto')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'proto_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'\n\021com.aphyr.riemannB\005Proto'
+  _STATE._serialized_start=16
+  _STATE._serialized_end=145
+  _EVENT._serialized_start=148
+  _EVENT._serialized_end=354
+  _QUERY._serialized_start=356
+  _QUERY._serialized_end=379
+  _MSG._serialized_start=381
+  _MSG._serialized_end=484
+  _ATTRIBUTE._serialized_start=486
+  _ATTRIBUTE._serialized_end=525
+# @@protoc_insertion_point(module_scope)

duct/interfaces.py

@@ -0,0 +1,21 @@
+"""
+.. module:: interfaces
+   :synopsis: Generic interfaces
+
+.. moduleauthor:: Colin Alston <colin@imcol.in>
+"""
+
+from zope.interface import Interface
+
+# pylint: disable=E0239,W0221
+class IDuctProtocol(Interface):
+    """Interface for Duct client protocols"""
+
+    def sendEvent(self, event):
+        """Sends an event to this client"""
+
+class IDuctSource(Interface):
+    """Interface for Duct metric sources"""
+
+    def get(self):
+        """Return this source data"""