gluetool 3.2__tar.gz → 3.4__tar.gz

{gluetool-3.2 → gluetool-3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gluetool
-Version: 3.2
+Version: 3.4
 Summary: Python framework for constructing command-line pipelines.
 Home-page: https://gluetool.readthedocs.org/
 License: BSD
@@ -30,7 +30,6 @@ Requires-Dist: docutils
 Requires-Dist: lxml
 Requires-Dist: mock
 Requires-Dist: mypy-extensions
-Requires-Dist: packaging
 Requires-Dist: psutil
 Requires-Dist: requests
 Requires-Dist: requests-toolbelt

{gluetool-3.2 → gluetool-3.4}/gluetool/action.py

@@ -3,9 +3,9 @@ Actions are pieces of workflow. Actions have a name and start/end time, and acti
 
 This module provides simple instrumentation to enable logging of actions, and - when client
 libraries are installed - submission of actions to storages compatible with
-<https://opentracing.io/docs/overview/>`_.
+`OpenTelemetry <https://opentelemetry.io/>`_.
 
-Actions - which are wrapping Open Tracing's _span_ objects - can form an acyclic graph - actions
+Actions - which are wrapping OpenTelemetry's _span_ objects - can form an acyclic graph - actions
 can have a single parent, it is then possible to start with a "root" action (e.g. "running the pipeline")
 from which are children actions spawned (e.g. "executing module") and these actions can become parents
 to actions spawned deeper in modules, end so on and on, forming a nice overview of what the code did, how
@@ -34,37 +34,43 @@ When a supported tracing client is installed, spans created by actions actions a
 
 Supported clients:
 
-    * https://github.com/jaegertracing/jaeger-client-python
+    * `OpenTelemetry SDK <https://opentelemetry-python.readthedocs.io/>`_ with OTLP exporter
 
 To control behavior of tracing subsystem, you can use following environment variables:
 
     * ``GLUETOOL_TRACING_DISABLE`` - when set to anything, tracing won't be enabled even when a client
       is available.
     * ``GLUETOOL_TRACING_SERVICE_NAME`` - given string is used to name the trace produced by ``gluetool`` execution.
-    * ``GLUETOOL_TRACING_REPORTING_HOST`` - given string represents a hostname where service, capturing
-        traces, listens.
-    * ``GLUETOOL_TRACING_REPORTING_PORT`` - given integer represents a port number where service, capturing
-        traces, listens.
+    * ``GLUETOOL_TRACING_REPORTING_ENDPOINT`` - OTLP endpoint URL (e.g. ``http://localhost:4317``).
+    * ``GLUETOOL_TRACING_INSECURE`` - controls whether the gRPC connection uses TLS. The connection
+      is insecure (plaintext) by default. Set to ``0``, ``false``, or ``no`` to enable TLS.
+    * ``GLUETOOL_TRACING_TLS_CERT`` - path to a custom CA certificate bundle (PEM) for TLS connections.
+      Only used when ``GLUETOOL_TRACING_INSECURE`` is disabled.
 """
 
 import datetime
 import os
 import threading
-import time
 
 from six import iteritems
 
 from .log import Logging
-from .result import Result
 
 try:
     if os.getenv('GLUETOOL_TRACING_DISABLE'):
         raise ImportError('Tracing is disabled')
 
-    import jaeger_client as tracing_client
+    import grpc
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 
-except (ImportError, AttributeError):
-    tracing_client = None
+    HAS_TRACING = True
+
+except ImportError:
+    HAS_TRACING = False
 
 # Type annotations
 # pylint: disable=unused-import, wrong-import-order
@@ -72,18 +78,16 @@ from typing import cast, Any, Dict, Generator, Iterator, List, Optional, Union
 
 from .log import ContextAdapter  # noqa
 
-TracingClientType = Any  # pylint: disable=invalid-name
-
 
 TRACING_DISABLE_ENVVAR = 'GLUETOOL_TRACING_DISABLE'
 TRACING_SERVICE_NAME_ENVVAR = 'GLUETOOL_TRACING_SERVICE_NAME'
-TRACING_REPORTING_HOST_ENVVAR = 'GLUETOOL_TRACING_REPORTING_HOST'
-TRACING_REPORTING_PORT_ENVVAR = 'GLUETOOL_TRACING_REPORTING_PORT'
+TRACING_REPORTING_ENDPOINT_ENVVAR = 'GLUETOOL_TRACING_REPORTING_ENDPOINT'
 TRACING_FLUSH_TIMEOUT_ENVVAR = 'GLUETOOL_TRACING_FLUSH_TIMEOUT'
+TRACING_INSECURE_ENVVAR = 'GLUETOOL_TRACING_INSECURE'
+TRACING_TLS_CERT_ENVVAR = 'GLUETOOL_TRACING_TLS_CERT'
 
 DEFAULT_TRACING_SERVICE_NAME = 'gluetool'
-DEFAULT_TRACING_REPORTING_HOST = '127.0.0.1'
-DEFAULT_TRACING_REPORTING_PORT = 5775
+DEFAULT_TRACING_REPORTING_ENDPOINT = 'http://localhost:4317'
 DEFAULT_TRACING_FLUSH_TIMEOUT = 30
 
 
@@ -93,54 +97,50 @@ class Tracer(object):
 
     :param str service_name: name to apply to all traces produced by this tracer.
     :param ContextAdapter logger: logger to use for logging.
-    :param str reporting_host: address to which tracer should submit traces.
-    :param int reporting_port: port to which tracer should submit tracers.
+    :param str endpoint: OTLP endpoint URL to which tracer should submit traces.
     """
     # pylint: disable=too-few-public-methods
 
-    TRACER: Optional[TracingClientType] = None
+    TRACER: Optional[Any] = None
+    _PROVIDER: Optional[Any] = None
 
-    # pylint: disable=too-many-arguments
     def __init__(self,
                  service_name: Optional[str] = None,
                  logger: Optional[ContextAdapter] = None,
-                 reporting_host: Optional[str] = None,
-                 reporting_port: Optional[int] = None) -> None:
+                 endpoint: Optional[str] = None) -> None:
 
-        if not tracing_client or os.getenv(TRACING_DISABLE_ENVVAR):
+        if not HAS_TRACING or os.getenv(TRACING_DISABLE_ENVVAR):
             return
 
         self.logger = logger or Logging.get_logger()
 
-        if not service_name:
-            service_name = os.getenv(TRACING_SERVICE_NAME_ENVVAR, default=DEFAULT_TRACING_SERVICE_NAME)
+        resolved_service_name: str = service_name or os.getenv(
+            TRACING_SERVICE_NAME_ENVVAR, DEFAULT_TRACING_SERVICE_NAME
+        ) or DEFAULT_TRACING_SERVICE_NAME
 
-        if not reporting_host:
-            reporting_host = os.getenv(TRACING_REPORTING_HOST_ENVVAR, default=DEFAULT_TRACING_REPORTING_HOST)
+        resolved_endpoint: str = endpoint or os.getenv(
+            TRACING_REPORTING_ENDPOINT_ENVVAR, DEFAULT_TRACING_REPORTING_ENDPOINT
+        ) or DEFAULT_TRACING_REPORTING_ENDPOINT
 
-        if not reporting_port:
-            # pylint: disable=bad-option-value,invalid-envvar-default
-            reporting_port = int(os.getenv(TRACING_REPORTING_PORT_ENVVAR, default=DEFAULT_TRACING_REPORTING_PORT))
-
-        config = tracing_client.Config(
-            config={
-                # Using `const` sampler - the same sampling decission for all spans,
-                # and that decision is "record" (because `param == 1`).
-                'sampler': {
-                    'type': 'const',
-                    'param': 1
-                },
-                'local_agent': {
-                    'reporting_host': reporting_host,
-                    'reporting_port': reporting_port
-                },
-                'logging': True
-            },
-            service_name=service_name,
-            validate=True
-        )
+        insecure = os.getenv(TRACING_INSECURE_ENVVAR, '').lower() not in ('0', 'false', 'no')
+        tls_cert = os.getenv(TRACING_TLS_CERT_ENVVAR)
+
+        resource = Resource.create({'service.name': resolved_service_name})
+        provider = TracerProvider(resource=resource)
+
+        if insecure:
+            exporter = OTLPSpanExporter(endpoint=resolved_endpoint, insecure=True)
+        elif tls_cert:
+            with open(tls_cert, 'rb') as f:
+                credentials = grpc.ssl_channel_credentials(root_certificates=f.read())
+            exporter = OTLPSpanExporter(endpoint=resolved_endpoint, credentials=credentials)
+        else:
+            exporter = OTLPSpanExporter(endpoint=resolved_endpoint)
+        provider.add_span_processor(BatchSpanProcessor(exporter))
+        trace.set_tracer_provider(provider)
 
-        Tracer.TRACER = config.initialize_tracer()
+        Tracer._PROVIDER = provider
+        Tracer.TRACER = trace.get_tracer(resolved_service_name)
 
     def close(self, flush_timeout: Optional[int] = None, logger: Optional[ContextAdapter] = None) -> None:
         """
@@ -151,31 +151,19 @@ class Tracer(object):
         :param ContextAdapter logger: logger to use for logging.
         """.format(TRACING_FLUSH_TIMEOUT_ENVVAR, DEFAULT_TRACING_FLUSH_TIMEOUT)
 
-        if not Tracer.TRACER:
+        if not Tracer._PROVIDER:
             return
 
         logger = logger or self.logger
 
         logger.info('Flushing tracing data')
 
-        # Make pylint happy about circular imports by not using global import.
-        # pylint: disable=cyclic-import
-        from .utils import wait
-
         if not flush_timeout:
             # pylint: disable=bad-option-value,invalid-envvar-default
             flush_timeout = int(os.getenv(TRACING_FLUSH_TIMEOUT_ENVVAR, default=DEFAULT_TRACING_FLUSH_TIMEOUT))
 
-        # yield to IOLoop to flush the spans - https://github.com/jaegertracing/jaeger-client-python/issues/50
-        time.sleep(2)
-
-        future = Tracer.TRACER.close()
-
-        def _check_flush() -> Result[bool, str]:
-
-            return Result.Ok(True) if future.done() else Result.Error('flush pending')
-
-        wait('tracing flush', _check_flush, timeout=flush_timeout, tick=2, logger=logger)
+        Tracer._PROVIDER.force_flush(timeout_millis=flush_timeout * 1000)
+        Tracer._PROVIDER.shutdown()
 
 
 class Action(object):
@@ -287,14 +275,15 @@ class Action(object):
 
         self.tags = tags or {}
 
-        if Tracer.TRACER:
-            if parent:
-                parent_span = parent.span
-
+        if Tracer.TRACER and HAS_TRACING:
+            if parent and parent.span:
+                parent_ctx = trace.set_span_in_context(parent.span)
             else:
-                parent_span = None
+                parent_ctx = None
 
-            self.span: Any = Tracer.TRACER.start_span(label, child_of=parent_span, tags=tags)
+            self.span: Any = Tracer.TRACER.start_span(
+                label, context=parent_ctx, attributes=tags
+            )
 
         else:
             self.span = None
@@ -323,7 +312,7 @@ class Action(object):
         Action._drop_action(self)
 
         if self.span:
-            self.span.finish()
+            self.span.end()
 
         self.logger.debug("action '{}', child of '{}', with span '{}', finished at {}".format(
             self.label,
@@ -345,7 +334,7 @@ class Action(object):
         self.tags[name] = value
 
         if self.span:
-            self.span.set_tag(name, value)
+            self.span.set_attribute(name, value)
 
     def set_tags(self, tags: Dict[str, Any]) -> None:
 

{gluetool-3.2 → gluetool-3.4}/gluetool/glue.py

@@ -1788,6 +1788,15 @@ class Glue(Configurable):
                 'action': 'store_true',
                 'default': False
             },
+            'sigterm-timeout': {
+                'help': """
+                        Time, in seconds, to wait for a child process to exit after sending it ``SIGTERM``,
+                        before resorting to ``SIGKILL``. Takes precedence over the ``GLUETOOL_SIGTERM_TIMEOUT``
+                        environment variable (Default: 60).
+                        """,
+                'type': int,
+                'default': None
+            },
         }),
         ('Output control', {
             ('c', 'colors'): {
@@ -2510,6 +2519,7 @@ class Glue(Configurable):
         * GLUETOOL_TRACING_SERVICE_NAME (string) - name of the trace produced by tool execution
         * GLUETOOL_TRACING_REPORTING_HOST (string) - a hostname where tracing collector listens
         * GLUETOOL_TRACING_REPORTING_PORT (int) - a port on which tracing collector listens
+        * GLUETOOL_SIGTERM_TIMEOUT (int) - seconds to wait for a child to exit after SIGTERM before SIGKILL
         """).format(module_dirs, data_dirs, module_config_dirs)
 
         self._parse_args(args,

{gluetool-3.2 → gluetool-3.4}/gluetool/log.py

@@ -1132,17 +1132,17 @@ class Logging(object):
         assert Logging.stderr_handler is not None
         logger.addHandler(Logging.stderr_handler)
 
-        # Extra handling for Jaeger logger:
-        # - replace it with a context adapter to keep track of what's being done by Jaeger code,
-        # - patch its `info` with `debug` - Jaeger is logging things with INFO severity, which spoils
+        # Extra handling for OpenTelemetry logger:
+        # - replace it with a context adapter to keep track of what's being done by tracing code,
+        # - patch its `info` with `debug` - OpenTelemetry logs with INFO severity, which spoils
         #   our output, DEBUG is perfectly fine for us.
-        jaeger_logger = logging.getLogger('jaeger_tracing')
+        otel_logger = logging.getLogger('opentelemetry')
 
-        if logger == jaeger_logger:
-            jaeger_context_adapter = PackageAdapter(Logging.get_logger(), 'tracing')
+        if logger == otel_logger:
+            otel_context_adapter = PackageAdapter(Logging.get_logger(), 'tracing')
 
             for attr in ('debug', 'info', 'warning', 'error', 'exception'):
-                setattr(logger, attr, getattr(jaeger_context_adapter, attr))
+                setattr(logger, attr, getattr(otel_context_adapter, attr))
 
             logger.info = logger.debug  # type: ignore
 
@@ -1172,7 +1172,7 @@ class Logging(object):
 
     OUR_LOGGERS = (
         logging.getLogger('gluetool'),
-        logging.getLogger('jaeger_tracing'),
+        logging.getLogger('opentelemetry'),
         logging.getLogger('urllib3')
     )
 

gluetool-3.4/gluetool/tests/test_action.py

@@ -0,0 +1,109 @@
+import os
+
+import pytest
+from unittest.mock import patch, MagicMock
+
+import gluetool.action
+from gluetool.action import (
+    TRACING_INSECURE_ENVVAR,
+    TRACING_TLS_CERT_ENVVAR,
+    Tracer,
+)
+
+
+@pytest.fixture(autouse=True)
+def _reset_tracer():
+    """Reset Tracer class state between tests."""
+    Tracer.TRACER = None
+    Tracer._PROVIDER = None
+    yield
+    Tracer.TRACER = None
+    Tracer._PROVIDER = None
+
+
+@pytest.fixture()
+def mock_otel():
+    """Mock all OpenTelemetry objects used by Tracer.__init__."""
+    with patch.object(gluetool.action, 'HAS_TRACING', True), \
+         patch.object(gluetool.action, 'Resource'), \
+         patch.object(gluetool.action, 'TracerProvider'), \
+         patch.object(gluetool.action, 'BatchSpanProcessor'), \
+         patch.object(gluetool.action, 'OTLPSpanExporter') as mock_exporter_cls, \
+         patch.object(gluetool.action, 'trace'), \
+         patch.object(gluetool.action, 'grpc', create=True) as mock_grpc:
+        yield {
+            'exporter_cls': mock_exporter_cls,
+            'grpc': mock_grpc,
+        }
+
+
+@pytest.mark.parametrize('env_value', [
+    '',      # not set
+    'true',
+    'yes',
+    '1',
+    'anything',
+])
+def test_insecure_by_default(env_value, mock_otel):
+    """Connection is insecure when GLUETOOL_TRACING_INSECURE is unset or not explicitly disabled."""
+
+    env = {TRACING_INSECURE_ENVVAR: env_value} if env_value else {}
+
+    with patch.dict(os.environ, env, clear=False):
+        if not env_value:
+            os.environ.pop(TRACING_INSECURE_ENVVAR, None)
+        Tracer()
+
+    _, kwargs = mock_otel['exporter_cls'].call_args
+    assert kwargs.get('insecure') is True
+
+
+@pytest.mark.parametrize('env_value', ['0', 'false', 'no', 'False', 'NO', 'No'])
+def test_secure_when_insecure_disabled(env_value, mock_otel):
+    """Connection uses TLS when GLUETOOL_TRACING_INSECURE is set to 0/false/no."""
+
+    with patch.dict(os.environ, {TRACING_INSECURE_ENVVAR: env_value}, clear=False):
+        os.environ.pop(TRACING_TLS_CERT_ENVVAR, None)
+        Tracer()
+
+    _, kwargs = mock_otel['exporter_cls'].call_args
+    assert 'insecure' not in kwargs
+    assert 'credentials' not in kwargs
+
+
+def test_tls_cert_used_when_secure(mock_otel, tmp_path):
+    """Custom CA cert is loaded and passed as gRPC credentials when TLS is enabled."""
+
+    cert_content = b'--- FAKE CA CERT ---'
+    cert_file = tmp_path / 'ca.pem'
+    cert_file.write_bytes(cert_content)
+
+    mock_creds = MagicMock()
+    mock_otel['grpc'].ssl_channel_credentials.return_value = mock_creds
+
+    with patch.dict(os.environ, {
+        TRACING_INSECURE_ENVVAR: 'false',
+        TRACING_TLS_CERT_ENVVAR: str(cert_file),
+    }, clear=False):
+        Tracer()
+
+    mock_otel['grpc'].ssl_channel_credentials.assert_called_once_with(root_certificates=cert_content)
+    _, kwargs = mock_otel['exporter_cls'].call_args
+    assert kwargs.get('credentials') is mock_creds
+
+
+def test_tls_cert_ignored_when_insecure(mock_otel, tmp_path):
+    """GLUETOOL_TRACING_TLS_CERT is ignored when connection is insecure."""
+
+    cert_file = tmp_path / 'ca.pem'
+    cert_file.write_bytes(b'--- FAKE CA CERT ---')
+
+    with patch.dict(os.environ, {
+        TRACING_TLS_CERT_ENVVAR: str(cert_file),
+    }, clear=False):
+        os.environ.pop(TRACING_INSECURE_ENVVAR, None)
+        Tracer()
+
+    mock_otel['grpc'].ssl_channel_credentials.assert_not_called()
+    _, kwargs = mock_otel['exporter_cls'].call_args
+    assert kwargs.get('insecure') is True

{gluetool-3.2 → gluetool-3.4}/gluetool/tests/test_tool.py

@@ -175,3 +175,69 @@ def test_signal(monkeypatch, log, tested_signal, exception, terminate_process_tr
     assert log.match(levelno=logging.WARNING, message=warnmsg)
     assert log.match(levelno=logging.DEBUG, message='Exiting with status 0')
     assert log.match(levelno=logging.INFO, message='gluetool some-version')
+
+
+def test_sigterm_timeout_default(monkeypatch):
+    monkeypatch.delenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, raising=False)
+
+    assert gluetool.tool.sigterm_timeout() == gluetool.tool.DEFAULT_SIGTERM_TIMEOUT
+
+
+@pytest.mark.parametrize('value,expected', [
+    ('0', 0),
+    ('1', 1),
+    ('120', 120),
+], ids=['zero', 'one', 'custom'])
+def test_sigterm_timeout_envvar(monkeypatch, value, expected):
+    monkeypatch.setenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, value)
+
+    assert gluetool.tool.sigterm_timeout() == expected
+
+
+def test_sigterm_timeout_envvar_invalid(monkeypatch):
+    monkeypatch.setenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, 'not-a-number')
+
+    with pytest.raises(ValueError):
+        gluetool.tool.sigterm_timeout()
+
+
+def test_sigterm_timeout_option_overrides_envvar(monkeypatch):
+    # The `sigterm-timeout` option (command line or configuration file) takes precedence over the env variable.
+    monkeypatch.setenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, '30')
+
+    glue = MagicMock()
+    glue.option = MagicMock(return_value=90)
+
+    assert gluetool.tool.sigterm_timeout(glue) == 90
+    glue.option.assert_called_once_with('sigterm-timeout')
+
+
+def test_sigterm_timeout_option_unset_falls_back_to_envvar(monkeypatch):
+    # When the option is not set, the env variable is used.
+    monkeypatch.setenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, '30')
+
+    glue = MagicMock()
+    glue.option = MagicMock(return_value=None)
+
+    assert gluetool.tool.sigterm_timeout(glue) == 30
+
+
+def test_sigterm_timeout_option_unset_falls_back_to_default(monkeypatch):
+    # When neither the option nor the env variable is set, the default is used.
+    monkeypatch.delenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, raising=False)
+
+    glue = MagicMock()
+    glue.option = MagicMock(return_value=None)
+
+    assert gluetool.tool.sigterm_timeout(glue) == gluetool.tool.DEFAULT_SIGTERM_TIMEOUT
+
+
+def test_sigterm_timeout_option_from_cmdline(monkeypatch):
+    # The `sigterm-timeout` option is wired into the command-line parser and parsed as an integer.
+    monkeypatch.delenv(gluetool.tool.SIGTERM_TIMEOUT_ENVVAR, raising=False)
+
+    glue = NonLoadingGlue()
+    glue._parse_args(['--sigterm-timeout', '120'])
+
+    assert glue.option('sigterm-timeout') == 120
+    assert gluetool.tool.sigterm_timeout(glue) == 120

{gluetool-3.2 → gluetool-3.4}/gluetool/tool.py

@@ -44,6 +44,36 @@ DEFAULT_GLUETOOL_CONFIG_PATHS = [
 
 DEFAULT_SIGTERM_TIMEOUT = 60
 
+# Environment variable overriding the time (in seconds) ``gluetool`` waits for a child process to exit after
+# sending it ``SIGTERM`` before resorting to ``SIGKILL``.
+SIGTERM_TIMEOUT_ENVVAR = 'GLUETOOL_SIGTERM_TIMEOUT'
+
+
+def sigterm_timeout(glue: Optional['gluetool.glue.Glue'] = None) -> int:
+    """
+    Return the time, in seconds, ``gluetool`` waits for a child process to exit after sending it ``SIGTERM``
+    before resorting to ``SIGKILL``.
+
+    The value is resolved with the following precedence:
+
+    1. the ``sigterm-timeout`` option (set on the command line or in a configuration file),
+    2. the :py:data:`SIGTERM_TIMEOUT_ENVVAR` environment variable,
+    3. :py:data:`DEFAULT_SIGTERM_TIMEOUT`.
+
+    :param glue: ``Glue`` instance to read the ``sigterm-timeout`` option from. When not provided, the option
+        is skipped and the value is resolved from the environment variable or the default.
+    """
+
+    if glue is not None:
+        option = glue.option('sigterm-timeout')
+        if option is not None:
+            return cast(int, option)
+
+    if SIGTERM_TIMEOUT_ENVVAR in os.environ:
+        return int(os.environ[SIGTERM_TIMEOUT_ENVVAR])
+
+    return DEFAULT_SIGTERM_TIMEOUT
+
 
 def handle_exc(func: Callable[..., Any]) -> Callable[..., Any]:
 
@@ -305,7 +335,7 @@ Will try to submit it to Sentry but giving up on everything else.
             child.terminate()
 
             try:
-                child.wait(timeout=DEFAULT_SIGTERM_TIMEOUT)
+                child.wait(timeout=sigterm_timeout(Glue))
 
             except psutil.TimeoutExpired:
                 Glue.warn("Sending SIGKILL to child process '{}' (PID {})".format(child.name(), child.pid))

{gluetool-3.2 → gluetool-3.4}/pyproject.toml

@@ -4,7 +4,7 @@ dynamic = ["version"]
 
 [tool.poetry]
 name = "gluetool"
-version = "3.2"
+version = "3.4"
 description = "Python framework for constructing command-line pipelines."
 authors = [
   "Milos Prchlik <mprchlik@redhat.com>",
@@ -42,7 +42,6 @@ include = [
   "assets/html-log/semantic.min.js",
   "gluetool_modules/__init__.py",
   "gluetool_modules/bash_completion.py",
-  "gluetool_modules/dep_list.py",
   "gluetool_modules/yaml_pipeline.py"
 ]
 
@@ -67,7 +66,6 @@ urlnormalizer = "*"
 # depending on extended argument annotations.
 mypy-extensions = "*"
 zipp = "*"
-packaging = "*"
 MarkupSafe = "*"
 attrs = "*"
 cattrs = "*"
@@ -102,9 +100,13 @@ gluetool-html-log = "gluetool.html_log:main"
 
 [tool.poetry.plugins."gluetool.modules"]
 ".bash_completion" = "gluetool_modules.bash_completion:BashCompletion"
-".dep_list" = "gluetool_modules.dep_list:DepList"
 ".yaml_pipeline" = "gluetool_modules.yaml_pipeline:YAMLPipeline"
 
+
+[tool.poetry.group.dev.dependencies]
+opentelemetry-sdk = "^1.40.0"
+opentelemetry-exporter-otlp-proto-grpc = "^1.40.0"
+
 [tool.poetry-dynamic-versioning]
 enable = false
 

gluetool-3.2/gluetool/core.moduleinfo

@@ -1,33 +0,0 @@
----
-- name: citool-core
-  description: Continuous Integration Swiss Army Knife
-  dependencies:
-    yum:
-      - krb5-devel
-    pip:
-      - alabaster==0.7.10
-      - Babel==2.5.1
-      - beautifulsoup4==4.5.3
-      - certifi==2017.7.27.1
-      - chardet==3.0.4
-      - colorama==0.3.7
-      - contextlib2==0.5.5
-      - docutils==0.13.1
-      - enum34==1.1.6
-      - idna==2.6
-      - imagesize==0.7.1
-      - Jinja2==2.9.6
-      - Mako==1.0.6
-      - MarkupSafe==1.0
-      - Pygments==2.2.0
-      - pytz==2017.2
-      - raven==6.0.0
-      - requests==2.18.4
-      - ruamel.ordereddict==0.4.13
-      - ruamel.yaml==0.15.34
-      - six==1.11.0
-      - snowballstemmer==1.2.1
-      - Sphinx==1.5.2
-      - tabulate==0.7.7
-      - urllib3==1.22
-      - urlnorm==1.1.4

gluetool-3.2/gluetool_modules/dep_list.py

@@ -1,285 +0,0 @@
-import os
-import operator
-
-from packaging.version import Version
-
-from six import iteritems, itervalues
-
-import gluetool
-
-
-class ModuleInfo(object):
-    # pylint: disable=too-few-public-methods
-    def __init__(self, data):
-        self.name = data['name']
-        self.description = data.get('description', '')
-        self.dependencies = data.get('dependencies', {})
-        deps = self.dependencies
-
-        self.repo = self.normalize_repos(deps.get('repo', []))
-        self.yum = deps.get('yum', [])
-        self.pip = deps.get('pip', [])
-        self.tasks = deps.get('ansible_tasks', [])
-
-    @staticmethod
-    def normalize_repos(repos):
-        for repo in repos:
-            if 'name' not in repo:
-                repo['name'] = 'repo' + str(id(repo))
-        return repos
-
-
-class ModuleInfoGroup(object):
-    def __init__(self, logger):
-        self.items = {}
-        self.logger = logger
-        self.ops_map = {
-            '>': operator.gt,
-            '<': operator.lt,
-            '>=': operator.ge,
-            '<=': operator.le,
-            '==': operator.eq
-        }
-
-    def add_moduleinfo(self, moduleinfo):
-        if moduleinfo.name in self.items:
-            self.logger.warn("Module '{}' is already loaded".format(moduleinfo.name))
-            return False
-
-        self.items[moduleinfo.name] = moduleinfo
-        return True
-
-    def get_dependencies(self, only_modules):
-        data = {
-            'repo': [],
-            'yum': [],
-            'pip': [],
-            'ansible_tasks': []
-        }
-        for moduleinfo in itervalues(self.items):
-            if only_modules and moduleinfo.name not in only_modules:
-                self.logger.debug("Skip module '{}'".format(moduleinfo.name))
-                continue
-
-            self.logger.info('Collect dependencies for \'{}\''.format(moduleinfo.name))
-            if moduleinfo.yum:
-                data['yum'].extend(moduleinfo.yum)
-            if moduleinfo.pip:
-                data['pip'].extend(moduleinfo.pip)
-            if moduleinfo.repo:
-                data['repo'].extend(moduleinfo.repo)
-            if moduleinfo.tasks:
-                data['ansible_tasks'].extend(moduleinfo.tasks)
-
-        self.logger.debug('pip dependencies:\n{}'.format(data['pip']))
-        data['pip'] = self.pip_version_unify(data['pip'])
-
-        return data
-
-    def pip_version_unify(self, pip_deps):
-        result = []
-        versions = {}
-        for item in pip_deps:
-            pkg, oper, version = ModuleInfoGroup.parse_pkgver(item)
-            if pkg in versions:
-                version_limit = versions[pkg]
-            else:
-                version_limit = gluetool.utils.Bunch(pkg=pkg, equal=None, lower=None, upper=None)
-                versions[pkg] = version_limit
-            if oper and version:
-                self.logger.debug(item)
-                self.limit_version(version_limit, oper, version)
-        self.logger.debug(versions)
-        for _, item in iteritems(versions):
-            result.append(self.get_allowed_version_bounds(item))
-        return result
-
-    def get_allowed_version_bounds(self, item):
-        if not item.equal and not item.lower and not item.upper:
-            return item.pkg
-        if item.equal:
-            if item.upper:
-                oper, version = item.upper
-                relate = self.ops_map[oper]
-                if not relate(Version(item.equal), Version(version)):
-                    raise gluetool.GlueError("Cannot find common version for package '{}'".format(item.pkg))
-            if item.lower:
-                oper, version = item.lower
-                relate = self.ops_map[oper]
-                if not relate(Version(item.equal), Version(version)):
-                    raise gluetool.GlueError("Cannot find common version for package '{}'".format(item.pkg))
-            return '{}=={}'.format(item.pkg, item.equal)
-        if item.upper and item.lower:
-            lower_operator, lower_version = item.lower
-            upper_operator, upper_version = item.upper
-            relate = self.ops_map[lower_operator]
-            if not relate(Version(upper_version), Version(lower_version)):
-                raise gluetool.GlueError("Cannot find common version for package '{}'".format(item.pkg))
-            relate = self.ops_map[upper_operator]
-            if not relate(Version(lower_version), Version(upper_version)):
-                raise gluetool.GlueError("Cannot find common version for package '{}'".format(item.pkg))
-            return '{}{}{},{}{}'.format(item.pkg, lower_operator, lower_version, upper_operator, upper_version)
-        if item.upper:
-            upper_operator, upper_version = item.upper
-            return '{}{}{}'.format(item.pkg, upper_operator, upper_version)
-        if item.lower:
-            lower_operator, lower_version = item.lower
-            return '{}{}{}'.format(item.pkg, lower_operator, lower_version)
-
-    @staticmethod
-    def limit_version(item, oper, version):
-        if oper in ['>', '>=']:
-            if item.lower:
-                _, saved_version = item.lower
-                if Version(saved_version) < Version(version):
-                    item.lower = [oper, version]
-                elif Version(saved_version) == Version(version) and oper == '>':
-                    item.lower = [oper, version]
-            else:
-                item.lower = [oper, version]
-        elif oper in ['<', '<=']:
-            if item.upper:
-                _, saved_version = item.upper
-                if Version(saved_version) > Version(version):
-                    item.upper = [oper, version]
-                elif Version(saved_version) == Version(version) and oper == '<':
-                    item.upper = [oper, version]
-            else:
-                item.upper = [oper, version]
-        elif oper in ['==']:
-            if item.equal and item.equal != version:
-                raise gluetool.GlueError(
-                    "Different versions '{}' and '{}' of package '{}' required".format(item.equal, version, item.pkg)
-                )
-            item.equal = version
-        else:
-            raise gluetool.GlueError("Unsupported operator '{}'".format(oper))
-
-    @staticmethod
-    def parse_pkgver(string):
-        ops = ['==', '<=', '>=', '<', '>']
-        for item in ops:
-            if item in string:
-                split = string.split(item)
-                return split[0], item, split[1]
-        return string, None, None
-
-
-class DepList(gluetool.Module):
-    """
-    Module collect dependencies from moduleinfo files.
-    Type of dependencies:
-
-        repo: list of yum repositories to import on host before yum dependencies are installed
-        yum: resolved with yum/dnf manager
-        pip: resolved with pip manager, also version collisions are checked, if specified, \
-             if more modules specify same package, but only one specify version, this version will be taken
-        ansible_tasks: list of ansible tasks as you know it from playbooks
-
-    Example of moduleinfo file, file can contain more definitions
-
-    .. code-block:: yaml
-
-       ---
-
-       name: postgresql
-       description: Connect to PostgreSQL database
-       dependencies:
-         yum:
-           - postgresql
-         pip:
-           - psycopg2==2.7.1
-         repo:
-           - name: dummyrepo
-             baseurl: http://dummy.url/release/repo
-             gpgcheck: 0
-         ansible_tasks:
-           - name: ensure file is present
-             file: path=/etc/magic state=file
-    """
-    name = 'dep-list'
-    description = 'Collect module dependencies for citool modules'
-    options = {
-        'module-dirs': {
-            'help': 'Where to find modules'
-        },
-        'only-modules': {
-            'help': 'If specified, modules for which dependencies will beresolved'
-        },
-        'output': {
-            'help': 'Basename of the output file: ``FILE``-requirements.yml and ``FILE``-tasks.yml will be created.',
-            'metavar': 'FILE'
-        }
-    }
-    shared_functions = ['prepare_dependencies']
-
-    @staticmethod
-    def collect_moduleinfo_files(path):
-        collection = []
-        for root, _, files in os.walk(path):
-            for filename in files:
-                if filename.endswith('.moduleinfo'):
-                    collection.append(os.path.join(root, filename))
-        return collection
-
-    def load_moduleinfo_files(self, files):
-        loaded = ModuleInfoGroup(self.logger)
-        for filename in files:
-            yaml = gluetool.utils.load_yaml(filename)
-            try:
-                item = ModuleInfo(yaml)
-                if loaded.add_moduleinfo(item):
-                    self.info("Module info '{}' loaded".format(item.name))
-                    self.debug("Module: '{}'\nDescription: '{}'\nDependencies:\n'{}'".format(
-                        item.name, item.description, gluetool.log.format_dict(item.dependencies)
-                    ))
-            except KeyError:
-                self.warn("Module info '{}' cannot be properly loaded".format(filename))
-        return loaded
-
-    def prepare_dependencies(self, module_dirs, only_modules):
-        dirs = module_dirs
-
-        files = []
-        for path in dirs:
-            self.info("Searching for modules within path '{}'".format(path))
-            files += self.collect_moduleinfo_files(path)
-            self.debug(files)
-        loaded = self.load_moduleinfo_files(files)
-        dependencies = loaded.get_dependencies(only_modules)
-        self.debug('Resulting dependencies:\n{}'.format(gluetool.log.format_dict(dependencies)))
-
-        return dependencies
-
-    def execute(self):
-        dirs = self.option('module-dirs')
-        if dirs:
-            dirs = [s.strip() for s in dirs.split(',')]
-        modules = self.option('only-modules')
-        if modules:
-            if modules == '*':
-                modules = None
-            else:
-                modules = [s.strip() for s in modules.split(',')]
-        outfile = self.option('output')
-        if dirs and outfile:
-            dependencies = self.prepare_dependencies(dirs, modules)
-
-            requirements_file, tasks_file = '{}-requirements.yml'.format(outfile), '{}-tasks.yml'.format(outfile)
-
-            gluetool.utils.dump_yaml({
-                'compose_requirements': {
-                    'repositories': dependencies['repo'],
-                    'pip': dependencies['pip'],
-                    'yum': dependencies['yum']
-                }
-            }, requirements_file)
-
-            self.info("Requirements written into '{}'".format(requirements_file))
-
-            gluetool.utils.dump_yaml(dependencies['ansible_tasks'], tasks_file)
-
-            self.info("Requirements written into '{}'".format(tasks_file))
-
-        else:
-            self.info('To execute provide --module-dirs and --output, skipping dependencies generation in execute')