qontract-reconcile 0.10.1rc1201__py3-none-any.whl → 0.10.2.dev1__py3-none-any.whl

reconcile/run_integration.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+
+import logging
+import os
+import sys
+import time
+from collections.abc import Callable
+from importlib import metadata
+
+import click
+from prometheus_client import (
+    push_to_gateway,
+    start_http_server,
+)
+from prometheus_client.exposition import basic_auth_handler
+
+from reconcile.status import ExitCodes
+from reconcile.utils.metrics import (
+    execution_counter,
+    pushgateway_registry,
+    pushgateway_run_status,
+    pushgateway_run_time,
+    run_status,
+    run_time,
+)
+from reconcile.utils.runtime.environment import (
+    LOG_DATEFMT,
+    log_fmt,
+)
+
+SHARDS = int(os.environ.get("SHARDS", 1))
+SHARD_ID = int(os.environ.get("SHARD_ID", 0))
+SHARD_ID_LABEL = os.environ.get("SHARD_KEY", f"{SHARD_ID}-{SHARDS}")
+PREFIX_LOG_LEVEL = os.environ.get("PREFIX_LOG_LEVEL", "false")
+
+INTEGRATION_NAME = os.environ.get("INTEGRATION_NAME")
+COMMAND_NAME = os.environ.get("COMMAND_NAME", "qontract-reconcile")
+
+RUN_ONCE = os.environ.get("RUN_ONCE")
+DRY_RUN = (
+    os.environ.get("MANAGER_DRY_RUN")
+    if INTEGRATION_NAME == "integrations-manager"
+    else os.environ.get("DRY_RUN")
+)
+INTEGRATION_EXTRA_ARGS = os.environ.get("INTEGRATION_EXTRA_ARGS")
+CONFIG = os.environ.get("CONFIG", "/config/config.toml")
+PROMETHEUS_PORT = os.environ.get("PROMETHEUS_PORT", 9090)
+
+LOG_FILE = os.environ.get("LOG_FILE")
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+SLEEP_DURATION_SECS = os.environ.get("SLEEP_DURATION_SECS", 600)
+SLEEP_ON_ERROR = os.environ.get("SLEEP_ON_ERROR", 10)
+
+PUSHGATEWAY_ENABLED = os.environ.get("PUSHGATEWAY_ENABLED", False)
+
+LOG = logging.getLogger(__name__)
+
+# Messages to stdout
+STREAM_HANDLER = logging.StreamHandler(sys.stdout)
+STREAM_HANDLER.setFormatter(
+    logging.Formatter(fmt=log_fmt(dry_run_option=DRY_RUN), datefmt=LOG_DATEFMT)
+)
+HANDLERS = [STREAM_HANDLER]
+
+# Messages to the log file
+if LOG_FILE is not None:
+    FILE_HANDLER = logging.FileHandler(LOG_FILE)
+    logFileFormat = "%(message)s"
+    if PREFIX_LOG_LEVEL == "true":
+        logFileFormat = "[%(levelname)s] %(message)s"
+    FILE_HANDLER.setFormatter(logging.Formatter(fmt=logFileFormat))
+    HANDLERS.append(FILE_HANDLER)  # type: ignore
+
+# Setting up the root logger
+logging.basicConfig(level=LOG_LEVEL, handlers=HANDLERS)
+
+
+class PushgatewayBadConfigError(Exception):
+    pass
+
+
+def _parse_dry_run_flag(dry_run: str | None) -> str | None:
+    dry_run_options = ["--dry-run", "--no-dry-run"]
+    if dry_run is not None and dry_run not in dry_run_options:
+        msg = (
+            f'Invalid DRY_RUN option given: "{dry_run}".'
+            f"Only the following options are allowed: {dry_run_options}"
+        )
+        logging.error(msg)
+        raise ValueError(msg)
+    return dry_run if dry_run else None
+
+
+def build_entry_point_args(
+    command: click.Command,
+    config: str,
+    dry_run: str | None,
+    integration_name: str,
+    extra_args: str | None,
+) -> list[str]:
+    args = ["--config", config]
+    if dry_run_flag := _parse_dry_run_flag(dry_run):
+        args.append(dry_run_flag)
+
+    # if the integration_name is a known sub command,
+    # we add it right before the extra_args
+    if (
+        integration_name
+        and isinstance(command, click.MultiCommand)
+        and command.get_command(None, integration_name)  # type: ignore
+    ):
+        args.append(integration_name)
+
+    if extra_args is not None:
+        args.extend(extra_args.split())
+    return args
+
+
+def build_entry_point_func(command_name: str) -> click.Command:
+    """
+    Use the entry point information from setup.py to
+    find the function to invoke for a command.
+    """
+    console_script_entry_points = {
+        ep.name: ep for ep in metadata.entry_points().select(group="console_scripts")
+    }
+    entry_point: metadata.EntryPoint | None = console_script_entry_points.get(
+        command_name
+    )
+    if entry_point:
+        return entry_point.load()
+    raise ValueError(
+        f"Command {command_name} unknown."
+        f"Have a look at setup.py for valid entry points."
+    )
+
+
+def _get_pushgateway_env_vars() -> dict[str, str]:
+    env = {}
+    missing_vars = []
+    for var in ["PUSHGATEWAY_USERNAME", "PUSHGATEWAY_PASSWORD", "PUSHGATEWAY_URL"]:
+        value = os.environ.get(var)
+        if not value:
+            missing_vars.append(var)
+            continue
+
+        env[var] = value
+
+    if missing_vars:
+        missing_str = ", ".join(missing_vars)
+        raise PushgatewayBadConfigError(
+            f"Failed to check env variables to configure Pushgateway: {missing_str}"
+        )
+
+    return env
+
+
+def _push_gateway_basic_auth_handler(
+    url: str,
+    method: str,
+    timeout: float | None,
+    headers: list[tuple[str, str]],
+    data: bytes,
+) -> Callable[[], None]:
+    username = os.environ.get("PUSHGATEWAY_USERNAME")
+    password = os.environ.get("PUSHGATEWAY_PASSWORD")
+
+    # We should not get here, but this will make mypy happy
+    if not username or not password:
+        raise PushgatewayBadConfigError(
+            "Failed to check env variables to configure Pushgateway."
+        )
+
+    return basic_auth_handler(url, method, timeout, headers, data, username, password)
+
+
+def main() -> None:
+    """
+    This entry point script expects certain env variables
+    * COMMAND_NAME (optional, defaults to qontract-reconcile)
+      an entry point as defined in setup.py must be a click.Command
+    * INTEGRATION_NAME
+      used as name for the subcommand for command if present as a subcommand
+      on the click command
+    * INTEGRATION_EXTRA_ARGS (optional)
+      space separated list of arguments that will be passed to the command
+      or subcommand
+    * CONFIG
+      path to the config toml file
+    * LOG_LEVEL
+      Log level (defaults to INFO)
+    * LOG_FILE
+      path for the logfile to write to
+    * DRY_RUN (optional)
+      this is not a boolean but must contain the actual dry-run flag value,
+      so --dry-run or --no-dry-run
+    * RUN_ONCE (optional)
+      if 'true', execute the integration once and exit
+      otherwise run the integration in a loop controlled by SLEEP_DURATION_SECS
+      and SLEEP_ON_ERROR
+    * SLEEP_DURATION_SECS (default 600)
+      amount of seconds to sleep between successful integration runs
+    * SLEEP_ON_ERROR (default 10)
+      amount of seconds to sleep before another integration run is started
+    * PUSHGATEWAY_ENABLED (defaults to false)
+      send metrics to a Prometheus Pushgateway after the run. In expects
+      "PUSHGATEWAY_USERNAME", "PUSHGATEWAY_PASSWORD" and "PUSHGATEWAY_URL" to be defined.
+
+
+    Based on those variables, the following command will be executed
+      $COMMAND --config $CONFIG $DRY_RUN $INTEGRATION_NAME \
+        $INTEGRATION_EXTRA_ARGS
+    """
+    if len(sys.argv) > 1 and sys.argv[1] == "--help":
+        print(main.__doc__)
+        sys.exit(0)
+
+    if not INTEGRATION_NAME:
+        raise ValueError("INTEGRATION_NAME env variable is required")
+
+    start_http_server(int(PROMETHEUS_PORT))
+
+    command = build_entry_point_func(COMMAND_NAME)
+    while True:
+        args = build_entry_point_args(
+            command, CONFIG, DRY_RUN, INTEGRATION_NAME, INTEGRATION_EXTRA_ARGS
+        )
+        sleep = SLEEP_DURATION_SECS
+        start_time = time.monotonic()
+        # Running the integration via Click, so we don't have to replicate
+        # the CLI logic here
+        execution_counter.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).inc()
+        try:
+            with command.make_context(info_name=COMMAND_NAME, args=args) as ctx:  # type: ignore
+                ctx.ensure_object(dict)
+                command.invoke(ctx)
+                return_code = 0
+        # This is for when the integration explicitly
+        # calls sys.exit(N)
+        except SystemExit as exc_obj:
+            return_code = int(exc_obj.code)  # type: ignore[arg-type]
+        # We have to be generic since we don't know what can happen
+        # in the integrations, but we want to continue the loop anyway
+        except Exception:
+            sleep = SLEEP_ON_ERROR
+            LOG.exception(f"Error running {COMMAND_NAME}")
+            return_code = ExitCodes.ERROR
+
+        time_spent = time.monotonic() - start_time
+
+        run_time.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).set(time_spent)
+        run_status.labels(
+            integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+        ).set(return_code)
+
+        if PUSHGATEWAY_ENABLED:
+            try:
+                env = _get_pushgateway_env_vars()
+                pushgateway_run_time.labels(
+                    integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+                ).set(time_spent)
+                pushgateway_run_status.labels(
+                    integration=INTEGRATION_NAME, shards=SHARDS, shard_id=SHARD_ID_LABEL
+                ).set(return_code)
+
+                grouping_key = {
+                    "integration": INTEGRATION_NAME,
+                    "shards": SHARDS,
+                    "shard_id": SHARD_ID_LABEL,
+                }
+                push_to_gateway(
+                    gateway=env["PUSHGATEWAY_URL"],
+                    job="qontract-reconcile",
+                    registry=pushgateway_registry,
+                    handler=_push_gateway_basic_auth_handler,
+                    grouping_key=grouping_key,
+                )
+            except PushgatewayBadConfigError as err:
+                LOG.exception(f"Error pushing to PushGateway: {err}")
+                return_code = ExitCodes.ERROR
+
+        if RUN_ONCE:
+            sys.exit(return_code)
+
+        time.sleep(int(sleep))
+
+
+if __name__ == "__main__":
+    main()

reconcile/utils/binary.py

@@ -1,6 +1,6 @@
 import re
+import shutil
 import subprocess
-from distutils.spawn import find_executable  # pylint: disable=deprecated-module
 from functools import wraps
 
 
@@ -13,7 +13,7 @@ def binary(binaries=None):
         @wraps(f)
         def f_binary(*args, **kwargs):
             for b in binaries:
-                if not find_executable(b):
+                if not shutil.which(b):
                     raise Exception(
                         f"Aborting: Could not find binary: {b}. "
                         + f"Hint: https://command-not-found.com/{b}"

reconcile/utils/mr/README.md

@@ -0,0 +1,198 @@
+# Merge Requests
+
+Module responsible for abstracting a Gitlab Merge Request.
+
+## Creating New MergeRequest Classes
+
+Each type of merge request requires a class that inherits
+from reconcile.utils.mr.base.MergeRequestBase.
+
+That class has to comply with the specification below.
+
+### Class Name
+
+The class name is given by the class variable `name`. That name will be
+used all around, the most important place being the SQS Message parameter
+`pr_type`, used later to create an instance of the same class.
+
+While the name can be anything it is mandatory that it is defined.
+
+### Class Initialization
+
+The `__init__` method has to call `super().__init__()` right after the minimum
+parameters initialization. That is required to compose the SQS Message that
+will later be used to create a new instance of the same class.
+
+### Class Methods
+
+The minimum methods that have to be defined are:
+
+* `title`: a property that is used to give the Gitlab Merge Request a title.
+  It can also be used for building commit messages or as content to committed
+  files.
+* `description`: a description for the Merge Request.
+* `process`: this method is called when submitting the Merge Request to gitlab.
+  it's the place for the merge request changes, like creating, updating and
+  deleting files. The `process` method is called after the local branch is
+  created and right after it the `gitlab_cli.project.mergerequests.create()`
+  is called.
+
+### Example
+
+This is an example of a minimum implementation for a new MergeRequest class:
+
+```python
+from reconcile.utils.mr.base import MergeRequestBase
+
+
+class CreateDeleteUser(MergeRequestBase):
+
+    name = 'create_delete_user_mr'
+
+    def __init__(self, username, paths):
+        self.username = username
+        self.paths = paths
+
+        # Called right after the minimum parameters to recreate
+        # an instance with the same data. This is important for
+        # building up the SQS message payload.
+        super().__init__()
+
+    @property
+    def title(self) -> str:
+        return f'[{self.name}] delete user {self.username}'
+
+    @property
+    def description(self) -> str:
+        return f'delete user {self.username}'
+
+    def process(self, gitlab_cli):
+        for path in self.paths:
+            gitlab_cli.delete_file(branch_name=self.branch,
+                                   file_path=path,
+                                   commit_message=self.title)
+```
+
+## Sending MRs to SQS
+
+To send a Merge Request to SQS, create the corresponding MergeRequest object:
+
+```python
+from reconcile.utils.mr import CreateAppInterfaceNotificator
+
+
+notification = {
+    'notification_type': 'Outage',
+    'description': 'The AppSRE team is currently investigating an outage',
+    'short_description': 'Short Description',
+}
+
+merge_request = CreateAppInterfaceNotificator(notification=notification)
+
+```
+
+then create the SQS Client instance:
+
+```python
+from reconcile import queries
+
+from reconcile.utils.sqs_gateway import SQSGateway
+from reconcile.utils.secret_reader import SecretReader
+
+
+accounts = queries.get_queue_aws_accounts()
+secretReader = SecretReader(queries.get_secret_reader_settings())
+sqs_cli = SQSGateway(accounts, secret_reader=secret_reader)
+```
+
+and then submit the merge request to the SQS:
+
+```python
+merge_request.submit_to_sqs(sqs_cli=sqs_cli)
+```
+
+## Sending MRs to Gitlab
+
+To get the message from SQS and use it for sending a Merge Request to gitlab,
+first get the SQS messages:
+
+
+```python
+from reconcile import queries
+
+from reconcile.utils.sqs_gateway import SQSGateway
+from reconcile.utils.secret_reader import SecretReader
+
+
+accounts = queries.get_queue_aws_accounts()
+settings = queries.get_app_interface_settings()
+
+secretReader = SecretReader(queries.get_secret_reader_settings())
+sqs_cli = SQSGateway(accounts, secret_reader=secret_reader)
+messages = sqs_cli.receive_messages()
+```
+
+then create the Gitlab client instance:
+
+```python
+from reconcile.utils.gitlab_api import GitLabApi
+
+instance = queries.get_gitlab_instance()
+saas_files = queries.get_saas_files_minimal()
+gitlab_cli = GitLabApi(instance, project_id=gitlab_project_id,
+                       settings=settings)
+```
+
+and then loop the messages, creating the MergeRequest objects and submitting
+the merge requests:
+
+```python
+from reconcile.utils.mr import init_from_sqs_message
+
+for message in messages:
+    receipt_handle, body = message[0], message[1]
+    merge_request = init_from_sqs_message(body)
+    merge_request.submit_to_gitlab(gitlab_cli=gitlab_cli)
+    sqs_cli.delete_message(receipt_handle)
+```
+
+## Using the MR Module for Qontract-reconcile Integrations
+
+A given integration might be executed in different environments, like:
+
+* Developer local machine.
+* OpenShift cluster outside the VPN.
+* OpenShift cluster inside the VPN.
+* Jenkins periodic job.
+
+Because we don't want the integrations to care if they are running inside or
+outside the VPN, we use the `reconcile.mr_client_gateway.init()` to get the
+on of SQS or GitLab client, according to the App Interface settings. Example:
+
+```python
+from reconcile import mr_client_gateway
+from reconcile.utils.mr import CreateDeleteAwsAccessKey
+
+
+def run(dry_run, gitlab_project_id):
+    ...
+    mr = CreateDeleteAwsAccessKey(...)
+    with mr_client_gateway.init(gitlab_project_id=gitlab_project_id) as mr_cli:
+      mr.submit(cli=mr_cli)
+```
+
+If we want to override what's set in App Interface and get a specific client,
+say `gitlab`, we would:
+
+```python
+from reconcile import mr_client_gateway
+from reconcile.utils.mr import CreateDeleteAwsAccessKey
+
+
+def run(dry_run, gitlab_project_id):
+    ...
+    mr = CreateDeleteAwsAccessKey(...)
+    with mr_client_gateway.init(sqs_or_gitlab='gitlab',
+                           gitlab_project_id=gitlab_project_id) as mr_cli:
+      mr.submit(cli=mr_cli)
+```

reconcile/utils/oc_map.py

@@ -122,7 +122,7 @@ class OCMap:
                     connection_parameters=connection_parameters
                 )
             try:
-                oc_client: OCCli | OCLogMsg = self._oc_cls(
+                oc_client: OCCli | OCLogMsg = self._oc_cls(  # type: ignore[assignment]
                     connection_parameters=connection_parameters,
                     init_projects=self._init_projects,
                     init_api_resources=self._init_api_resources,
@@ -133,7 +133,7 @@ class OCMap:
                     cluster,
                     OCLogMsg(
                         log_level=logging.ERROR,
-                        message=f"[{cluster}]" f" is unreachable: {e}",
+                        message=f"[{cluster}] is unreachable: {e}",
                     ),
                     privileged,
                 )

qontract_reconcile-0.10.1rc1201.dist-info/METADATA

@@ -1,64 +0,0 @@
-Metadata-Version: 2.1
-Name: qontract-reconcile
-Version: 0.10.1rc1201
-Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
-Home-page: https://github.com/app-sre/qontract-reconcile
-Author: Red Hat App-SRE Team
-Author-email: sd-app-sre@redhat.com
-License: Apache License 2.0
-Classifier: Development Status :: 2 - Pre-Alpha
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.11
-Requires-Dist: sretoolbox~=2.6
-Requires-Dist: Click<9.0,>=7.0
-Requires-Dist: gql==3.1.0
-Requires-Dist: toml<0.11.0,>=0.10.0
-Requires-Dist: jsonpath-rw<1.5.0,>=1.4.0
-Requires-Dist: PyGithub<1.59,>=1.58
-Requires-Dist: hvac<0.8.0,>=0.7.0
-Requires-Dist: ldap3<2.10.0,>=2.9.1
-Requires-Dist: anymarkup<0.9.0,>=0.7.0
-Requires-Dist: python-gitlab~=4.6
-Requires-Dist: semver~=3.0
-Requires-Dist: boto3==1.34.94
-Requires-Dist: botocore==1.34.94
-Requires-Dist: urllib3~=2.2
-Requires-Dist: slack-sdk<4.0,>=3.10
-Requires-Dist: pypd<1.2.0,>=1.1.0
-Requires-Dist: Jinja2<3.2.0,>=2.10.1
-Requires-Dist: jira~=3.1
-Requires-Dist: pyOpenSSL~=23.0
-Requires-Dist: ruamel.yaml<0.18.0,>=0.17.22
-Requires-Dist: terrascript==0.9.0
-Requires-Dist: tabulate<0.9.0,>=0.8.6
-Requires-Dist: UnleashClient~=5.11
-Requires-Dist: prometheus-client~=0.8
-Requires-Dist: sentry-sdk~=2.0
-Requires-Dist: jenkins-job-builder~=4.3.0
-Requires-Dist: parse==1.18.0
-Requires-Dist: sendgrid<6.5.0,>=6.4.8
-Requires-Dist: dnspython~=2.1
-Requires-Dist: requests~=2.32
-Requires-Dist: kubernetes~=24.0
-Requires-Dist: websocket-client<0.55.0,>=0.35
-Requires-Dist: sshtunnel>=0.4.0
-Requires-Dist: croniter<1.1.0,>=1.0.15
-Requires-Dist: pydantic~=1.10.6
-Requires-Dist: MarkupSafe==2.1.1
-Requires-Dist: filetype~=1.2.0
-Requires-Dist: psycopg2~=2.9
-Requires-Dist: packaging~=23.1
-Requires-Dist: deepdiff==6.7.1
-Requires-Dist: jsonpath-ng==1.5.3
-Requires-Dist: networkx~=2.8
-Requires-Dist: rich<14.0.0,>=13.3.0
-Requires-Dist: dateparser~=1.1.7
-Requires-Dist: pyjwt~=2.7
-Requires-Dist: requests-oauthlib~=1.3
-Requires-Dist: dt==1.1.61
-Requires-Dist: jsonpatch~=1.33
-Requires-Dist: jsonpointer~=2.4
-Requires-Dist: yamllint==1.34.0
-

qontract_reconcile-0.10.1rc1201.dist-info/top_level.txt

@@ -1,3 +0,0 @@
-reconcile
-release
-tools