qontract-reconcile 0.10.1rc193__py3-none-any.whl → 0.10.1rc195__py3-none-any.whl

{qontract_reconcile-0.10.1rc193.dist-info → qontract_reconcile-0.10.1rc195.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc193
+Version: 0.10.1rc195
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc193.dist-info → qontract_reconcile-0.10.1rc195.dist-info}/RECORD

@@ -41,7 +41,7 @@ reconcile/jenkins_webhooks.py,sha256=j8vhJMWcRhOdc9XzRSm0CPj84jsF3e4Syjm7r1BIsDE
 reconcile/jenkins_webhooks_cleaner.py,sha256=JsN_NVPfZJwv1JtSzZXDIHUqGiefL-DRffFnDGau9aY,1539
 reconcile/jenkins_worker_fleets.py,sha256=nwlsLSly85RukBGCVsEpoc92BVxvCjJetFlpNC5OwPc,5282
 reconcile/jira_permissions_validator.py,sha256=Iul5-2_QgQ8joGfP542UQYA0Y5Qm5chvdRzUltCo_yM,1434
-reconcile/jira_watcher.py,sha256=epeKY1ljMMKw1lTS50cIwCefEiSBtcywnejCNzasdfU,3653
+reconcile/jira_watcher.py,sha256=CsSipmf7ThuNzj9Xy41vedPwJ-axxy47OPXMCZqZyks,5634
 reconcile/ldap_users.py,sha256=uEWQ0V41tN9KCZi4ZKPamjrJ6djSpdpvDBo7yJ0e7ZI,3008
 reconcile/mr_client_gateway.py,sha256=WhjMd-sIXDFCV8-rt8CEjurJ5OYB1pOD0K3o0tZRXQg,1885
 reconcile/ocm_additional_routers.py,sha256=KfcFDVbNoc6n5dHWjYdAf1_DiVqVG6Tw23WLKoV8cdg,3306
@@ -87,7 +87,7 @@ reconcile/quay_mirror.py,sha256=dOKvUeiXXs8ryqLSxgOITQPMOnx0bTZakFKb4UqhaQc,1340
 reconcile/quay_mirror_org.py,sha256=E1OdRe-ppxTkNCwu20iVRhEdG1fPDBroLY02NgiMN7c,10381
 reconcile/quay_permissions.py,sha256=_3PCWjNWoU7VHlYgHzUevvL_jJmEMsWfXV_nzjeiyhU,4099
 reconcile/quay_repos.py,sha256=7609RBVQihis96FNOOe-i9tCTYwcTVy4WpKAL6HpnkU,7031
-reconcile/queries.py,sha256=_UPrClhUh5c7fwzW3984RhLxq3XziGZ-GvWDiz7DbAE,48876
+reconcile/queries.py,sha256=F6Q5RRWovTGh_s7hiqKThttjhYySLPjLSJn5HdaJ864,47914
 reconcile/query_validator.py,sha256=oLEZIAsQCzxmmZ7b9dSw-OKuEjpI1dbVu4XfCfjpmi8,1503
 reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
 reconcile/resource_scraper.py,sha256=vo1N9vLJCYWvXlTwFRIpEuWjx_39ZV9zxJlpoPq4g3U,2330
@@ -212,6 +212,8 @@ reconcile/gql_definitions/jenkins_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-
 reconcile/gql_definitions/jenkins_configs/jenkins_configs.py,sha256=8U4S1iRIQ86VTrKGd5k0xgi-DlwhYKOlevK_OWIKC98,2784
 reconcile/gql_definitions/jira_permissions_validator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/jira_permissions_validator/jira_boards_for_permissions_validator.py,sha256=5iHCIIfZYKopghfc7-BpoG2uvxNrm_qYwzdsRI_kVig,2266
+reconcile/gql_definitions/jira_watcher/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/jira_watcher/jira_watcher_boards.py,sha256=BNB4DHdviKC3dKRQ3Dzs6P-mHWUfkRK7KlQ90GhuK4o,4011
 reconcile/gql_definitions/jumphosts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/jumphosts/jumphosts.py,sha256=V0K6V9_O8LFztCcBtJfaZ9PMkpXF2N43ju_R1ajTzho,2384
 reconcile/gql_definitions/ocp_release_mirror/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -509,7 +511,7 @@ reconcile/utils/imap_client.py,sha256=byFAJATbITJPsGECSbvXBOcCnoeTUpDFiEjzOAxLm_
 reconcile/utils/instrumented_wrappers.py,sha256=eVwMoa6FCrYxLv3RML3WpZF9qKVfCTjMxphgVXG03OM,1073
 reconcile/utils/jenkins_api.py,sha256=BlHOUDG3Nn5K4qPU-js2ewV_IhciISfCnQwfaZlewu8,7054
 reconcile/utils/jinja2_ext.py,sha256=l628RR9r9dAGBWLVegoCbSqnjojeizNGiq9Cstt02nE,1129
-reconcile/utils/jira_client.py,sha256=pQw4LKZL5d-Guaj4BMiIVrL6EZsmAMYII1-b8xYZ8Yk,5021
+reconcile/utils/jira_client.py,sha256=r5KsAD-Kxj6pUbwhlq1ME_B_tbgkC5vwf4w_Nk_dksI,5022
 reconcile/utils/jjb_client.py,sha256=bj40UNRbvQ93K-1tndzJI6Drt_sebRXq-qUpfDx2ozw,14496
 reconcile/utils/jsonpath.py,sha256=NRpAEijKN4cMDjo7qivNPqpm0__GQQ1TiE0PBEBO45s,5572
 reconcile/utils/jump_host.py,sha256=yJZBRnoCJ-fknhT9f_cli_1J5hBRh1PsRVDyDWpjhws,4935
@@ -523,7 +525,7 @@ reconcile/utils/oc.py,sha256=AIkBSXLnsJCE_9HCMN66-QK359pMBNPs4Q2XPI9zjyU,64780
 reconcile/utils/oc_connection_parameters.py,sha256=mUP0WtpozdFDufcdNKy2OGweRvI3Faq8YLiR0ngJYes,9799
 reconcile/utils/oc_filters.py,sha256=RWn8pC5A7ZZT7C6WPq9bOw5KBNkiAb5puFSr_FpdAf8,1358
 reconcile/utils/oc_map.py,sha256=nT69J5pdPeIDnIYjD9fwY6GkE3BMQCf-AF0rmHJuUNw,9068
-reconcile/utils/ocm_base_client.py,sha256=bUE9uUJVVwHaNNKdL86pIA_GnpHraXvtS-3KMRmszps,5177
+reconcile/utils/ocm_base_client.py,sha256=7WHzSQ2j3FhZ1d8DA9v2szfg0tpzavfGCaVgGkvoquY,5003
 reconcile/utils/openshift_resource.py,sha256=fLqURJcjrg8kd9jxbJ91fJBB5woiB4KT_wMKyADzffo,24214
 reconcile/utils/openssl.py,sha256=QVvhzhpChq_4Daf_5wE1qeZJr4thg3DDjJPn4bOPD4E,365
 reconcile/utils/output.py,sha256=htcMXMe0y2dNnwu8MW8x0JZ5YBaEJRy5w4tR8yLF0OU,1738
@@ -612,8 +614,8 @@ tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_qontract_cli.py,sha256=awwTHEc2DWlykuqGIYM0WOBoSL0KRnOraCLk3C7izis,1401
 tools/test/test_sd_app_sre_alert_report.py,sha256=JeLhgzpKCPgLvptwg_4ZvJHLVWKNG1T5845HXTkMBxA,1826
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc193.dist-info/METADATA,sha256=sC7-hMIzVi8e6YolhzhJwX5zi-t-xPd6RjWyXH1kh50,2320
-qontract_reconcile-0.10.1rc193.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-qontract_reconcile-0.10.1rc193.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
-qontract_reconcile-0.10.1rc193.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc193.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc195.dist-info/METADATA,sha256=PUU-Wdx0R2NJQ6PZRfei54GtnyNCp2GZsMVHQn2PXvg,2320
+qontract_reconcile-0.10.1rc195.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+qontract_reconcile-0.10.1rc195.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
+qontract_reconcile-0.10.1rc195.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc195.dist-info/RECORD,,

reconcile/gql_definitions/jira_watcher/jira_watcher_boards.py

@@ -0,0 +1,149 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query JiraWatcherBoards {
+  jira_boards: jira_boards_v1 {
+    path
+    name
+    server {
+      serverUrl
+      token {
+        ... VaultSecret
+      }
+    }
+    slack {
+      workspace {
+        name
+        integrations {
+          name
+          token {
+            ... VaultSecret
+          }
+          channel
+          icon_emoji
+          username
+        }
+        api_client {
+          global {
+            max_retries
+            timeout
+          }
+          methods {
+            name
+            args
+          }
+        }
+      }
+      channel
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union = True
+        extra = Extra.forbid
+
+
+class JiraServerV1(ConfiguredBaseModel):
+    server_url: str = Field(..., alias="serverUrl")
+    token: VaultSecret = Field(..., alias="token")
+
+
+class SlackWorkspaceIntegrationV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    token: VaultSecret = Field(..., alias="token")
+    channel: str = Field(..., alias="channel")
+    icon_emoji: str = Field(..., alias="icon_emoji")
+    username: str = Field(..., alias="username")
+
+
+class SlackWorkspaceApiClientGlobalConfigV1(ConfiguredBaseModel):
+    max_retries: Optional[int] = Field(..., alias="max_retries")
+    timeout: Optional[int] = Field(..., alias="timeout")
+
+
+class SlackWorkspaceApiClientMethodConfigV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    args: Json = Field(..., alias="args")
+
+
+class SlackWorkspaceApiClientV1(ConfiguredBaseModel):
+    q_global: Optional[SlackWorkspaceApiClientGlobalConfigV1] = Field(
+        ..., alias="global"
+    )
+    methods: Optional[list[SlackWorkspaceApiClientMethodConfigV1]] = Field(
+        ..., alias="methods"
+    )
+
+
+class SlackWorkspaceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    integrations: Optional[list[SlackWorkspaceIntegrationV1]] = Field(
+        ..., alias="integrations"
+    )
+    api_client: Optional[SlackWorkspaceApiClientV1] = Field(..., alias="api_client")
+
+
+class SlackOutputV1(ConfiguredBaseModel):
+    workspace: SlackWorkspaceV1 = Field(..., alias="workspace")
+    channel: Optional[str] = Field(..., alias="channel")
+
+
+class JiraBoardV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    name: str = Field(..., alias="name")
+    server: JiraServerV1 = Field(..., alias="server")
+    slack: Optional[SlackOutputV1] = Field(..., alias="slack")
+
+
+class JiraWatcherBoardsQueryData(ConfiguredBaseModel):
+    jira_boards: Optional[list[JiraBoardV1]] = Field(..., alias="jira_boards")
+
+
+def query(query_func: Callable, **kwargs: Any) -> JiraWatcherBoardsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        JiraWatcherBoardsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return JiraWatcherBoardsQueryData(**raw_data)

reconcile/jira_watcher.py

@@ -1,10 +1,34 @@
 import logging
+from collections.abc import (
+    Callable,
+    Iterable,
+    Mapping,
+)
+from typing import Optional
 
-from reconcile import queries
-from reconcile.slack_base import slackapi_from_slack_workspace
+from reconcile.gql_definitions.common.jira_settings import AppInterfaceSettingsV1
+from reconcile.gql_definitions.jira_watcher.jira_watcher_boards import (
+    JiraBoardV1,
+    SlackOutputV1,
+)
+from reconcile.gql_definitions.jira_watcher.jira_watcher_boards import (
+    query as query_jira_boards,
+)
+from reconcile.slack_base import (
+    SlackApi,
+    slackapi_from_slack_workspace,
+)
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.jira_settings import get_jira_settings
+from reconcile.utils import gql
 from reconcile.utils.defer import defer
 from reconcile.utils.jira_client import JiraClient
-from reconcile.utils.secret_reader import SecretReader
+from reconcile.utils.secret_reader import (
+    SecretReaderBase,
+    create_secret_reader,
+)
 from reconcile.utils.sharding import is_in_shard_round_robin
 from reconcile.utils.state import (
     State,
@@ -14,8 +38,18 @@ from reconcile.utils.state import (
 QONTRACT_INTEGRATION = "jira-watcher"
 
 
-def fetch_current_state(jira_board, settings):
-    jira = JiraClient(jira_board, settings=settings)
+def fetch_current_state(
+    jira_board: JiraBoardV1,
+    settings: AppInterfaceSettingsV1,
+    secret_reader: SecretReaderBase,
+) -> tuple[JiraClient, dict[str, dict]]:
+    token = secret_reader.read_secret(jira_board.server.token)
+    jira = JiraClient.create(
+        project_name=jira_board.name,
+        token=token,
+        server_url=jira_board.server.server_url,
+        jira_watcher_settings=settings.jira_watcher,
+    )
     issues = jira.get_issues(fields=["key", "status", "summary"])
     return jira, {
         issue.key: {"status": issue.fields.status.name, "summary": issue.fields.summary}
@@ -23,11 +57,18 @@ def fetch_current_state(jira_board, settings):
     }
 
 
-def fetch_previous_state(state, project):
+def fetch_previous_state(state: State, project: str) -> dict[str, dict]:
     return state.get(project, {})
 
 
-def format_message(server, key, data, event, previous_state=None, current_state=None):
+def format_message(
+    server: str,
+    key: str,
+    data: Mapping,
+    event: str,
+    previous_state: Optional[Mapping] = None,
+    current_state: Optional[Mapping] = None,
+) -> str:
     summary = data["summary"]
     info = (
         ": {} -> {}".format(previous_state["status"], current_state["status"])
@@ -38,8 +79,12 @@ def format_message(server, key, data, event, previous_state=None, current_state=
     return "{} ({}) {}{}".format(url, summary, event, info)
 
 
-def calculate_diff(server, current_state, previous_state):
-    messages = []
+def calculate_diff(
+    server: str,
+    current_state: Mapping[str, Mapping],
+    previous_state: Mapping[str, Mapping],
+) -> list[str]:
+    messages: list[str] = []
     new_issues = [
         format_message(server, key, data, "created")
         for key, data in current_state.items()
@@ -66,52 +111,74 @@ def calculate_diff(server, current_state, previous_state):
     return messages
 
 
-def init_slack(jira_board):
-    secret_reader = SecretReader(queries.get_secret_reader_settings())
-    slack_info = jira_board["slack"]
-
+def init_slack(slack: SlackOutputV1, secret_reader: SecretReaderBase) -> SlackApi:
     return slackapi_from_slack_workspace(
-        slack_info,
+        slack.dict(by_alias=True),
         secret_reader,
         QONTRACT_INTEGRATION,
-        channel=slack_info.get("channel"),
+        channel=slack.channel,
         init_usergroups=False,
     )
 
 
-def act(dry_run, jira_board, diffs):
+def act(
+    dry_run: bool,
+    slack: SlackOutputV1,
+    diffs: Iterable[str],
+    secret_reader: SecretReaderBase,
+) -> None:
+    slack_api: Optional[SlackApi] = None
     if not dry_run and diffs:
-        slack = init_slack(jira_board)
+        slack_api = init_slack(slack=slack, secret_reader=secret_reader)
 
-    for diff in reversed(diffs):
+    for diff in reversed(list(diffs)):
         logging.info(diff)
         if not dry_run:
-            slack.chat_post_message(diff)
+            if not slack_api:
+                raise RuntimeError("Slack API not initialized")
+            slack_api.chat_post_message(diff)
 
 
-def write_state(state: State, project, state_to_write):
+def write_state(
+    state: State, project: str, state_to_write: Mapping[str, Mapping]
+) -> None:
     state.add(project, value=state_to_write, force=True)
 
 
 @defer
-def run(dry_run, defer):
-    jira_boards = [j for j in queries.get_jira_boards() if j.get("slack")]
-    settings = queries.get_app_interface_settings()
+def run(dry_run: bool, defer: Optional[Callable]) -> None:
+    gql_api = gql.get_api()
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    jira_boards = query_jira_boards(query_func=gql_api.query).jira_boards or []
+    settings = get_jira_settings(gql_api=gql_api)
     state = init_state(integration=QONTRACT_INTEGRATION)
-    defer(state.cleanup)
+    if defer:
+        defer(state.cleanup)
     for index, jira_board in enumerate(jira_boards):
-        if not is_in_shard_round_robin(jira_board["name"], index):
+        if not jira_board.slack:
+            continue
+        if not is_in_shard_round_robin(jira_board.name, index):
             continue
-        jira, current_state = fetch_current_state(jira_board, settings)
+        jira, current_state = fetch_current_state(
+            jira_board=jira_board, settings=settings, secret_reader=secret_reader
+        )
         if not current_state:
             logging.warning(
                 "not acting on empty Jira boards. "
                 + "please create a ticket to get started."
             )
             continue
-        previous_state = fetch_previous_state(state, jira.project)
+        previous_state = fetch_previous_state(state=state, project=jira.project)
         if previous_state:
-            diffs = calculate_diff(jira.server, current_state, previous_state)
-            act(dry_run, jira_board, diffs)
+            diffs = calculate_diff(
+                jira_board.server.server_url, current_state, previous_state
+            )
+            act(
+                dry_run=dry_run,
+                slack=jira_board.slack,
+                diffs=diffs,
+                secret_reader=secret_reader,
+            )
         if not dry_run:
             write_state(state, jira.project, current_state)

reconcile/queries.py

@@ -2171,62 +2171,6 @@ def get_pipelines_providers():
     return pipelines_providers
 
 
-JIRA_BOARDS_QUERY = """
-{
-  jira_boards: jira_boards_v1 {
-    path
-    name
-    server {
-      serverUrl
-      token {
-        path
-        field
-        version
-        format
-      }
-    }
-    {% if with_slack %}
-    slack {
-      workspace {
-        name
-        integrations {
-          name
-          token {
-            path
-            field
-            version
-            format
-          }
-          channel
-          icon_emoji
-          username
-        }
-        api_client {
-          global {
-            max_retries
-            timeout
-          }
-          methods {
-            name
-            args
-          }
-        }
-      }
-      channel
-    }
-    {% endif %}
-  }
-}
-"""
-
-
-def get_jira_boards(with_slack: Optional[bool] = True):
-    """Returns Jira boards resources defined in app-interface"""
-    gqlapi = gql.get_api()
-    query = Template(JIRA_BOARDS_QUERY).render(with_slack=with_slack)
-    return gqlapi.query(query)["jira_boards"]
-
-
 # Use APATH as the place holder because Python formatting interferes
 # with graphql use of curly braces
 JIRA_BOARDS_QUICK_QUERY = """

reconcile/utils/jira_client.py

@@ -97,7 +97,7 @@ class JiraClient:
             project=project_name,
         )
 
-    def get_issues(self, fields: Optional[Mapping] = None) -> list[Issue]:
+    def get_issues(self, fields: Optional[Iterable] = None) -> list[Issue]:
         block_size = 100
         block_num = 0
 

reconcile/utils/ocm_base_client.py

@@ -89,23 +89,17 @@ class OCMBaseClient:
             params_copy = params.copy()
         params_copy["size"] = max_page_size
 
-        # fetch pages
-        records_seen = 0
         while True:
-            rs = self.get(api_path, params=params)
+            rs = self.get(api_path, params=params_copy)
             for item in rs.get("items", []):
                 yield item
-            total_records = rs.get("total", 0)
             current_page = rs.get("page", 0)
             records_on_page = rs.get("size", len(rs.get("items", [])))
-            records_seen += records_on_page
-            if total_records > records_seen and (
-                max_pages and max_pages < current_page
-            ):
-                # more page available
-                params_copy["page"] = current_page + 1
-            else:
+            if records_on_page < max_page_size:
                 return
+            if max_pages is not None and current_page >= max_pages:
+                return
+            params_copy["page"] = current_page + 1
 
     def post(
         self,