qontract-reconcile 0.10.2.dev24__tar.gz → 0.10.2.dev26__tar.gz

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev24
+Version: 0.10.2.dev26
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/cli.py

@@ -1158,7 +1158,10 @@ def jenkins_webhooks_cleaner(ctx):
 
 
 @integration.command(short_help="Validate permissions in Jira.")
-@click.option("--jira-board-name", help="The Jira board to act on.", default=None)
+@click.option(
+    "--jira-board-name", help="The Jira board to act on.", default=None, multiple=True
+)
+@click.option("--board-check-interval", help="Check interval in minutes", default=120)
 @enable_extended_early_exit
 @extended_early_exit_cache_ttl_seconds
 @log_cached_log_output
@@ -1166,6 +1169,7 @@ def jenkins_webhooks_cleaner(ctx):
 def jira_permissions_validator(
     ctx,
     jira_board_name,
+    board_check_interval,
     enable_extended_early_exit,
     extended_early_exit_cache_ttl_seconds,
     log_cached_log_output,
@@ -1176,6 +1180,7 @@ def jira_permissions_validator(
         reconcile.jira_permissions_validator,
         ctx.obj,
         jira_board_name=jira_board_name,
+        board_check_interval=board_check_interval,
         enable_extended_early_exit=enable_extended_early_exit,
         extended_early_exit_cache_ttl_seconds=extended_early_exit_cache_ttl_seconds,
         log_cached_log_output=log_cached_log_output,

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/gql_definitions/integrations/integrations.py

@@ -191,22 +191,6 @@ query Integrations {
             }
           }
         }
-
-        ... on JiraBoardSharding_v1 {
-            shardSpecOverrides {
-              shard {
-                name
-                disable {
-                  integrations
-                }
-              }
-              imageRef
-              disabled
-              resources {
-                ... DeployResourcesFields
-              }
-            }
-        }
       }
     }
   }
@@ -358,30 +342,10 @@ class CloudflareDNSZoneShardingV1(IntegrationShardingV1):
     shard_spec_overrides: Optional[list[CloudflareDNSZoneShardSpecOverrideV1]] = Field(..., alias="shardSpecOverrides")
 
 
-class DisableJiraBoardAutomationsV1(ConfiguredBaseModel):
-    integrations: Optional[list[str]] = Field(..., alias="integrations")
-
-
-class JiraBoardV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    disable: Optional[DisableJiraBoardAutomationsV1] = Field(..., alias="disable")
-
-
-class JiraBoardShardSpecOverrideV1(ConfiguredBaseModel):
-    shard: JiraBoardV1 = Field(..., alias="shard")
-    image_ref: Optional[str] = Field(..., alias="imageRef")
-    disabled: Optional[bool] = Field(..., alias="disabled")
-    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")
-
-
-class JiraBoardShardingV1(IntegrationShardingV1):
-    shard_spec_overrides: Optional[list[JiraBoardShardSpecOverrideV1]] = Field(..., alias="shardSpecOverrides")
-
-
 class IntegrationManagedV1(ConfiguredBaseModel):
     namespace: NamespaceV1 = Field(..., alias="namespace")
     spec: IntegrationSpecV1 = Field(..., alias="spec")
-    sharding: Optional[Union[StaticShardingV1, OpenshiftClusterShardingV1, OCMOrganizationShardingV1, AWSAccountShardingV1, CloudflareDNSZoneShardingV1, JiraBoardShardingV1, IntegrationShardingV1]] = Field(..., alias="sharding")
+    sharding: Optional[Union[StaticShardingV1, OpenshiftClusterShardingV1, OCMOrganizationShardingV1, AWSAccountShardingV1, CloudflareDNSZoneShardingV1, IntegrationShardingV1]] = Field(..., alias="sharding")
 
 
 class IntegrationV1(ConfiguredBaseModel):

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/gql_definitions/introspection.json

@@ -29766,11 +29766,6 @@
                             "kind": "OBJECT",
                             "name": "AWSAccountSharding_v1",
                             "ofType": null
-                        },
-                        {
-                            "kind": "OBJECT",
-                            "name": "JiraBoardSharding_v1",
-                            "ofType": null
                         }
                     ]
                 },
@@ -49774,122 +49769,6 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
-                {
-                    "kind": "OBJECT",
-                    "name": "JiraBoardSharding_v1",
-                    "description": null,
-                    "fields": [
-                        {
-                            "name": "strategy",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "NON_NULL",
-                                "name": null,
-                                "ofType": {
-                                    "kind": "SCALAR",
-                                    "name": "String",
-                                    "ofType": null
-                                }
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "shardSpecOverrides",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "LIST",
-                                "name": null,
-                                "ofType": {
-                                    "kind": "NON_NULL",
-                                    "name": null,
-                                    "ofType": {
-                                        "kind": "OBJECT",
-                                        "name": "JiraBoardShardSpecOverride_v1",
-                                        "ofType": null
-                                    }
-                                }
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        }
-                    ],
-                    "inputFields": null,
-                    "interfaces": [
-                        {
-                            "kind": "INTERFACE",
-                            "name": "IntegrationSharding_v1",
-                            "ofType": null
-                        }
-                    ],
-                    "enumValues": null,
-                    "possibleTypes": null
-                },
-                {
-                    "kind": "OBJECT",
-                    "name": "JiraBoardShardSpecOverride_v1",
-                    "description": null,
-                    "fields": [
-                        {
-                            "name": "shard",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "NON_NULL",
-                                "name": null,
-                                "ofType": {
-                                    "kind": "OBJECT",
-                                    "name": "JiraBoard_v1",
-                                    "ofType": null
-                                }
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "imageRef",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "disabled",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "Boolean",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        }
-                    ],
-                    "inputFields": null,
-                    "interfaces": [],
-                    "enumValues": null,
-                    "possibleTypes": null
-                },
                 {
                     "kind": "OBJECT",
                     "name": "StaticSubSharding_v1",

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/integrations_manager.py

@@ -41,7 +41,6 @@ from reconcile.utils.runtime.sharding import (
     AWSAccountShardingStrategy,
     CloudflareDnsZoneShardingStrategy,
     IntegrationShardManager,
-    JiraBoardShardingStrategy,
     OCMOrganizationShardingStrategy,
     OpenshiftClusterShardingStrategy,
     ShardSpec,
@@ -258,7 +257,6 @@ def run(
             OpenshiftClusterShardingStrategy.IDENTIFIER: OpenshiftClusterShardingStrategy(),
             CloudflareDnsZoneShardingStrategy.IDENTIFIER: CloudflareDnsZoneShardingStrategy(),
             OCMOrganizationShardingStrategy.IDENTIFIER: OCMOrganizationShardingStrategy(),
-            JiraBoardShardingStrategy.IDENTIFIER: JiraBoardShardingStrategy(),
         },
         integration_runtime_meta=integration_runtime_meta,
     )

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/jira_permissions_validator.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+import time
 from collections.abc import Callable, Iterable
 from enum import IntFlag, auto
 from typing import Any, TypedDict
@@ -20,18 +21,21 @@ from reconcile.typed_queries.app_interface_vault_settings import (
 from reconcile.typed_queries.jira_settings import get_jira_settings
 from reconcile.typed_queries.jiralert_settings import get_jiralert_settings
 from reconcile.utils import gql, metrics
+from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.extended_early_exit import (
     ExtendedEarlyExitRunnerResult,
     extended_early_exit_run,
 )
 from reconcile.utils.jira_client import JiraClient, JiraWatcherSettings
+from reconcile.utils.runtime.integration import DesiredStateShardConfig
 from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
 from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.state import State, init_state
 from reconcile.utils.unleash import get_feature_toggle_state
 
 QONTRACT_INTEGRATION = "jira-permissions-validator"
-QONTRACT_INTEGRATION_VERSION = make_semver(1, 1, 0)
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 2, 0)
 
 NameToIdMap = dict[str, str]
 
@@ -66,6 +70,7 @@ class ValidationError(IntFlag):
 
 class RunnerParams(TypedDict):
     boards: list[JiraBoardV1]
+    board_check_interval: int
     dry_run: bool
 
 
@@ -201,12 +206,19 @@ def validate_boards(
     jira_boards: Iterable[JiraBoardV1],
     default_issue_type: str,
     default_reopen_state: str,
+    board_check_interval: int,
     dry_run: bool,
+    state: State,
     jira_client_class: type[JiraClient] = JiraClient,
 ) -> bool:
     error = False
     jira_clients: dict[str, JiraClient] = {}
     for board in jira_boards:
+        last_successful_run = state.get(board.name, 0)
+        if not dry_run and time.time() <= last_successful_run + board_check_interval:
+            logging.debug(f"[{board.name}] Skipping board")
+            continue
+
         logging.debug(f"[{board.name}] checking ...")
         if board.server.server_url not in jira_clients:
             jira_clients[board.server.server_url] = jira_client_class.create(
@@ -230,6 +242,9 @@ def validate_boards(
                 case 0:
                     # no errors
                     logging.debug(f"[{board.name}] is valid")
+                    if not dry_run:
+                        # remember time of the last successful run
+                        state[board.name] = time.time()
                 case ValidationError.PERMISSION_ERROR:
                     # we don't have all the permissions, but we can create jira tickets
                     metrics_container.set_gauge(
@@ -257,13 +272,17 @@ def validate_boards(
 
 
 def get_jira_boards(
-    query_func: Callable, jira_board_name: str | None = None
+    query_func: Callable,
+    jira_board_names: list[str] | None = None,
 ) -> list[JiraBoardV1]:
     return [
         board
         for board in query_jira_boards(query_func=query_func).jira_boards or []
         if integration_is_enabled(QONTRACT_INTEGRATION, board)
-        and (not jira_board_name or board.name.lower() == jira_board_name.lower())
+        and (
+            not jira_board_names
+            or board.name.lower() in [name.lower() for name in jira_board_names]
+        )
     ]
 
 
@@ -273,16 +292,18 @@ def export_boards(boards: list[JiraBoardV1]) -> list[dict]:
 
 def run(
     dry_run: bool,
-    jira_board_name: str | None = None,
+    jira_board_name: list[str] | None = None,
+    board_check_interval: int = 3600,
     enable_extended_early_exit: bool = False,
     extended_early_exit_cache_ttl_seconds: int = 3600,
     log_cached_log_output: bool = False,
 ) -> None:
     gql_api = gql.get_api()
-    boards = get_jira_boards(query_func=gql_api.query, jira_board_name=jira_board_name)
+    boards = get_jira_boards(query_func=gql_api.query, jira_board_names=jira_board_name)
     runner_params: RunnerParams = {
         "boards": boards,
         "dry_run": dry_run,
+        "board_check_interval": board_check_interval,
     }
     if enable_extended_early_exit and get_feature_toggle_state(
         "jira-permissions-validator-extended-early-exit",
@@ -300,7 +321,7 @@ def run(
             # don't use `dry_run` in the cache key because this is a read-only integration
             dry_run=False,
             cache_source=cache_source,
-            shard=jira_board_name or "",
+            shard="_".join(set(jira_board_name)) if jira_board_name else "",
             ttl_seconds=extended_early_exit_cache_ttl_seconds,
             logger=logging.getLogger(),
             runner=runner,
@@ -312,12 +333,21 @@ def run(
         runner(**runner_params)
 
 
-def runner(boards: list[JiraBoardV1], dry_run: bool) -> ExtendedEarlyExitRunnerResult:
+@defer
+def runner(
+    boards: list[JiraBoardV1],
+    dry_run: bool,
+    board_check_interval: int,
+    defer: Callable | None = None,
+) -> ExtendedEarlyExitRunnerResult:
     gql_api = gql.get_api()
     settings = get_jira_settings(gql_api=gql_api)
     jiralert_settings = get_jiralert_settings(query_func=gql_api.query)
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    state = init_state(integration=QONTRACT_INTEGRATION, secret_reader=secret_reader)
+    if defer:
+        defer(state.cleanup)
 
     with metrics.transactional_metrics("jira-boards") as metrics_container:
         error = validate_boards(
@@ -327,7 +357,9 @@ def runner(boards: list[JiraBoardV1], dry_run: bool) -> ExtendedEarlyExitRunnerR
             jira_boards=boards,
             default_issue_type=jiralert_settings.default_issue_type,
             default_reopen_state=jiralert_settings.default_reopen_state,
+            board_check_interval=board_check_interval,
             dry_run=dry_run,
+            state=state,
         )
 
     if error:
@@ -337,13 +369,22 @@ def runner(boards: list[JiraBoardV1], dry_run: bool) -> ExtendedEarlyExitRunnerR
 
 
 def early_exit_desired_state(
-    *args: Any, jira_board_name: str | None = None, **kwargs: Any
+    *args: Any, jira_board_name: list[str] | None = None, **kwargs: Any
 ) -> dict[str, Any]:
     return {
         "boards": export_boards(
             get_jira_boards(
                 query_func=gql.get_api().query,
-                jira_board_name=jira_board_name,
+                jira_board_names=jira_board_name,
             )
         )
     }
+
+
+def desired_state_shard_config() -> DesiredStateShardConfig:
+    return DesiredStateShardConfig(
+        shard_arg_name="jira_board_name",
+        shard_path_selectors={"boards[*].name"},
+        shard_arg_is_collection=True,
+        sharded_run_review=lambda proposal: len(proposal.proposed_shards) <= 2,
+    )

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/reconcile/utils/runtime/sharding.py

@@ -19,8 +19,6 @@ from reconcile.gql_definitions.integrations.integrations import (
     IntegrationManagedV1,
     IntegrationShardingV1,
     IntegrationSpecV1,
-    JiraBoardShardingV1,
-    JiraBoardShardSpecOverrideV1,
     OCMOrganizationShardingV1,
     OCMOrganizationShardSpecOverrideV1,
     OpenshiftClusterShardingV1,
@@ -30,7 +28,6 @@ from reconcile.gql_definitions.integrations.integrations import (
     SubShardingV1,
 )
 from reconcile.gql_definitions.sharding import aws_accounts as sharding_aws_accounts
-from reconcile.gql_definitions.sharding import jira_boards as sharding_jira_boards
 from reconcile.gql_definitions.sharding import (
     ocm_organization as sharding_ocm_organization,
 )
@@ -437,67 +434,6 @@ class CloudflareDnsZoneShardingStrategy:
         return shards
 
 
-class JiraBoardShardingStrategy:
-    IDENTIFIER = "per-jira-board"
-
-    def __init__(
-        self,
-        jira_boards: Iterable[sharding_jira_boards.JiraBoardV1] | None = None,
-    ):
-        if not jira_boards:
-            self.jira_boards = (
-                sharding_jira_boards.query(query_func=gql.get_api().query).jira_boards
-                or []
-            )
-        else:
-            self.jira_boards = list(jira_boards)
-
-    def get_shard_spec_overrides(
-        self, sharding: IntegrationShardingV1 | None
-    ) -> dict[str, JiraBoardShardSpecOverrideV1]:
-        spos: dict[str, JiraBoardShardSpecOverrideV1] = {}
-
-        if isinstance(sharding, JiraBoardShardingV1) and sharding.shard_spec_overrides:
-            for sp in sharding.shard_spec_overrides or []:
-                spos[sp.shard.name] = sp
-        return spos
-
-    def check_integration_sharding_params(self, meta: IntegrationMeta) -> None:
-        if "--jira-board-name" not in meta.args:
-            raise ValueError(
-                f"the integration {meta.name} does not support the required argument "
-                " --jira-board-name for the 'per-jira-board' sharding strategy."
-            )
-
-    def build_shard_spec(
-        self,
-        jira_board: sharding_jira_boards.JiraBoardV1,
-        integration_spec: IntegrationSpecV1,
-        spo: JiraBoardShardSpecOverrideV1 | None,
-    ) -> ShardSpec:
-        return ShardSpec(
-            shard_key=jira_board.name,
-            shard_name_suffix=f"-{jira_board.name.lower()}",
-            extra_args=(integration_spec.extra_args or "")
-            + f" --jira-board-name {jira_board.name}",
-            shard_spec_overrides=spo,
-        )
-
-    def build_integration_shards(
-        self,
-        integration_meta: IntegrationMeta,
-        integration_managed: IntegrationManagedV1,
-    ) -> list[ShardSpec]:
-        self.check_integration_sharding_params(integration_meta)
-        spos = self.get_shard_spec_overrides(integration_managed.sharding)
-        shards = []
-        for board in self.jira_boards:
-            spo = spos.get(board.name)
-            base_shard = self.build_shard_spec(board, integration_managed.spec, spo)
-            shards.append(base_shard)
-        return shards
-
-
 @dataclass
 class IntegrationShardManager:
     strategies: dict[str, ShardingStrategy]

{qontract_reconcile-0.10.2.dev24 → qontract_reconcile-0.10.2.dev26}/tools/cli_commands/erv2.py

@@ -185,10 +185,10 @@ class Erv2Cli:
                         "run",
                         "--name",
                         "erv2",
-                        "--mount",
-                        f"type=bind,source={input_file!s},target=/inputs/input.json",
-                        "--mount",
-                        f"type=bind,source={credentials!s},target=/credentials",
+                        "-v",
+                        f"{input_file!s}:/inputs/input.json:Z",
+                        "-v",
+                        f"{credentials!s}:/credentials:Z",
                         "-e",
                         "AWS_SHARED_CREDENTIALS_FILE=/credentials",
                         "--entrypoint",
@@ -236,10 +236,10 @@ class Erv2Cli:
                     "erv2-debug-shell",
                     "--rm",
                     "-it",
-                    "--mount",
-                    f"type=bind,source={input_file!s},target=/inputs/input.json",
-                    "--mount",
-                    f"type=bind,source={credentials!s},target=/credentials",
+                    "-v",
+                    f"{input_file!s}:/inputs/input.json:Z",
+                    "-v",
+                    f"{credentials!s}:/credentials:Z",
                     "-e",
                     "AWS_SHARED_CREDENTIALS_FILE=/credentials",
                     "--entrypoint",
@@ -269,10 +269,10 @@ class Erv2Cli:
                     "--name",
                     "erv2-unlock",
                     "--rm",
-                    "--mount",
-                    f"type=bind,source={input_file!s},target=/inputs/input.json",
-                    "--mount",
-                    f"type=bind,source={credentials!s},target=/credentials",
+                    "-v",
+                    f"{input_file!s}:/inputs/input.json:Z",
+                    "-v",
+                    f"{credentials!s}:/credentials:Z",
                     "-e",
                     "AWS_SHARED_CREDENTIALS_FILE=/credentials",
                     "--entrypoint",

qontract_reconcile-0.10.2.dev24/reconcile/gql_definitions/sharding/jira_boards.py

@@ -1,60 +0,0 @@
-"""
-Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
-"""
-from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
-from datetime import datetime  # noqa: F401 # pylint: disable=W0611
-from enum import Enum  # noqa: F401 # pylint: disable=W0611
-from typing import (  # noqa: F401 # pylint: disable=W0611
-    Any,
-    Optional,
-    Union,
-)
-
-from pydantic import (  # noqa: F401 # pylint: disable=W0611
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
-
-
-DEFINITION = """
-query JiraBoardSharding {
-  jira_boards: jira_boards_v1 {
-    name
-  }
-}
-"""
-
-
-class ConfiguredBaseModel(BaseModel):
-    class Config:
-        smart_union=True
-        extra=Extra.forbid
-
-
-class JiraBoardV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class JiraBoardShardingQueryData(ConfiguredBaseModel):
-    jira_boards: Optional[list[JiraBoardV1]] = Field(..., alias="jira_boards")
-
-
-def query(query_func: Callable, **kwargs: Any) -> JiraBoardShardingQueryData:
-    """
-    This is a convenience function which queries and parses the data into
-    concrete types. It should be compatible with most GQL clients.
-    You do not have to use it to consume the generated data classes.
-    Alternatively, you can also mime and alternate the behavior
-    of this function in the caller.
-
-    Parameters:
-        query_func (Callable): Function which queries your GQL Server
-        kwargs: optional arguments that will be passed to the query function
-
-    Returns:
-        JiraBoardShardingQueryData: queried data parsed into generated classes
-    """
-    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return JiraBoardShardingQueryData(**raw_data)