qontract-reconcile 0.10.2.dev21__tar.gz → 0.10.2.dev23__tar.gz

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev21
+Version: 0.10.2.dev23
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/cli.py

@@ -1158,12 +1158,14 @@ def jenkins_webhooks_cleaner(ctx):
 
 
 @integration.command(short_help="Validate permissions in Jira.")
+@click.option("--jira-board-name", help="The Jira board to act on.", default=None)
 @enable_extended_early_exit
 @extended_early_exit_cache_ttl_seconds
 @log_cached_log_output
 @click.pass_context
 def jira_permissions_validator(
     ctx,
+    jira_board_name,
     enable_extended_early_exit,
     extended_early_exit_cache_ttl_seconds,
     log_cached_log_output,
@@ -1173,6 +1175,7 @@ def jira_permissions_validator(
     run_integration(
         reconcile.jira_permissions_validator,
         ctx.obj,
+        jira_board_name=jira_board_name,
         enable_extended_early_exit=enable_extended_early_exit,
         extended_early_exit_cache_ttl_seconds=extended_early_exit_cache_ttl_seconds,
         log_cached_log_output=log_cached_log_output,

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/gql_definitions/integrations/integrations.py

@@ -191,6 +191,22 @@ query Integrations {
             }
           }
         }
+
+        ... on JiraBoardSharding_v1 {
+            shardSpecOverrides {
+              shard {
+                name
+                disable {
+                  integrations
+                }
+              }
+              imageRef
+              disabled
+              resources {
+                ... DeployResourcesFields
+              }
+            }
+        }
       }
     }
   }
@@ -342,10 +358,30 @@ class CloudflareDNSZoneShardingV1(IntegrationShardingV1):
     shard_spec_overrides: Optional[list[CloudflareDNSZoneShardSpecOverrideV1]] = Field(..., alias="shardSpecOverrides")
 
 
+class DisableJiraBoardAutomationsV1(ConfiguredBaseModel):
+    integrations: Optional[list[str]] = Field(..., alias="integrations")
+
+
+class JiraBoardV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    disable: Optional[DisableJiraBoardAutomationsV1] = Field(..., alias="disable")
+
+
+class JiraBoardShardSpecOverrideV1(ConfiguredBaseModel):
+    shard: JiraBoardV1 = Field(..., alias="shard")
+    image_ref: Optional[str] = Field(..., alias="imageRef")
+    disabled: Optional[bool] = Field(..., alias="disabled")
+    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")
+
+
+class JiraBoardShardingV1(IntegrationShardingV1):
+    shard_spec_overrides: Optional[list[JiraBoardShardSpecOverrideV1]] = Field(..., alias="shardSpecOverrides")
+
+
 class IntegrationManagedV1(ConfiguredBaseModel):
     namespace: NamespaceV1 = Field(..., alias="namespace")
     spec: IntegrationSpecV1 = Field(..., alias="spec")
-    sharding: Optional[Union[StaticShardingV1, OpenshiftClusterShardingV1, OCMOrganizationShardingV1, AWSAccountShardingV1, CloudflareDNSZoneShardingV1, IntegrationShardingV1]] = Field(..., alias="sharding")
+    sharding: Optional[Union[StaticShardingV1, OpenshiftClusterShardingV1, OCMOrganizationShardingV1, AWSAccountShardingV1, CloudflareDNSZoneShardingV1, JiraBoardShardingV1, IntegrationShardingV1]] = Field(..., alias="sharding")
 
 
 class IntegrationV1(ConfiguredBaseModel):

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/gql_definitions/introspection.json

@@ -29766,6 +29766,11 @@
                             "kind": "OBJECT",
                             "name": "AWSAccountSharding_v1",
                             "ofType": null
+                        },
+                        {
+                            "kind": "OBJECT",
+                            "name": "JiraBoardSharding_v1",
+                            "ofType": null
                         }
                     ]
                 },
@@ -45344,6 +45349,42 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "managed_by_erv2",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "delete",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "module_overrides",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "ExternalResourcesModuleOverrides_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -49733,6 +49774,122 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
+                {
+                    "kind": "OBJECT",
+                    "name": "JiraBoardSharding_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "strategy",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "SCALAR",
+                                    "name": "String",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "shardSpecOverrides",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "OBJECT",
+                                        "name": "JiraBoardShardSpecOverride_v1",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [
+                        {
+                            "kind": "INTERFACE",
+                            "name": "IntegrationSharding_v1",
+                            "ofType": null
+                        }
+                    ],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
+                {
+                    "kind": "OBJECT",
+                    "name": "JiraBoardShardSpecOverride_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "shard",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "JiraBoard_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "imageRef",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "resources",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "DeployResources_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "disabled",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
                 {
                     "kind": "OBJECT",
                     "name": "StaticSubSharding_v1",

qontract_reconcile-0.10.2.dev23/reconcile/gql_definitions/sharding/jira_boards.py

@@ -0,0 +1,60 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query JiraBoardSharding {
+  jira_boards: jira_boards_v1 {
+    name
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class JiraBoardV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class JiraBoardShardingQueryData(ConfiguredBaseModel):
+    jira_boards: Optional[list[JiraBoardV1]] = Field(..., alias="jira_boards")
+
+
+def query(query_func: Callable, **kwargs: Any) -> JiraBoardShardingQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        JiraBoardShardingQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return JiraBoardShardingQueryData(**raw_data)

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/integrations_manager.py

@@ -41,6 +41,7 @@ from reconcile.utils.runtime.sharding import (
     AWSAccountShardingStrategy,
     CloudflareDnsZoneShardingStrategy,
     IntegrationShardManager,
+    JiraBoardShardingStrategy,
     OCMOrganizationShardingStrategy,
     OpenshiftClusterShardingStrategy,
     ShardSpec,
@@ -257,6 +258,7 @@ def run(
             OpenshiftClusterShardingStrategy.IDENTIFIER: OpenshiftClusterShardingStrategy(),
             CloudflareDnsZoneShardingStrategy.IDENTIFIER: CloudflareDnsZoneShardingStrategy(),
             OCMOrganizationShardingStrategy.IDENTIFIER: OCMOrganizationShardingStrategy(),
+            JiraBoardShardingStrategy.IDENTIFIER: JiraBoardShardingStrategy(),
         },
         integration_runtime_meta=integration_runtime_meta,
     )

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/jira_permissions_validator.py

@@ -31,7 +31,7 @@ from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.unleash import get_feature_toggle_state
 
 QONTRACT_INTEGRATION = "jira-permissions-validator"
-QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 1, 0)
 
 NameToIdMap = dict[str, str]
 
@@ -256,11 +256,14 @@ def validate_boards(
     return error
 
 
-def get_jira_boards(query_func: Callable) -> list[JiraBoardV1]:
+def get_jira_boards(
+    query_func: Callable, jira_board_name: str | None = None
+) -> list[JiraBoardV1]:
     return [
         board
         for board in query_jira_boards(query_func=query_func).jira_boards or []
         if integration_is_enabled(QONTRACT_INTEGRATION, board)
+        and (not jira_board_name or board.name.lower() == jira_board_name.lower())
     ]
 
 
@@ -270,12 +273,13 @@ def export_boards(boards: list[JiraBoardV1]) -> list[dict]:
 
 def run(
     dry_run: bool,
+    jira_board_name: str | None = None,
     enable_extended_early_exit: bool = False,
     extended_early_exit_cache_ttl_seconds: int = 3600,
     log_cached_log_output: bool = False,
 ) -> None:
     gql_api = gql.get_api()
-    boards = get_jira_boards(query_func=gql_api.query)
+    boards = get_jira_boards(query_func=gql_api.query, jira_board_name=jira_board_name)
     runner_params: RunnerParams = {
         "boards": boards,
         "dry_run": dry_run,
@@ -296,7 +300,7 @@ def run(
             # don't use `dry_run` in the cache key because this is a read-only integration
             dry_run=False,
             cache_source=cache_source,
-            shard="",
+            shard=jira_board_name or "",
             ttl_seconds=extended_early_exit_cache_ttl_seconds,
             logger=logging.getLogger(),
             runner=runner,
@@ -332,5 +336,14 @@ def runner(boards: list[JiraBoardV1], dry_run: bool) -> ExtendedEarlyExitRunnerR
     return ExtendedEarlyExitRunnerResult(payload=export_boards(boards), applied_count=0)
 
 
-def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
-    return {"boards": export_boards(get_jira_boards(query_func=gql.get_api().query))}
+def early_exit_desired_state(
+    *args: Any, jira_board_name: str | None = None, **kwargs: Any
+) -> dict[str, Any]:
+    return {
+        "boards": export_boards(
+            get_jira_boards(
+                query_func=gql.get_api().query,
+                jira_board_name=jira_board_name,
+            )
+        )
+    }

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/reconcile/utils/runtime/sharding.py

@@ -19,6 +19,8 @@ from reconcile.gql_definitions.integrations.integrations import (
     IntegrationManagedV1,
     IntegrationShardingV1,
     IntegrationSpecV1,
+    JiraBoardShardingV1,
+    JiraBoardShardSpecOverrideV1,
     OCMOrganizationShardingV1,
     OCMOrganizationShardSpecOverrideV1,
     OpenshiftClusterShardingV1,
@@ -28,6 +30,7 @@ from reconcile.gql_definitions.integrations.integrations import (
     SubShardingV1,
 )
 from reconcile.gql_definitions.sharding import aws_accounts as sharding_aws_accounts
+from reconcile.gql_definitions.sharding import jira_boards as sharding_jira_boards
 from reconcile.gql_definitions.sharding import (
     ocm_organization as sharding_ocm_organization,
 )
@@ -434,6 +437,67 @@ class CloudflareDnsZoneShardingStrategy:
         return shards
 
 
+class JiraBoardShardingStrategy:
+    IDENTIFIER = "per-jira-board"
+
+    def __init__(
+        self,
+        jira_boards: Iterable[sharding_jira_boards.JiraBoardV1] | None = None,
+    ):
+        if not jira_boards:
+            self.jira_boards = (
+                sharding_jira_boards.query(query_func=gql.get_api().query).jira_boards
+                or []
+            )
+        else:
+            self.jira_boards = list(jira_boards)
+
+    def get_shard_spec_overrides(
+        self, sharding: IntegrationShardingV1 | None
+    ) -> dict[str, JiraBoardShardSpecOverrideV1]:
+        spos: dict[str, JiraBoardShardSpecOverrideV1] = {}
+
+        if isinstance(sharding, JiraBoardShardingV1) and sharding.shard_spec_overrides:
+            for sp in sharding.shard_spec_overrides or []:
+                spos[sp.shard.name] = sp
+        return spos
+
+    def check_integration_sharding_params(self, meta: IntegrationMeta) -> None:
+        if "--jira-board-name" not in meta.args:
+            raise ValueError(
+                f"the integration {meta.name} does not support the required argument "
+                " --jira-board-name for the 'per-jira-board' sharding strategy."
+            )
+
+    def build_shard_spec(
+        self,
+        jira_board: sharding_jira_boards.JiraBoardV1,
+        integration_spec: IntegrationSpecV1,
+        spo: JiraBoardShardSpecOverrideV1 | None,
+    ) -> ShardSpec:
+        return ShardSpec(
+            shard_key=jira_board.name,
+            shard_name_suffix=f"-{jira_board.name.lower()}",
+            extra_args=(integration_spec.extra_args or "")
+            + f" --jira-board-name {jira_board.name}",
+            shard_spec_overrides=spo,
+        )
+
+    def build_integration_shards(
+        self,
+        integration_meta: IntegrationMeta,
+        integration_managed: IntegrationManagedV1,
+    ) -> list[ShardSpec]:
+        self.check_integration_sharding_params(integration_meta)
+        spos = self.get_shard_spec_overrides(integration_managed.sharding)
+        shards = []
+        for board in self.jira_boards:
+            spo = spos.get(board.name)
+            base_shard = self.build_shard_spec(board, integration_managed.spec, spo)
+            shards.append(base_shard)
+        return shards
+
+
 @dataclass
 class IntegrationShardManager:
     strategies: dict[str, ShardingStrategy]

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/tools/cli_commands/erv2.py

@@ -255,6 +255,41 @@ class Erv2Cli:
                 print(e.stdout.decode("utf-8"))
             raise
 
+    def force_unlock(self, credentials: Path, lock_id: str) -> None:
+        """Run 'terraform force-unlock' in a CDKTF container."""
+        input_file = self.temp / "input.json"
+        input_file.write_text(self.input_data)
+
+        try:
+            run(["docker", "pull", self.image], check=True, capture_output=True)
+            run(
+                [
+                    "docker",
+                    "run",
+                    "--name",
+                    "erv2-unlock",
+                    "--rm",
+                    "--mount",
+                    f"type=bind,source={input_file!s},target=/inputs/input.json",
+                    "--mount",
+                    f"type=bind,source={credentials!s},target=/credentials",
+                    "-e",
+                    "AWS_SHARED_CREDENTIALS_FILE=/credentials",
+                    "--entrypoint",
+                    "/bin/bash",
+                    self.image,
+                    "-c",
+                    f"cdktf synth && cd cdktf.out/stacks/CDKTF/ && terraform init && terraform force-unlock -force '{lock_id}'",
+                ],
+                check=True,
+            )
+        except CalledProcessError as e:
+            if e.stderr:
+                print(e.stderr.decode("utf-8"))
+            if e.stdout:
+                print(e.stdout.decode("utf-8"))
+            raise
+
 
 class TfRun(Protocol):
     def __call__(self, path: Path, cmd: list[str]) -> str: ...

{qontract_reconcile-0.10.2.dev21 → qontract_reconcile-0.10.2.dev23}/tools/qontract_cli.py

@@ -4359,6 +4359,47 @@ def debug_shell(ctx) -> None:
         erv2cli.enter_shell(credentials_file)
 
 
+@external_resources.command()
+@binary(["docker"])
+@click.option(
+    "--lock-id",
+    required=True,
+    help="The terraform lock ID to unlock",
+    prompt=True,
+)
+@click.pass_context
+def force_unlock(ctx, lock_id: str) -> None:
+    """Manually unlock the ERv2 terraform state."""
+    # use a temporary directory in $HOME. The MacOS colima default configuration allows docker mounts from $HOME.
+    with tempfile.TemporaryDirectory(
+        dir=Path.home(), prefix="erv2-unlock."
+    ) as _tempdir:
+        tempdir = Path(_tempdir)
+        with progress_spinner() as progress:
+            with task(progress, "Preparing environment ..."):
+                credentials_file = tempdir / "credentials"
+                credentials_file.write_text(
+                    ctx.obj["secret_reader"].read_with_parameters(
+                        path=f"app-sre/external-resources/{ctx.obj['provisioner']}",
+                        field="credentials",
+                        format=None,
+                        version=None,
+                    )
+                )
+            os.environ["AWS_SHARED_CREDENTIALS_FILE"] = str(credentials_file)
+
+        erv2cli = Erv2Cli(
+            provision_provider=ctx.obj["provision_provider"],
+            provisioner=ctx.obj["provisioner"],
+            provider=ctx.obj["provider"],
+            identifier=ctx.obj["identifier"],
+            secret_reader=ctx.obj["secret_reader"],
+            temp_dir=tempdir,
+            progress_spinner=progress,
+        )
+        erv2cli.force_unlock(credentials_file, lock_id)
+
+
 @get.command(help="Get all container images in app-interface defined namespaces")
 @cluster_name
 @namespace_name