qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/runtime/integration.py

@@ -2,11 +2,11 @@ from abc import (
     ABC,
     abstractmethod,
 )
+from collections.abc import Callable
 from dataclasses import dataclass
 from types import ModuleType
 from typing import (
     Any,
-    Callable,
     Generic,
     Optional,
     TypeVar,
@@ -14,6 +14,14 @@ from typing import (
 
 from pydantic import BaseModel
 
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.utils.secret_reader import (
+    SecretReaderBase,
+    create_secret_reader,
+)
+
 
 @dataclass
 class ShardedRunProposal:
@@ -145,13 +153,23 @@ class QontractReconcileIntegration(ABC, Generic[RunParamsTypeVar]):
 
     def __init__(self, params: RunParamsTypeVar) -> None:
         self.params: RunParamsTypeVar = params
+        self._secret_reader: SecretReaderBase | None = None
 
     @property
     @abstractmethod
-    def name(self) -> str:
-        ...
+    def name(self) -> str: ...
 
-    def get_early_exit_desired_state(self) -> Optional[dict[str, Any]]:
+    @property
+    def secret_reader(self) -> SecretReaderBase:
+        """
+        Returns a function that can be used to read secrets from the vault or another secret reader.
+        """
+        if self._secret_reader is None:
+            vault_settings = get_app_interface_vault_settings()
+            self._secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+        return self._secret_reader
+
+    def get_early_exit_desired_state(self) -> dict[str, Any] | None:
         """
         An integration that wants to support early exit on its desired state
         must implement this method and return the desired state as a dictionary.
@@ -213,13 +231,11 @@ class QontractReconcileIntegration(ABC, Generic[RunParamsTypeVar]):
             self.get_desired_state_shard_config()
         )
         if sharding_config:
-            sharded_params = self.params.copy_and_update(
-                {
-                    sharding_config.shard_arg_name: [shard]
-                    if sharding_config.shard_arg_is_collection
-                    else shard
-                }
-            )
+            sharded_params = self.params.copy_and_update({
+                sharding_config.shard_arg_name: [shard]
+                if sharding_config.shard_arg_is_collection
+                else shard
+            })
             return type(self)(sharded_params)
 
         raise NotImplementedError(
@@ -283,7 +299,7 @@ class ModuleBasedQontractReconcileIntegration(
             return self.params.module.QONTRACT_INTEGRATION.replace("_", "-")
         raise NotImplementedError("Integration missing QONTRACT_INTEGRATION.")
 
-    def get_early_exit_desired_state(self) -> Optional[dict[str, Any]]:
+    def get_early_exit_desired_state(self) -> dict[str, Any] | None:
         if self._integration_supports(EARLY_EXIT_DESIRED_STATE_FUNCTION):
             return self.params.module.early_exit_desired_state(
                 *self.params.args, **self.params.kwargs

reconcile/utils/runtime/meta.py

@@ -1,14 +1,12 @@
 import dataclasses
 from dataclasses import dataclass
-from typing import Optional
 
 
 @dataclass
 class IntegrationMeta:
-
     name: str
     args: list[str]
-    short_help: Optional[str]
+    short_help: str | None
 
     def to_dict(self):
         return dataclasses.asdict(self)

reconcile/utils/runtime/runner.py

@@ -3,7 +3,6 @@ import sys
 from dataclasses import dataclass
 from typing import (
     Any,
-    Optional,
     TypeVar,
 )
 
@@ -44,7 +43,7 @@ class IntegrationRunConfiguration:
     potential problems with the integration or the configuration data from app-interface.
     """
 
-    early_exit_compare_sha: Optional[str]
+    early_exit_compare_sha: str | None
     """
     The SHA of the bundle to compare the current desired state against.
     """
@@ -66,11 +65,11 @@ class IntegrationRunConfiguration:
     A debug flag to control whether the URL of the GraphQL endpoint in use is printed.
     """
 
-    def main_bundle_desired_state(self) -> Optional[dict[str, Any]]:
+    def main_bundle_desired_state(self) -> dict[str, Any] | None:
         self.switch_to_main_bundle()
         return self.integration.get_early_exit_desired_state()
 
-    def comparison_bundle_desired_state(self) -> Optional[dict[str, Any]]:
+    def comparison_bundle_desired_state(self) -> dict[str, Any] | None:
         self.switch_to_comparison_bundle()
         data = (  # pylint: disable=assignment-from-none
             self.integration.get_early_exit_desired_state()
@@ -78,7 +77,7 @@ class IntegrationRunConfiguration:
         self.switch_to_main_bundle()
         return data
 
-    def switch_to_main_bundle(self, validate_schemas: Optional[bool] = None) -> None:
+    def switch_to_main_bundle(self, validate_schemas: bool | None = None) -> None:
         final_validate_schemas = (
             validate_schemas if validate_schemas is not None else self.valdiate_schemas
         )
@@ -89,9 +88,7 @@ class IntegrationRunConfiguration:
             print_url=self.print_url,
         )
 
-    def switch_to_comparison_bundle(
-        self, validate_schemas: Optional[bool] = None
-    ) -> None:
+    def switch_to_comparison_bundle(self, validate_schemas: bool | None = None) -> None:
         final_validate_schemas = (
             validate_schemas if validate_schemas is not None else self.valdiate_schemas
         )
@@ -105,7 +102,7 @@ class IntegrationRunConfiguration:
 
 def get_desired_state_diff(
     run_cfg: IntegrationRunConfiguration,
-) -> Optional[DesiredStateDiff]:
+) -> DesiredStateDiff | None:
     """
     Calculates the desired state diff between the current bundle and the
     comparison bundle for an integration. If the integration does not support
@@ -172,7 +169,7 @@ def _integration_wet_run(
 
 def _integration_dry_run(
     integration: QontractReconcileIntegration[RunParamsTypeVar],
-    desired_state_diff: Optional[DesiredStateDiff],
+    desired_state_diff: DesiredStateDiff | None,
 ) -> None:
     """
     Runs an integration in dry-run mode, i.e. not actually making any changes
@@ -216,7 +213,7 @@ def _integration_dry_run(
             return_exceptions=True,
         )
 
-        for shard, result in zip(affected_shard_list, results):
+        for shard, result in zip(affected_shard_list, results, strict=False):
             if _is_task_result_an_error(result):
                 logging.error(f"Failed to run integration shard {shard}: {result}")
         failed_shards_count = sum(1 for _ in filter(_is_task_result_an_error, results))

reconcile/utils/runtime/sharding.py

@@ -2,14 +2,15 @@ import copy
 from collections.abc import Iterable
 from dataclasses import dataclass
 from typing import (
-    Optional,
     Protocol,
-    Union,
 )
 
 from pydantic import BaseModel
 
 from reconcile.gql_definitions.common.clusters_minimal import ClusterV1
+from reconcile.gql_definitions.fragments.minimal_ocm_organization import (
+    MinimalOCMOrganization,
+)
 from reconcile.gql_definitions.integrations.integrations import (
     AWSAccountShardingV1,
     AWSAccountShardSpecOverrideV1,
@@ -18,6 +19,8 @@ from reconcile.gql_definitions.integrations.integrations import (
     IntegrationManagedV1,
     IntegrationShardingV1,
     IntegrationSpecV1,
+    OCMOrganizationShardingV1,
+    OCMOrganizationShardSpecOverrideV1,
     OpenshiftClusterShardingV1,
     OpenshiftClusterShardSpecOverrideV1,
     StaticShardingV1,
@@ -25,6 +28,9 @@ from reconcile.gql_definitions.integrations.integrations import (
     SubShardingV1,
 )
 from reconcile.gql_definitions.sharding import aws_accounts as sharding_aws_accounts
+from reconcile.gql_definitions.sharding import (
+    ocm_organization as sharding_ocm_organization,
+)
 from reconcile.gql_definitions.terraform_cloudflare_dns import (
     terraform_cloudflare_zones,
 )
@@ -38,15 +44,15 @@ from reconcile.utils.runtime.meta import IntegrationMeta
 
 class ShardSpec(BaseModel):
     # Base Sharding
-    shards: Optional[str] = ""
-    shard_id: Optional[str] = ""
-    shard_spec_overrides: Optional[
-        Union[
-            AWSAccountShardSpecOverrideV1,
-            OpenshiftClusterShardSpecOverrideV1,
-            CloudflareDNSZoneShardSpecOverrideV1,
-        ]
-    ] = None
+    shards: str | None = ""
+    shard_id: str | None = ""
+    shard_spec_overrides: (
+        AWSAccountShardSpecOverrideV1
+        | OpenshiftClusterShardSpecOverrideV1
+        | CloudflareDNSZoneShardSpecOverrideV1
+        | OCMOrganizationShardSpecOverrideV1
+        | None
+    ) = None
 
     # Key sharding
     shard_key: str = ""
@@ -101,7 +107,7 @@ class StaticShardingStrategy:
     def create_sub_shards(
         base_shard: ShardSpec, sub_sharding: SubShardingV1
     ) -> list[ShardSpec]:
-        if base_shard.shard_id != "" or base_shard.shards != "":
+        if base_shard.shard_id or base_shard.shards:
             raise ValueError(
                 "Static sub_sharding can only be applied to Key based sharding"
             )
@@ -127,9 +133,8 @@ class AWSAccountShardingStrategy:
 
     def __init__(
         self,
-        aws_accounts: Optional[Iterable[sharding_aws_accounts.AWSAccountV1]] = None,
+        aws_accounts: Iterable[sharding_aws_accounts.AWSAccountV1] | None = None,
     ):
-
         if not aws_accounts:
             self.aws_accounts = (
                 sharding_aws_accounts.query(query_func=gql.get_api().query).aws_accounts
@@ -154,9 +159,8 @@ class AWSAccountShardingStrategy:
         ]
 
     def get_shard_spec_overrides(
-        self, sharding: Optional[IntegrationShardingV1]
+        self, sharding: IntegrationShardingV1 | None
     ) -> dict[str, AWSAccountShardSpecOverrideV1]:
-
         spos: dict[str, AWSAccountShardSpecOverrideV1] = {}
 
         if isinstance(sharding, AWSAccountShardingV1) and sharding.shard_spec_overrides:
@@ -176,9 +180,8 @@ class AWSAccountShardingStrategy:
         self,
         aws_account: sharding_aws_accounts.AWSAccountV1,
         integration_spec: IntegrationSpecV1,
-        spo: Optional[AWSAccountShardSpecOverrideV1],
+        spo: AWSAccountShardSpecOverrideV1 | None,
     ) -> ShardSpec:
-
         return ShardSpec(
             shard_key=aws_account.name,
             shard_name_suffix=f"-{aws_account.name}",
@@ -192,7 +195,6 @@ class AWSAccountShardingStrategy:
         integration_meta: IntegrationMeta,
         integration_managed: IntegrationManagedV1,
     ) -> list[ShardSpec]:
-
         self.check_integration_sharding_params(integration_meta)
         spos = self.get_shard_spec_overrides(integration_managed.sharding)
         shards = []
@@ -203,11 +205,75 @@ class AWSAccountShardingStrategy:
         return shards
 
 
+class OCMOrganizationShardingStrategy:
+    IDENTIFIER = "per-ocm-organization"
+
+    def __init__(
+        self,
+        ocm_organizations: Iterable[MinimalOCMOrganization] | None = None,
+    ):
+        if not ocm_organizations:
+            self.ocm_organizations = (
+                sharding_ocm_organization.query(
+                    query_func=gql.get_api().query
+                ).ocm_organizations
+                or []
+            )
+        else:
+            self.ocm_organizations = list(ocm_organizations)
+
+    def get_shard_spec_overrides(
+        self, sharding: IntegrationShardingV1 | None
+    ) -> dict[str, OCMOrganizationShardSpecOverrideV1]:
+        spos: dict[str, OCMOrganizationShardSpecOverrideV1] = {}
+
+        if (
+            isinstance(sharding, OCMOrganizationShardingV1)
+            and sharding.shard_spec_overrides
+        ):
+            for sp in sharding.shard_spec_overrides or []:
+                spos[sp.shard.name] = sp
+        return spos
+
+    def check_integration_sharding_params(self, meta: IntegrationMeta) -> None:
+        if "--org-id" not in meta.args:
+            raise ValueError(
+                f"the integration {meta.name} does not support the required argument "
+                " --org-id for the 'per-ocm-organization' sharding strategy."
+            )
+
+    def build_shard_spec(
+        self,
+        org: MinimalOCMOrganization,
+        integration_spec: IntegrationSpecV1,
+        spo: OCMOrganizationShardSpecOverrideV1 | None,
+    ) -> ShardSpec:
+        return ShardSpec(
+            shard_key=org.org_id,
+            shard_name_suffix=f"-{org.org_id.lower()}",
+            extra_args=(integration_spec.extra_args or "") + f" --org-id {org.org_id}",
+            shard_spec_overrides=spo,
+        )
+
+    def build_integration_shards(
+        self,
+        integration_meta: IntegrationMeta,
+        integration_managed: IntegrationManagedV1,
+    ) -> list[ShardSpec]:
+        self.check_integration_sharding_params(integration_meta)
+        spos = self.get_shard_spec_overrides(integration_managed.sharding)
+        shards = []
+        for org in self.ocm_organizations:
+            spo = spos.get(org.name)
+            base_shard = self.build_shard_spec(org, integration_managed.spec, spo)
+            shards.append(base_shard)
+        return shards
+
+
 class OpenshiftClusterShardingStrategy:
     IDENTIFIER = "per-openshift-cluster"
 
-    def __init__(self, clusters: Optional[Iterable[ClusterV1]] = None):
-
+    def __init__(self, clusters: Iterable[ClusterV1] | None = None):
         if not clusters:
             self.clusters = get_clusters_minimal()
         else:
@@ -231,9 +297,8 @@ class OpenshiftClusterShardingStrategy:
         ]
 
     def get_shard_spec_overrides(
-        self, sharding: Optional[IntegrationShardingV1]
+        self, sharding: IntegrationShardingV1 | None
     ) -> dict[str, OpenshiftClusterShardSpecOverrideV1]:
-
         spos: dict[str, OpenshiftClusterShardSpecOverrideV1] = {}
 
         if (
@@ -256,9 +321,8 @@ class OpenshiftClusterShardingStrategy:
         self,
         cluster: ClusterV1,
         integration_spec: IntegrationSpecV1,
-        spo: Optional[OpenshiftClusterShardSpecOverrideV1],
+        spo: OpenshiftClusterShardSpecOverrideV1 | None,
     ) -> ShardSpec:
-
         return ShardSpec(
             shard_key=cluster.name,
             shard_name_suffix=f"-{cluster.name}",
@@ -268,7 +332,7 @@ class OpenshiftClusterShardingStrategy:
         )
 
     def build_sub_shards(
-        self, base_shard: ShardSpec, spo: Optional[OpenshiftClusterShardSpecOverrideV1]
+        self, base_shard: ShardSpec, spo: OpenshiftClusterShardSpecOverrideV1 | None
     ) -> list[ShardSpec]:
         sub_shards = []
         if spo and spo.sub_sharding and spo.sub_sharding.strategy:
@@ -285,7 +349,6 @@ class OpenshiftClusterShardingStrategy:
         integration_meta: IntegrationMeta,
         integration_managed: IntegrationManagedV1,
     ) -> list[ShardSpec]:
-
         self.check_integration_sharding_params(integration_meta)
         spos = self.get_shard_spec_overrides(integration_managed.sharding)
         shards = []
@@ -308,9 +371,7 @@ class CloudflareDnsZoneShardingStrategy:
 
     IDENTIFIER = "per-cloudflare-dns-zone"
 
-    def __init__(
-        self, cloudflare_zones: Optional[Iterable[CloudflareDnsZoneV1]] = None
-    ):
+    def __init__(self, cloudflare_zones: Iterable[CloudflareDnsZoneV1] | None = None):
         if not cloudflare_zones:
             self.cloudflare_zones = (
                 terraform_cloudflare_zones.query(query_func=gql.get_api().query).zones
@@ -323,9 +384,8 @@ class CloudflareDnsZoneShardingStrategy:
         return f"{dns_zone.account.name}-{dns_zone.identifier}"
 
     def get_shard_spec_overrides(
-        self, sharding: Optional[IntegrationShardingV1]
+        self, sharding: IntegrationShardingV1 | None
     ) -> dict[str, CloudflareDNSZoneShardSpecOverrideV1]:
-
         spos: dict[str, CloudflareDNSZoneShardSpecOverrideV1] = {}
 
         if (
@@ -349,9 +409,8 @@ class CloudflareDnsZoneShardingStrategy:
         self,
         dns_zone: CloudflareDnsZoneV1,
         integration_spec: IntegrationSpecV1,
-        spo: Optional[CloudflareDNSZoneShardSpecOverrideV1],
+        spo: CloudflareDNSZoneShardSpecOverrideV1 | None,
     ) -> ShardSpec:
-
         return ShardSpec(
             shard_key=self._get_shard_key(dns_zone),
             shard_name_suffix=f"-{self._get_shard_key(dns_zone)}",
@@ -365,7 +424,6 @@ class CloudflareDnsZoneShardingStrategy:
         integration_meta: IntegrationMeta,
         integration_managed: IntegrationManagedV1,
     ) -> list[ShardSpec]:
-
         self.check_integration_sharding_params(integration_meta)
         spos = self.get_shard_spec_overrides(integration_managed.sharding)
         shards = []
@@ -378,7 +436,6 @@ class CloudflareDnsZoneShardingStrategy:
 
 @dataclass
 class IntegrationShardManager:
-
     strategies: dict[str, ShardingStrategy]
     integration_runtime_meta: dict[str, IntegrationMeta]
 

reconcile/utils/saasherder/__init__.py

@@ -9,9 +9,9 @@ from reconcile.utils.saasherder.saasherder import (
 )
 
 __all__ = [
+    "UNIQUE_SAAS_FILE_ENV_COMBO_LEN",
     "Providers",
     "SaasHerder",
     "TriggerSpecUnion",
     "TriggerTypes",
-    "UNIQUE_SAAS_FILE_ENV_COMBO_LEN",
 ]