qontract-reconcile 0.10.1rc816__py3-none-any.whl → 0.10.1rc818__py3-none-any.whl

{qontract_reconcile-0.10.1rc816.dist-info → qontract_reconcile-0.10.1rc818.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc816
+Version: 0.10.1rc818
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc816.dist-info → qontract_reconcile-0.10.1rc818.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=laiCfiVjXsDnT5z18q_XGkH6tZdBSxKVva7nF4B0OQw,101452
+reconcile/cli.py,sha256=UnVVYcG6lqGAhUcXXCUsxw-eVlU4AyHjI5fAE4E_4DQ,101994
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -376,6 +376,8 @@ reconcile/gql_definitions/terraform_resources/database_access_manager.py,sha256=
 reconcile/gql_definitions/terraform_resources/terraform_resources_namespaces.py,sha256=bY53hFt1Ki0oqjP03OX4nT80zdD29YwvnlKW3Df7zs8,42401
 reconcile/gql_definitions/terraform_tgw_attachments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/terraform_tgw_attachments/aws_accounts.py,sha256=GjCuLHOgm3eHbkpK7Q2i7l6tori5Y62uFlz3M89BYtA,2602
+reconcile/gql_definitions/unleash_feature_toggles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/unleash_feature_toggles/feature_toggles.py,sha256=AJsLOy2AhiMbfhoRPEwNF39FEKfg6Snwfi4CXscBZ2I,3227
 reconcile/gql_definitions/vault_instances/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/vault_instances/vault_instances.py,sha256=8G26YFo1rqRkrPuQJzgEkZt9KiU1PRx3qyw3LMI5ZMI,7515
 reconcile/gql_definitions/vault_policies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -538,7 +540,6 @@ reconcile/test/test_terraform_users.py,sha256=XOAfGvITCJPI1LTlISmHbA4ONMQMkxYUMT
 reconcile/test/test_terraform_vpc_peerings.py,sha256=ubcsKh0TrUIwuI1-W3ETIgzsFvzAyeoFmEJFC-IK6JY,20538
 reconcile/test/test_terraform_vpc_peerings_build_desired_state.py,sha256=DAfpb12I0PlqnuVUHK2vh4LH4d1OylT3H2GE_3TGZZI,47852
 reconcile/test/test_three_way_diff_strategy.py,sha256=2fjEqE2w4pIzKq18PRcADTSe01aGwsZfMGloU8xfNaE,3346
-reconcile/test/test_unleash.py,sha256=krPgOVmwTE6lb773040Ely9BPbNYOeOIY0_8BK72dgo,6690
 reconcile/test/test_utils_jinja2.py,sha256=TpzQlpFnLGzNEZp5WOh0o7AuBiGEktqO4MuwiiJW2YY,3895
 reconcile/test/test_vault_replication.py,sha256=wlc4jm9f8P641UvvxIFFFc5_unJysNkOVrKJscjhQr0,16867
 reconcile/test/test_vault_utils.py,sha256=vbJnc89XAuE07qbTuWxHM5o9F6R9SO5aHXA38fwxT7A,1122
@@ -600,6 +601,8 @@ reconcile/typed_queries/cost_report/cost_namespaces.py,sha256=VXIqdGE5lwa5z4UTRO
 reconcile/typed_queries/cost_report/settings.py,sha256=xbTMMUQnbub2pav4B-ctzzRe7ijjTv2bqfqdtb9OnO0,589
 reconcile/typed_queries/terraform_tgw_attachments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/typed_queries/terraform_tgw_attachments/aws_accounts.py,sha256=T5HSeyBcGKP-LzDmIzs-WlBwOtSenYpm0Odw5--xdOg,410
+reconcile/unleash_feature_toggles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/unleash_feature_toggles/integration.py,sha256=FLSgqSobefNUWH5ZWpTizbNeC9bkX65xmg-mjfsiENc,10975
 reconcile/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/aggregated_list.py,sha256=pkYoBj7WwmaNgEefETqEOFTnQMcUzHE3mdsVdzGYj60,3372
 reconcile/utils/amtool.py,sha256=Rg6_TTvuS7M2QbOkIjy2sRetvf6aNVdv_DA9ugSgMi4,2303
@@ -663,7 +666,7 @@ reconcile/utils/promtool.py,sha256=kT2rFZSBaRqW7SSHAuYzGZzQxM5Dzk8KW1NnEUYZU_s,2
 reconcile/utils/quay_api.py,sha256=EuOegpb-7ntEjkKLFwM2Oo4Nw7SyFtmyl3sQ9aXMtrM,8152
 reconcile/utils/raw_github_api.py,sha256=ZHC-SZuAyRe1zaMoOU7Krt1-zecDxENd9c_NzQYqK9g,2968
 reconcile/utils/repo_owners.py,sha256=j-pUjc9PuDzq7KpjNLpnhqfU8tUG4nj2WMhFp4ick7g,6629
-reconcile/utils/rest_api_base.py,sha256=X5o4idyRCDzwnF5xFwmjyoaHmM1tXSZnykTA54Z7D2Q,4006
+reconcile/utils/rest_api_base.py,sha256=jezb2MfIxf83HNSGbNNBStuEmVuVUuwFQnRmZsQaq6w,4297
 reconcile/utils/ruamel.py,sha256=FzL4_L0FnMOUZmgThrZSMJs5MTdXwiy-E9MZWfk8bh8,397
 reconcile/utils/secret_reader.py,sha256=7g4TuBxkOl2NgsuZUCRcdI_hKLP3JhXlY1byBSxWU3A,10305
 reconcile/utils/semver_helper.py,sha256=-WfPOMSA2v1h7hT3PwVf-Htg7wOsoKlQC1JdmDX2Ars,1268
@@ -678,7 +681,6 @@ reconcile/utils/terraform_client.py,sha256=mZEKpu6nbfiQd60wRkc8-5sljBTUTOgaAKnF8
 reconcile/utils/terrascript_aws_client.py,sha256=VlvIHgrZRiMFVgx6a8ZHxoiJoDwqbtOmsZFnwwNrdL0,273199
 reconcile/utils/three_way_diff_strategy.py,sha256=nyqeQsLCoPI6e16k2CF3b9KNgQLU-rPf5RtfdUfVMwE,4468
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
-reconcile/utils/unleash.py,sha256=1D56CsZfE3ShDtN3IErE1T2eeIwNmxhK-yYbCotJ99E,3601
 reconcile/utils/vault.py,sha256=AYGG5aDJ7CSVhTFdZowfEg3iSQWenoAt676aGjHQMX8,14978
 reconcile/utils/vaultsecretref.py,sha256=3Ed2uBy36TzSvL0B-l4FoWQqB2SbBKDKEuUPIO608Bo,931
 reconcile/utils/vcs.py,sha256=iiAWQXNftKIRoakXEOPT6ubB_ybSuInIQ6jcMxa_NKk,8558
@@ -771,6 +773,9 @@ reconcile/utils/terrascript/cloudflare_client.py,sha256=2g_DYiWP4-k8KDxz9Nnxrk7Z
 reconcile/utils/terrascript/cloudflare_resources.py,sha256=quyHhbexW4Y2ksVyEouV0SVFtZLs42xPoELK4LvzAqw,16043
 reconcile/utils/terrascript/models.py,sha256=x9HReI0k71MHBpRTvvmPlE0G6rri5GTzPXM9cqyTWm0,475
 reconcile/utils/terrascript/resources.py,sha256=bQzglnO41KZZEIeXYgi-qlup1p8R03Qyx_V944LRPsc,1391
+reconcile/utils/unleash/__init__.py,sha256=ayCrW7Cw9dxQYq56QwrjtTcCnMp0xAHr1R7re9jo6RM,219
+reconcile/utils/unleash/client.py,sha256=RBVafhOUjdPo7T9TIlTpS6ABogaxfimYj5-3FyOOLts,3551
+reconcile/utils/unleash/server.py,sha256=iq5bB6SY1ZWxDD6v19ym2TmCrbQr7fi8dCLVL9_FHV4,4438
 release/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 release/test_version.py,sha256=4DKxDZkT0aoS6ibZFsH2fd_wNETij1qfn2pCgQtVCGo,2093
 release/version.py,sha256=cwKdXBxraLokNeid2hk-3jsR9bRP_Ku-0Eve7o5E6q0,4032
@@ -779,7 +784,7 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=upA-J-n-HXHKVDINRuMR7vTt-iJvQORKUVi9D3leQto,17738
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=tAM6OXJkPnbSvZWOuhWQs9oKpL6Fn7qMEEirzCeuUY8,115006
+tools/qontract_cli.py,sha256=yBZSb2WoafCrJllcDREqhU8gN_5D_SThA2XxuLO9TEU,115172
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=-U-lTGeLaci8yWPEblCJeev2DOlY1jM9QOOh-O1zts8,3376
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -806,8 +811,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc816.dist-info/METADATA,sha256=0QOFfcd_sODX-xn2jp2xwRVg-Lc3NEY3eCbA6roBlUg,2314
-qontract_reconcile-0.10.1rc816.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc816.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc816.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc816.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc818.dist-info/METADATA,sha256=VBhOHCd2GQlM3ZcA-6Jp7B6ApHslI8WL2mb42eBBDSg,2314
+qontract_reconcile-0.10.1rc818.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc818.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc818.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc818.dist-info/RECORD,,

reconcile/cli.py

@@ -3562,6 +3562,23 @@ def acs_notifiers(ctx):
     )
 
 
+@integration.command(short_help="Manage Unleash feature toggles.")
+@click.option("--instance", help="Reconcile just this Unlash instance.", default=None)
+@click.pass_context
+def unleash_feature_toggles(ctx, instance):
+    from reconcile.unleash_feature_toggles.integration import (
+        UnleashTogglesIntegration,
+        UnleashTogglesIntegrationParams,
+    )
+
+    run_class_integration(
+        integration=UnleashTogglesIntegration(
+            UnleashTogglesIntegrationParams(instance=instance)
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Automate Deadmanssnitch Creation/Deletion")
 @click.pass_context
 def deadmanssnitch(ctx):

reconcile/gql_definitions/unleash_feature_toggles/feature_toggles.py

@@ -0,0 +1,111 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query UnleashFeatureToggles {
+  instances: unleash_instances_v1 {
+    name
+    url
+    adminToken {
+      ...VaultSecret
+    }
+    allowUnmanagedFeatureToggles
+    projects {
+      name
+      feature_toggles {
+        name
+        description
+        delete
+        provider
+        unleash {
+          type
+          impressionData
+          environments
+        }
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class UnleashFeatureToggleV1(ConfiguredBaseModel):
+    q_type: Optional[str] = Field(..., alias="type")
+    impression_data: Optional[bool] = Field(..., alias="impressionData")
+    environments: Optional[Json] = Field(..., alias="environments")
+
+
+class FeatureToggleUnleashV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    description: str = Field(..., alias="description")
+    delete: Optional[bool] = Field(..., alias="delete")
+    provider: str = Field(..., alias="provider")
+    unleash: UnleashFeatureToggleV1 = Field(..., alias="unleash")
+
+
+class UnleashProjectV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    feature_toggles: Optional[list[FeatureToggleUnleashV1]] = Field(..., alias="feature_toggles")
+
+
+class UnleashInstanceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    url: str = Field(..., alias="url")
+    admin_token: Optional[VaultSecret] = Field(..., alias="adminToken")
+    allow_unmanaged_feature_toggles: Optional[bool] = Field(..., alias="allowUnmanagedFeatureToggles")
+    projects: Optional[list[UnleashProjectV1]] = Field(..., alias="projects")
+
+
+class UnleashFeatureTogglesQueryData(ConfiguredBaseModel):
+    instances: Optional[list[UnleashInstanceV1]] = Field(..., alias="instances")
+
+
+def query(query_func: Callable, **kwargs: Any) -> UnleashFeatureTogglesQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        UnleashFeatureTogglesQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return UnleashFeatureTogglesQueryData(**raw_data)

reconcile/unleash_feature_toggles/integration.py

@@ -0,0 +1,287 @@
+import logging
+from collections.abc import Callable, Iterable, Sequence, ValuesView
+from typing import Any
+
+from reconcile.gql_definitions.unleash_feature_toggles.feature_toggles import (
+    FeatureToggleUnleashV1,
+    UnleashInstanceV1,
+    UnleashProjectV1,
+)
+from reconcile.gql_definitions.unleash_feature_toggles.feature_toggles import (
+    query as unleash_instances_query,
+)
+from reconcile.utils import gql
+from reconcile.utils.differ import DiffPair, diff_any_iterables, diff_mappings
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.unleash.server import (
+    Environment,
+    FeatureToggle,
+    FeatureToggleType,
+    Project,
+    TokenAuth,
+    UnleashServer,
+)
+
+QONTRACT_INTEGRATION = "unleash-feature-toggles"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+
+
+class UnleashTogglesIntegrationParams(PydanticRunParams):
+    instance: str | None
+
+
+def feature_toggle_equal(c: FeatureToggle, d: FeatureToggleUnleashV1) -> bool:
+    """Check if two feature toggles are different, but ignore the actual toggle state."""
+    assert d.unleash.q_type  # make mypy happy
+    return (
+        c.description == d.description
+        and c.type == FeatureToggleType[d.unleash.q_type]
+        and c.impression_data == bool(d.unleash.impression_data)
+    )
+
+
+class UnleashFeatureToggleException(Exception):
+    """Raised when a feature toggle is manually created."""
+
+
+class UnleashFeatureToggleDeleteError(Exception):
+    """Raised when a feature toggle is not marked for deletion."""
+
+
+class UnleashTogglesIntegration(
+    QontractReconcileIntegration[UnleashTogglesIntegrationParams]
+):
+    """Manage Unleash feature toggles."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(
+        self, query_func: Callable | None = None
+    ) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        if not query_func:
+            query_func = gql.get_api().query
+        return {
+            "toggles": [ft.dict() for ft in self.get_unleash_instances(query_func)],
+        }
+
+    def get_unleash_instances(
+        self, query_func: Callable, instance: str | None = None
+    ) -> list[UnleashInstanceV1]:
+        """Get all Unleash instances with their projects and feature toggles."""
+        instances = [
+            ui
+            for ui in unleash_instances_query(query_func).instances or []
+            if (not instance or ui.name == instance) and ui.admin_token
+        ]
+
+        # Set default values for all feature toggles
+        for inst in instances:
+            for project in inst.projects or []:
+                for feature_toggle in project.feature_toggles or []:
+                    feature_toggle.unleash.q_type = (
+                        feature_toggle.unleash.q_type or "release"
+                    )
+                    feature_toggle.unleash.impression_data = (
+                        feature_toggle.unleash.impression_data or False
+                    )
+        return instances
+
+    def fetch_current_state(self, client: UnleashServer) -> list[Project]:
+        """Fetch the current state of all Unleash projects including their feature toggles."""
+        return client.projects(include_feature_toggles=True)
+
+    def validate_unleash_projects(
+        self,
+        current_projects: Iterable[Project],
+        desired_projects: Iterable[UnleashProjectV1],
+    ) -> ValuesView[DiffPair[Project, UnleashProjectV1]]:
+        """Validate that all projects referenced in the desired state are actually exist."""
+        diff = diff_any_iterables(
+            current=current_projects,
+            desired=desired_projects,
+            current_key=lambda c: c.name.lower(),
+            desired_key=lambda d: d.name.lower(),
+            equal=lambda c, d: c.name.lower() == d.name.lower(),
+        )
+        if diff.add:
+            raise ValueError(f"Non-existing projects: {','.join(diff.add.keys())}")
+        return diff.identical.values()
+
+    def _reconcile_feature_toggles(
+        self,
+        client: UnleashServer,
+        instance: UnleashInstanceV1,
+        project_id: str,
+        dry_run: bool,
+        current_state: Sequence[FeatureToggle],
+        desired_state: Iterable[FeatureToggleUnleashV1],
+    ) -> None:
+        """Reconcile the feature toggles themselves."""
+        diff = diff_any_iterables(
+            current=current_state,
+            desired=[d for d in desired_state if d.delete is not True],
+            current_key=lambda c: c.name,
+            desired_key=lambda d: d.name,
+            equal=feature_toggle_equal,
+        )
+        for add in diff.add.values():
+            logging.info(
+                f"[{instance.name}/{project_id}] Adding feature toggle {add.name}"
+            )
+            try:
+                assert add.unleash.q_type  # make mypy happy
+                feature_type = FeatureToggleType[add.unleash.q_type]
+            except KeyError:
+                raise ValueError(
+                    f"[{instance.name}/{project_id}/{add.name}] Invalid feature toggle type '{add.unleash.q_type}', Possible values are: {', '.join(FeatureToggleType.__members__)}"
+                )
+            if not dry_run:
+                client.create_feature_toggle(
+                    project_id=project_id,
+                    name=add.name,
+                    description=add.description,
+                    type=feature_type,
+                    impression_data=bool(add.unleash.impression_data),
+                )
+
+        for change in diff.change.values():
+            logging.info(
+                f"[{instance.name}/{project_id}] Changing feature toggle {change.desired.name}"
+            )
+            try:
+                assert change.desired.unleash.q_type  # make mypy happy
+                feature_type = FeatureToggleType[change.desired.unleash.q_type]
+            except KeyError:
+                raise ValueError(
+                    f"[{instance.name}/{project_id}/{change.current.name}] Invalid feature toggle type '{change.desired.unleash.q_type}', Possible values are: {', '.join(FeatureToggleType.__members__)}"
+                )
+            if not dry_run:
+                client.update_feature_toggle(
+                    project_id=project_id,
+                    name=change.current.name,
+                    description=change.desired.description,
+                    type=feature_type,
+                    impression_data=bool(change.desired.unleash.impression_data),
+                )
+
+        for delete in diff.delete.values():
+            desired_toggle = next(
+                (d for d in desired_state if d.name == delete.name and d.delete), None
+            )
+            if desired_toggle:
+                logging.info(
+                    f"[{instance.name}/{project_id}] Deleting feature toggle {delete.name}"
+                )
+                if not dry_run:
+                    client.delete_feature_toggle(
+                        project_id=project_id,
+                        name=delete.name,
+                    )
+            elif not instance.allow_unmanaged_feature_toggles:
+                raise UnleashFeatureToggleDeleteError(
+                    f"[{instance.name}/{project_id}] Found unmanaged feature toggles '{[d.name for d in diff.delete.values()]}'"
+                )
+
+    def _reconcile_states(
+        self,
+        client: UnleashServer,
+        instance: UnleashInstanceV1,
+        project_id: str,
+        dry_run: bool,
+        current_state: Sequence[FeatureToggle],
+        desired_state: Iterable[FeatureToggleUnleashV1],
+        available_environments: Iterable[Environment],
+    ) -> None:
+        """Reconcile the feature toggle states."""
+        current_toggle_states = {
+            (state.name, env.name): env.enabled
+            for state in current_state
+            for env in state.environments
+        }
+        desired_toggle_states = {
+            (state.name, env_name): enabled
+            for state in desired_state
+            for env_name, enabled in (state.unleash.environments or {}).items()
+        }
+
+        diff_result = diff_mappings(
+            current=current_toggle_states, desired=desired_toggle_states
+        )
+        for (name, env), pair in diff_result.change.items():
+            if env not in available_environments:
+                raise ValueError(
+                    f"[{instance.name}/{project_id}/{name}] Environment '{env}' does not exist in Unleash!"
+                )
+            if not dry_run:
+                client.set_feature_toggle_state(
+                    project_id=project_id,
+                    name=name,
+                    environment=env,
+                    enabled=pair.desired,
+                )
+
+    def reconcile(
+        self,
+        client: UnleashServer,
+        instance: UnleashInstanceV1,
+        project_id: str,
+        dry_run: bool,
+        current_state: Sequence[FeatureToggle],
+        desired_state: Iterable[FeatureToggleUnleashV1],
+    ) -> None:
+        """Reconcile the feature toggles."""
+        self._reconcile_feature_toggles(
+            client=client,
+            instance=instance,
+            project_id=project_id,
+            dry_run=dry_run,
+            current_state=current_state,
+            desired_state=desired_state,
+        )
+        self._reconcile_states(
+            client=client,
+            instance=instance,
+            project_id=project_id,
+            dry_run=dry_run,
+            current_state=current_state,
+            desired_state=desired_state,
+            available_environments=client.environments(project_id),
+        )
+
+    def run(self, dry_run: bool) -> None:
+        gql_api = gql.get_api()
+        instances = self.get_unleash_instances(
+            gql_api.query, instance=self.params.instance
+        )
+
+        for instance in instances:
+            assert instance.admin_token  # make mypy happy
+            with UnleashServer(
+                host=instance.url,
+                auth=TokenAuth(self.secret_reader.read_secret(instance.admin_token)),
+            ) as client:
+                try:
+                    project_pairs = self.validate_unleash_projects(
+                        current_projects=self.fetch_current_state(client),
+                        desired_projects=instance.projects or [],
+                    )
+                except ValueError:
+                    logging.error(f"[{instance.name}] Missing projects!")
+                    raise
+
+                for pair in project_pairs:
+                    self.reconcile(
+                        client=client,
+                        instance=instance,
+                        project_id=pair.current.pk,
+                        dry_run=dry_run,
+                        current_state=pair.current.feature_toggles,
+                        desired_state=pair.desired.feature_toggles or [],
+                    )

reconcile/utils/rest_api_base.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any, Self
 from urllib.parse import urljoin
 
@@ -76,7 +77,14 @@ class ApiBase:
     def _get(self, url: str) -> dict[str, Any]:
         response = self.session.get(urljoin(self.host, url), timeout=self.read_timeout)
         response.raise_for_status()
-        return response.json()
+        try:
+            return response.json()
+        except requests.exceptions.JSONDecodeError:
+            logging.error(
+                f"Failed to decode JSON response from {url}"
+                f"Response: {response.text}"
+            )
+            raise
 
     def _list(
         self, url: str, params: dict | None = None, attribute: str | None = None
@@ -100,7 +108,7 @@ class ApiBase:
             urljoin(self.host, url), json=data, timeout=self.read_timeout
         )
         response.raise_for_status()
-        if response.status_code == 204:
+        if response.status_code == 204 or not response.text:
             return {}
         return response.json()
 
@@ -109,7 +117,7 @@ class ApiBase:
             urljoin(self.host, url), json=data, timeout=self.read_timeout
         )
         response.raise_for_status()
-        if response.status_code == 204:
+        if response.status_code == 204 or not response.text:
             return {}
         return response.json()
 

reconcile/utils/unleash/__init__.py

@@ -0,0 +1,11 @@
+from .client import (
+    get_feature_toggle_default,
+    get_feature_toggle_state,
+    get_feature_toggles,
+)
+
+__all__ = [
+    "get_feature_toggles",
+    "get_feature_toggle_state",
+    "get_feature_toggle_default",
+]

reconcile/utils/{unleash.py → unleash/client.py}

@@ -18,22 +18,22 @@ client_lock = threading.Lock()
 
 
 class CacheDict(BaseCache):
-    def __init__(self):
-        self.cache = {}
+    def __init__(self) -> None:
+        self.cache: dict = {}
 
-    def set(self, key: str, value: Any):
+    def set(self, key: str, value: Any) -> None:
         self.cache[key] = value
 
-    def mset(self, data: dict):
+    def mset(self, data: dict) -> None:
         self.cache.update(data)
 
-    def get(self, key: str, default: Optional[Any] = None):
+    def get(self, key: str, default: Any | None = None) -> Any:
         return self.cache.get(key, default)
 
-    def exists(self, key: str):
+    def exists(self, key: str) -> bool:
         return key in self.cache
 
-    def destroy(self):
+    def destroy(self) -> None:
         self.cache = {}
 
 
@@ -85,28 +85,21 @@ def _get_unleash_api_client(api_url: str, auth_head: str) -> UnleashClient:
     return client
 
 
-def _shutdown_client():
-    # Intended for test usage only
-    with client_lock:
-        if client:
-            client.destroy()
-
-
-def get_feature_toggle_default(feature_name, context):
+def get_feature_toggle_default(feature_name: str, context: dict) -> bool:
     return True
 
 
-def get_feature_toggle_default_false(feature_name, context):
+def get_feature_toggle_default_false(feature_name: str, context: dict) -> bool:
     return False
 
 
 def get_feature_toggle_state(
-    integration_name: str, context: Optional[dict] = None, default: bool = True
+    integration_name: str, context: dict | None = None, default: bool = True
 ) -> bool:
     api_url = os.environ.get("UNLEASH_API_URL")
     client_access_token = os.environ.get("UNLEASH_CLIENT_ACCESS_TOKEN")
     if not (api_url and client_access_token):
-        return get_feature_toggle_default(None, None)
+        return get_feature_toggle_default("", {})
 
     c = _get_unleash_api_client(
         api_url,

reconcile/utils/unleash/server.py

@@ -0,0 +1,145 @@
+from enum import Enum
+
+import requests
+from pydantic import BaseModel, Field
+
+from reconcile.utils.rest_api_base import ApiBase, BearerTokenAuth
+
+
+class FeatureToggleType(str, Enum):
+    experiment = "experiment"
+    kill_switch = "kill-switch"
+    release = "release"
+    operational = "operational"
+    permission = "permission"
+
+
+class Environment(BaseModel):
+    name: str
+    enabled: bool
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Environment):
+            return self.name == other.name
+        return self.name == other
+
+
+class FeatureToggle(BaseModel):
+    name: str
+    type: FeatureToggleType = FeatureToggleType.release
+    description: str | None = None
+    impression_data: bool = Field(False, alias="impressionData")
+    environments: list[Environment]
+
+    class Config:
+        allow_population_by_field_name = True
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, FeatureToggle):
+            return self.name == other.name
+        return self.name == other
+
+
+class Project(BaseModel):
+    pk: str = Field(alias="id")
+    name: str
+    feature_toggles: list[FeatureToggle] = []
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class TokenAuth(BearerTokenAuth):
+    def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
+        r.headers["Authorization"] = self.token
+        return r
+
+
+class UnleashServer(ApiBase):
+    """Unleash server API client."""
+
+    def projects(self, include_feature_toggles: bool = False) -> list[Project]:
+        """List all projects."""
+        projects = []
+        for project_data in self._list("/api/admin/projects", attribute="projects"):
+            project = Project(**project_data)
+            if include_feature_toggles:
+                project.feature_toggles = self.feature_toggles(project.pk)
+            projects.append(project)
+        return projects
+
+    def feature_toggles(self, project_id: str) -> list[FeatureToggle]:
+        """List all feature toggles for a project."""
+        return [
+            FeatureToggle(**i)
+            for i in self._list(
+                f"/api/admin/projects/{project_id}/features", attribute="features"
+            )
+        ]
+
+    def environments(self, project_id: str) -> list[Environment]:
+        """Gets the environments that are available for this project. An environment is available for a project if enabled in the project configuration."""
+        return [
+            Environment(**i)
+            for i in self._list(
+                f"/api/admin/environments/project/{project_id}",
+                attribute="environments",
+            )
+        ]
+
+    def create_feature_toggle(
+        self,
+        project_id: str,
+        name: str,
+        description: str,
+        type: FeatureToggleType,
+        impression_data: bool,
+    ) -> None:
+        """Create a feature toggle."""
+        self._post(
+            f"/api/admin/projects/{project_id}/features",
+            data={
+                "name": name,
+                "description": description,
+                "type": type.value,
+                "impressionData": impression_data,
+            },
+        )
+
+    def update_feature_toggle(
+        self,
+        project_id: str,
+        name: str,
+        description: str,
+        type: FeatureToggleType,
+        impression_data: bool,
+    ) -> None:
+        """Create a feature toggle."""
+        self._put(
+            f"/api/admin/projects/{project_id}/features/{name}",
+            data={
+                "description": description,
+                "type": type.value,
+                "impressionData": impression_data,
+            },
+        )
+
+    def delete_feature_toggle(self, project_id: str, name: str) -> None:
+        """Delete a feature toggle."""
+        # First archive the feature toggle
+        self._delete(f"/api/admin/projects/{project_id}/features/{name}")
+        # Then delete it
+        self._post(
+            f"/api/admin/projects/{project_id}/delete",
+            data={"features": [name]},
+        )
+
+    def set_feature_toggle_state(
+        self, project_id: str, name: str, environment: str, enabled: bool
+    ) -> None:
+        """Set the state of a feature toggle."""
+        base_url = f"/api/admin/projects/{project_id}/features/{name}/environments/{environment}"
+        if enabled:
+            self._post(f"{base_url}/on")
+        else:
+            self._post(f"{base_url}/off")

tools/qontract_cli.py

@@ -1579,6 +1579,7 @@ def rds(ctx):
                 "engine": rds_attr("engine", overrides, defaults),
                 "engine_version": rds_attr("engine_version", overrides, defaults),
                 "instance_class": rds_attr("instance_class", overrides, defaults),
+                "storage_type": rds_attr("storage_type", overrides, defaults),
             }
             results.append(item)
 
@@ -1593,6 +1594,7 @@ def rds(ctx):
                 {"key": "engine", "sortable": True},
                 {"key": "engine_version", "sortable": True},
                 {"key": "instance_class", "sortable": True},
+                {"key": "storage_type", "sortable": True},
             ],
             "items": results,
         }
@@ -1617,6 +1619,7 @@ You can view the source of this Markdown to extract the JSON data.
             "engine",
             "engine_version",
             "instance_class",
+            "storage_type",
         ]
         ctx.obj["options"]["sort"] = False
         print_output(ctx.obj["options"], results, columns)

reconcile/test/test_unleash.py

@@ -1,214 +0,0 @@
-import os
-from collections.abc import Callable
-
-import pytest
-from pytest_httpserver import HTTPServer
-from UnleashClient.features import Feature
-
-import reconcile.utils.unleash
-from reconcile.utils.unleash import (
-    DisableClusterStrategy,
-    EnableClusterStrategy,
-    _get_unleash_api_client,
-    _shutdown_client,
-    get_feature_toggle_default,
-    get_feature_toggle_state,
-    get_feature_toggles,
-)
-
-
-@pytest.fixture
-def reset_client():
-    reconcile.utils.unleash.client = None
-
-
-def _setup_unleash_httpserver(features: dict, httpserver: HTTPServer) -> HTTPServer:
-    httpserver.expect_request("/client/features").respond_with_json(features)
-    httpserver.expect_request("/client/register", method="post").respond_with_data(
-        status=202
-    )
-    return httpserver
-
-
-@pytest.fixture
-def setup_unleash_disable_cluster_strategy(httpserver: HTTPServer):
-    def _(enabled: bool) -> HTTPServer:
-        features = {
-            "version": 2,
-            "features": [
-                {
-                    "strategies": [
-                        {
-                            "name": "disableCluster",
-                            "constraints": [],
-                            "parameters": {"cluster_name": "foo"},
-                        },
-                    ],
-                    "impressionData": False,
-                    "enabled": enabled,
-                    "name": "test-strategies",
-                    "description": "",
-                    "project": "default",
-                    "stale": False,
-                    "type": "release",
-                    "variants": [],
-                }
-            ],
-        }
-        return _setup_unleash_httpserver(features, httpserver)
-
-    return _
-
-
-@pytest.fixture
-def setup_unleash_enable_cluster_strategy(httpserver: HTTPServer):
-    def _(enabled: bool) -> HTTPServer:
-        features = {
-            "version": 2,
-            "features": [
-                {
-                    "strategies": [
-                        {
-                            "name": "enableCluster",
-                            "constraints": [],
-                            "parameters": {"cluster_name": "enabled-cluster"},
-                        },
-                    ],
-                    "impressionData": False,
-                    "enabled": enabled,
-                    "name": "test-strategies",
-                    "description": "",
-                    "project": "default",
-                    "stale": False,
-                    "type": "release",
-                    "variants": [],
-                }
-            ],
-        }
-
-        return _setup_unleash_httpserver(features, httpserver)
-
-    return _
-
-
-def test__get_unleash_api_client(mocker):
-    mocked_unleash_client = mocker.patch(
-        "reconcile.utils.unleash.UnleashClient",
-        autospec=True,
-    )
-    mocked_cache_dict = mocker.patch(
-        "reconcile.utils.unleash.CacheDict",
-        autospec=True,
-    )
-
-    c = _get_unleash_api_client("https://u/api", "foo")
-
-    mocked_unleash_client.assert_called_once_with(
-        url="https://u/api",
-        app_name="qontract-reconcile",
-        custom_headers={"Authorization": "foo"},
-        cache=mocked_cache_dict.return_value,
-        custom_strategies={
-            "enableCluster": EnableClusterStrategy,
-            "disableCluster": DisableClusterStrategy,
-        },
-    )
-    mocked_unleash_client.return_value.initialize_client.assert_called_once_with()
-    assert reconcile.utils.unleash.client == c
-
-
-def test__get_unleash_api_client_skip_create(mocker):
-    mocked_unleash_client = mocker.patch(
-        "reconcile.utils.unleash.UnleashClient",
-        autospec=True,
-    )
-    u = mocked_unleash_client.return_value
-    reconcile.utils.unleash.client = u
-
-    c = _get_unleash_api_client("https://u/api", "foo")
-
-    mocked_unleash_client.assert_not_called()
-    assert reconcile.utils.unleash.client == c == u
-
-
-def test_get_feature_toggle_default():
-    assert get_feature_toggle_default(None, None)
-
-
-def test_get_feature_toggle_state_env_missing():
-    assert get_feature_toggle_state("foo")
-
-
-def test_get_feature_toggle_state(mocker, monkeypatch):
-    def enabled_func(feature, context, fallback_function):
-        return feature == "enabled"
-
-    os.environ["UNLEASH_API_URL"] = "foo"
-    os.environ["UNLEASH_CLIENT_ACCESS_TOKEN"] = "bar"
-
-    defaultfunc = mocker.patch(
-        "reconcile.utils.unleash.get_feature_toggle_default", return_value=True
-    )
-    monkeypatch.setattr(
-        "reconcile.utils.unleash.client",
-        mocker.patch("UnleashClient.UnleashClient", autospec=True),
-    )
-    mocker.patch(
-        "UnleashClient.UnleashClient.is_enabled",
-        side_effect=enabled_func,
-    )
-
-    assert get_feature_toggle_state("enabled")
-    assert get_feature_toggle_state("disabled") is False
-    assert defaultfunc.call_count == 0
-
-
-def test_get_feature_toggles(mocker, monkeypatch):
-    c = mocker.patch("UnleashClient.UnleashClient")
-    c.features = {
-        "foo": Feature("foo", False, []),
-        "bar": Feature("bar", True, []),
-    }
-
-    monkeypatch.setattr("reconcile.utils.unleash.client", c)
-    toggles = get_feature_toggles("api", "token")
-
-    assert toggles["foo"] == "disabled"
-    assert toggles["bar"] == "enabled"
-
-
-def test_get_feature_toggle_state_with_strategy(
-    reset_client: None, setup_unleash_disable_cluster_strategy: Callable
-):
-    httpserver = setup_unleash_disable_cluster_strategy(True)
-    os.environ["UNLEASH_API_URL"] = httpserver.url_for("/")
-    os.environ["UNLEASH_CLIENT_ACCESS_TOKEN"] = "bar"
-    assert not get_feature_toggle_state(
-        "test-strategies", context={"cluster_name": "foo"}
-    )
-    assert get_feature_toggle_state("test-strategies", context={"cluster_name": "bar"})
-    _shutdown_client()
-
-
-def test_get_feature_toggle_state_disabled_with_strategy(
-    reset_client: None, setup_unleash_disable_cluster_strategy: Callable
-):
-    httpserver = setup_unleash_disable_cluster_strategy(False)
-    os.environ["UNLEASH_API_URL"] = httpserver.url_for("/")
-    os.environ["UNLEASH_CLIENT_ACCESS_TOKEN"] = "bar"
-    assert not get_feature_toggle_state(
-        "test-strategies", context={"cluster_name": "bar"}
-    )
-    _shutdown_client()
-
-
-def test_get_feature_toggle_state_with_enable_cluster_strategy(
-    reset_client: None, setup_unleash_enable_cluster_strategy: Callable
-):
-    httpserver = setup_unleash_enable_cluster_strategy(True)
-    os.environ["UNLEASH_API_URL"] = httpserver.url_for("/")
-    os.environ["UNLEASH_CLIENT_ACCESS_TOKEN"] = "bar"
-    assert get_feature_toggle_state(
-        "test-strategies", context={"cluster_name": "enabled-cluster"}
-    )
-    _shutdown_client()