qontract-reconcile 0.10.1rc1048__py3-none-any.whl → 0.10.1rc1049__py3-none-any.whl

{qontract_reconcile-0.10.1rc1048.dist-info → qontract_reconcile-0.10.1rc1049.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1048
+Version: 0.10.1rc1049
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1048.dist-info → qontract_reconcile-0.10.1rc1049.dist-info}/RECORD

@@ -72,7 +72,7 @@ reconcile/openshift_network_policies.py,sha256=DyjaeJvSFHmslbM8nyHCxpF9EtU2m-MJo
 reconcile/openshift_prometheus_rules.py,sha256=onowXab248zmHH8SbYDTc1W1bl7JiqRFU1xdTkZyLFg,1332
 reconcile/openshift_resourcequotas.py,sha256=yUi56PiOn3inMMfq_x_FEHmaW-reGipzoorjdar372g,2415
 reconcile/openshift_resources.py,sha256=I2nO_C37mG3rfyGrd4cGwN3mVseVGuTAHAyhFzLyqF4,1518
-reconcile/openshift_resources_base.py,sha256=SKlJffnYjnc7AHIOvi7Ui9TmDA93_t81d5t54wnqCOA,40745
+reconcile/openshift_resources_base.py,sha256=1A5_699p0rdsMwRQRPzePEfjhhq5eB2Obwxx4Ibr8jA,41205
 reconcile/openshift_rolebindings.py,sha256=9mlJ2FjWUoH-rsjtasreA_hV-K5Z_YR00qR_RR60OZM,6555
 reconcile/openshift_routes.py,sha256=fXvuPSjcjVw1X3j2EQvUAdbOepmIFdKk-M3qP8QzPiw,1075
 reconcile/openshift_saas_deploy.py,sha256=UZlm29JujJVS3MzSm6uehlC3y-jZxl6WVwMeKRdN11U,12773
@@ -571,7 +571,7 @@ reconcile/test/test_terraform_users.py,sha256=XOAfGvITCJPI1LTlISmHbA4ONMQMkxYUMT
 reconcile/test/test_terraform_vpc_peerings.py,sha256=bpjCjhmic07cw3XKSHf-2JvmLuWlyQG8laXlC-H7qtI,20796
 reconcile/test/test_terraform_vpc_peerings_build_desired_state.py,sha256=cHmr1_yhRgfdqlFX6TMw-aiKXebaRv0szl16M9YRJic,49988
 reconcile/test/test_three_way_diff_strategy.py,sha256=v3rNkQFNy5e1uyfeNSlNBA07fvrPGD0aXD91Lgv8oxc,4062
-reconcile/test/test_utils_jinja2.py,sha256=TpzQlpFnLGzNEZp5WOh0o7AuBiGEktqO4MuwiiJW2YY,3895
+reconcile/test/test_utils_jinja2.py,sha256=rKugJEPl0qFC9joenJBXyk2qe-9md31-4EdxvQ2h5cs,4058
 reconcile/test/test_vault_replication.py,sha256=wlc4jm9f8P641UvvxIFFFc5_unJysNkOVrKJscjhQr0,16867
 reconcile/test/test_vault_utils.py,sha256=vbJnc89XAuE07qbTuWxHM5o9F6R9SO5aHXA38fwxT7A,1122
 reconcile/test/test_version_bump.py,sha256=q6-3Y1roriI6YWpFwaHOMN7emEP3yL33sh_0VdbmG7E,511
@@ -652,7 +652,7 @@ reconcile/unleash_feature_toggles/integration.py,sha256=nx7BhtzCsTfPbOp60vI5MkNw
 reconcile/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/aggregated_list.py,sha256=km0xadW0jO4G_CqZPsXmoBURQ8c90FaTu5x4X1K1cZs,3357
 reconcile/utils/amtool.py,sha256=ngtBuVPETH6oAy5RnKzvreVbjwQCaATS_PYYwBprzjQ,2288
-reconcile/utils/aws_api.py,sha256=tYjTEW1uSkWsMWNFxwavJsI9xtq1dEeJ2HPlN5LAHzA,66861
+reconcile/utils/aws_api.py,sha256=1EC9l2cxMW2IvnhsXaIcq6iohZ4rRgqyHJsXPs9Vtes,67398
 reconcile/utils/aws_helper.py,sha256=MDbv5jrNdqqJ5pfBxniGdJXBBO_EYc2_Uf2w9ZzeMNs,2854
 reconcile/utils/batches.py,sha256=TtEm64a8lWhFuNbUVpFEmXVdU2Q0sTBrP_I0Cjbgh7g,320
 reconcile/utils/binary.py,sha256=7MaAFBpzuBUTJ_aA6G6-eult_BPMVyiXbBLD0Y6F-DM,2301
@@ -761,8 +761,8 @@ reconcile/utils/internal_groups/client.py,sha256=V3uubJJHqW_wjwrwbC0halZrhOy7_yx
 reconcile/utils/internal_groups/models.py,sha256=y_IqBVqfGqNXiu0VudvBWFrm_-uafVm5KgLG-ca8XAs,2281
 reconcile/utils/jinja2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jinja2/extensions.py,sha256=7K-uo6G2eCWa98MHT8fRPYIKCLQB_5D2keqQ_LyAfHM,1293
-reconcile/utils/jinja2/filters.py,sha256=tmiaYMhji5fv4B66YtR7zc-mE3wQLyj5I5SeX0WA2l4,4754
-reconcile/utils/jinja2/utils.py,sha256=a4vpt8f0UHerPAZs5zFESkVzZdEZRJjZ4mzJ19nubjI,8048
+reconcile/utils/jinja2/filters.py,sha256=JfO_14APySBPidsMvHXG-8dULNPddZCE15Umjk_aSBk,4830
+reconcile/utils/jinja2/utils.py,sha256=adxVFY4WvBGIToEEr8KwqLzp6uDxSipTBzZWwnRqbNQ,8700
 reconcile/utils/jobcontroller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jobcontroller/controller.py,sha256=n9twSH_xZtp5vknM3qByp0eWCn9ED4U1a6yOFBS-GTg,14488
 reconcile/utils/jobcontroller/models.py,sha256=x9YIvWfYOOvXNKToFVx1H7qDrZb0Sa1KI_4Y0gl7rMM,6336
@@ -867,8 +867,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1048.dist-info/METADATA,sha256=0w3Sz5JM9zH3Sa3crv-Owf8f3uYAgeEafoXtXm0TdXg,2213
-qontract_reconcile-0.10.1rc1048.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1048.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1048.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1048.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1049.dist-info/METADATA,sha256=D5PDDDmCzGGwdJ4MCHf2rBEApxpHBaSAtT97Krh_aWA,2213
+qontract_reconcile-0.10.1rc1049.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1049.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1049.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1049.dist-info/RECORD,,

reconcile/openshift_resources_base.py

@@ -1194,6 +1194,7 @@ def early_exit_monkey_patch() -> Generator:
         lookup_github_file_content=DEFAULT,
         url_makes_sense=DEFAULT,
         lookup_s3_object=DEFAULT,
+        list_s3_objects=DEFAULT,
     ) as mocks:
         # mock lookup_secret
         mocks["lookup_secret"].side_effect = (
@@ -1239,6 +1240,17 @@ def early_exit_monkey_patch() -> Generator:
         mocks["lookup_s3_object"].unsafe_callable = False
         mocks["lookup_s3_object"].alters_data = False
 
+        # mock list_s3_objects
+        mocks["list_s3_objects"].side_effect = (
+            lambda account_name,
+            bucket_name,
+            path,
+            region_name=None: f"list_s3_objects({account_name}, {bucket_name}, {path}, {region_name})"
+        )
+        # needed for jinja2 `is_safe_callable`
+        mocks["list_s3_objects"].unsafe_callable = False
+        mocks["list_s3_objects"].alters_data = False
+
         with patch(
             "reconcile.openshift_resources_base.check_alertmanager_config",
             return_value=True,

reconcile/test/test_utils_jinja2.py

@@ -6,6 +6,7 @@ from reconcile.utils.jinja2.filters import (
     hash_list,
     json_pointers,
     matches_jsonpath,
+    str_format,
 )
 
 
@@ -121,3 +122,9 @@ def test_json_pointers() -> None:
     assert json_pointers(input, "items[?@.name=='a']") == ["/items/0"]
     assert json_pointers(input, "items[?@.name=='b']") == ["/items/1", "/items/2"]
     assert json_pointers(input, "items[?@.name=='c']") == []
+
+
+def test_str_format() -> None:
+    value = "path/to/object"
+    format = "s3://%s"
+    assert str_format(value, format) == "s3://path/to/object"

reconcile/utils/aws_api.py

@@ -1665,6 +1665,24 @@ class AWSApi:  # pylint: disable=too-many-public-methods
             s3.get_object(Bucket=bucket_name, Key=path)["Body"].read().decode("utf-8")
         )
 
+    def list_s3_objects(
+        self,
+        account_name: str,
+        bucket_name: str,
+        path: str,
+        region_name: str | None = None,
+    ) -> list[str]:
+        s3 = self._account_s3_client(account_name, region_name=region_name)
+        objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=path, Delimiter="/")[
+            "Contents"
+        ]
+        return [
+            obj["Key"]
+            for obj in sorted(
+                objects, key=lambda obj: obj["LastModified"], reverse=True
+            )
+        ]
+
 
 def aws_config_file_path() -> str | None:
     config_file_path = os.path.expanduser(

reconcile/utils/jinja2/filters.py

@@ -136,3 +136,7 @@ def json_pointers(input: Any, jsonpath: str) -> list[str]:
     Finds the RFC6901 JSON pointers of the input elements matching the given jsonpath
     """
     return [_convert_pointer(str(i.full_path)) for i in _find_jsonpath(input, jsonpath)]
+
+
+def str_format(value: str, format: str) -> str:
+    return format % value

reconcile/utils/jinja2/utils.py

@@ -21,6 +21,7 @@ from reconcile.utils.jinja2.filters import (
     json_pointers,
     json_to_dict,
     matches_jsonpath,
+    str_format,
     urlescape,
     urlunescape,
     yaml_to_dict,
@@ -89,6 +90,7 @@ def compile_jinja2_template(
         "extract_jsonpath": extract_jsonpath,
         "matches_jsonpath": matches_jsonpath,
         "json_pointers": json_pointers,
+        "str_format": str_format,
     })
 
     return jinja_env.from_string(body)
@@ -146,6 +148,26 @@ def lookup_s3_object(
         )
 
 
+def list_s3_objects(
+    account_name: str,
+    bucket_name: str,
+    path: str,
+    region_name: str | None = None,
+) -> list[str]:
+    settings = queries.get_app_interface_settings()
+    accounts = queries.get_aws_accounts(name=account_name)
+    if not accounts:
+        raise Exception(f"aws account not found: {account_name}")
+
+    with AWSApi(1, accounts, settings=settings, init_users=False) as aws_api:
+        return aws_api.list_s3_objects(
+            account_name,
+            bucket_name,
+            path,
+            region_name=region_name,
+        )
+
+
 @retry()
 def lookup_secret(
     path: str,
@@ -214,6 +236,7 @@ def process_jinja2_template(
         "query": lookup_graphql_query_results,
         "url": url_makes_sense,
         "s3": lookup_s3_object,
+        "s3_ls": list_s3_objects,
         "flatten_dict": flatten,
         "yesterday": lambda: (datetime.datetime.now() - datetime.timedelta(1)).strftime(
             "%Y-%m-%d"