qontract-reconcile 0.10.1rc584__py3-none-any.whl → 0.10.1rc586__py3-none-any.whl

{qontract_reconcile-0.10.1rc584.dist-info → qontract_reconcile-0.10.1rc586.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc584
+Version: 0.10.1rc586
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc584.dist-info → qontract_reconcile-0.10.1rc586.dist-info}/RECORD

@@ -59,7 +59,7 @@ reconcile/ocm_github_idp.py,sha256=glwXMsIBcl38-OmDDQCpe0YoLLXfoRgVQmqwXMEXjds,3
 reconcile/ocm_groups.py,sha256=AmQ61fjJYS5PxwNEWtOAvOoJM86VfRQ0-ic6wgw6PU0,2888
 reconcile/ocm_machine_pools.py,sha256=eebJ6iiTdUcuKE5zBcfNxW1OGmPOvgBtmVu3xNVOoyY,16608
 reconcile/ocm_update_recommended_version.py,sha256=IYkfLXIprOW1jguZeELcGP1iBPuj-b53R-FTqKulMl8,4204
-reconcile/ocm_upgrade_scheduler_org_updater.py,sha256=ta8hMJ-su5mRcPpYrvB1COsojXV-SU3PzLPbQhy2Q0I,4190
+reconcile/ocm_upgrade_scheduler_org_updater.py,sha256=49Ss6sp9_n5F9914gXb-uEap4Vm2t-KPTJRFFViJMIo,4184
 reconcile/openshift_base.py,sha256=7aifvl-ay5wpY6encbUX9pGbKdjiwJmevZ3XWGRzpCM,49696
 reconcile/openshift_cluster_bots.py,sha256=eRPYZqWMKFNxLlSN0QG97V5t1iIESQ0BbGaiaQP5VB0,10940
 reconcile/openshift_clusterrolebindings.py,sha256=QfSy1Ik8eEY5XObc1Q4xyhqyErZenJmbPv_u9wcDNNo,5864
@@ -70,7 +70,7 @@ reconcile/openshift_namespaces.py,sha256=DboMc6t0vXD54lL9ZP9P9fQnCRo2g_0z5FWubtW
 reconcile/openshift_network_policies.py,sha256=_qqv7yj17OM1J8KJPsFmzFZ85gzESJeBocC672z4_WU,4231
 reconcile/openshift_resourcequotas.py,sha256=yUi56PiOn3inMMfq_x_FEHmaW-reGipzoorjdar372g,2415
 reconcile/openshift_resources.py,sha256=kwsY5cko7udEKNlhL2oKiKv_5wzEw9wmmwROE016ng8,1400
-reconcile/openshift_resources_base.py,sha256=1EAgMvGIpp_O4-PnJWgOI8UxphIsQ18kYaNRORdMf8s,46998
+reconcile/openshift_resources_base.py,sha256=3BnifJYoq7hQvrXjtuBppg36uR_tjm1kF2Q80-Pcbac,39349
 reconcile/openshift_rolebindings.py,sha256=LlImloBisEqzc36jaatic-TeM3hzqMEfxogF-dM4Yhw,6599
 reconcile/openshift_routes.py,sha256=fXvuPSjcjVw1X3j2EQvUAdbOepmIFdKk-M3qP8QzPiw,1075
 reconcile/openshift_saas_deploy.py,sha256=fmhopPEbyZsGQHRPzyzpKEvoBXEGN3aPxFi7Utq0emU,12788
@@ -137,10 +137,10 @@ reconcile/aws_saml_idp/integration.py,sha256=uqec-EnxnfGOgQtg33S-Q1wTCv0sVBHNo02
 reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_saml_roles/integration.py,sha256=kgAUaCfKA2ujvIym8sPSWoBf1EnRKi2vVbgKyi8eELU,5695
 reconcile/aws_version_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aws_version_sync/integration.py,sha256=dN_lZIeM5i0p6-yHYadGg65QgDs0nVWYJkPzU-KGwuo,15196
+reconcile/aws_version_sync/integration.py,sha256=0rgFLL2usTnngjIgFXIEfVXapFsDI1A493Mftqjfbk0,17292
 reconcile/aws_version_sync/utils.py,sha256=sVv-48PKi2VITlqqvmpbjnFDOPeGqfKzgkpIszlmjL0,1708
 reconcile/aws_version_sync/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aws_version_sync/merge_request_manager/merge_request.py,sha256=FeNcQaory5AXNVuVk-jJxPwtI4uSoURgkTH3rXAb2cc,6198
+reconcile/aws_version_sync/merge_request_manager/merge_request.py,sha256=lk8oiclpVcFbTCNSKHfDQ9H1sPha4e-39SumMItWUw4,6263
 reconcile/aws_version_sync/merge_request_manager/merge_request_manager.py,sha256=6gYFzpdJ-KcftPtOP3nhe3F9N184-UGfo_f8MQrzGxg,8322
 reconcile/change_owners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/change_owners/approver.py,sha256=GV8nwS-YJOJ8O-b9v3u60RSYECYH2EKAycjpoW6VmvU,2228
@@ -346,7 +346,7 @@ reconcile/oum/models.py,sha256=0ZyCnULRxAbIEXX60BkkPZVg53DCD6ZJ6wnNT2ANROM,1743
 reconcile/oum/providers.py,sha256=3kEjXvsTPzXc7gzrdO7hWqgzcMmMZMpk2S0X7wQUTWU,1767
 reconcile/oum/standalone.py,sha256=bzyV8wz3SrERG9zJRFiJCBzSIGwDNj9sNqUytngDw94,7368
 reconcile/prometheus_rules_tester/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/prometheus_rules_tester/integration.py,sha256=OBEVXqixTjnzi36VpPFR3rEEIDtPcSY0bopxd3M1vz8,9161
+reconcile/prometheus_rules_tester/integration.py,sha256=hCJJ0udrvgMM78_gFUGvZWOH8azI2tABQKpQq6lHhY0,9232
 reconcile/rhidp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/rhidp/common.py,sha256=suTh9T4dPOgrKi-rDALgjzCSL9JHGnkTYALFIIjNCJE,6801
 reconcile/rhidp/metrics.py,sha256=Yp0GtpjhieEdru0qkG3osBTJiKUzg6CAjwPoFTQDnCg,417
@@ -427,13 +427,13 @@ reconcile/test/test_ocm_clusters.py,sha256=NOxhJnlgOlRZF0-sfXCiRP0aiNiBiMl35MTCC
 reconcile/test/test_ocm_clusters_manifest_updates.py,sha256=jFRVfc5jby1kI2x_gT6wcqPPgkav1et9wZH6JqQbNSY,3278
 reconcile/test/test_ocm_machine_pools.py,sha256=3qo6t2Jfr1Wee0NUacyLTDmatp0o7CUNpkVOpHiOiGk,29737
 reconcile/test/test_ocm_update_recommended_version.py,sha256=iA4BVirTGVXlwcOyeR52IuNO81X_8NR6ZNd7ZFE7igs,4328
-reconcile/test/test_ocm_upgrade_scheduler_org_updater.py,sha256=zYRGUX7pAmxSv9oFYw2ZnPGa-YAPgDfmqXOJM4eE-8A,4353
+reconcile/test/test_ocm_upgrade_scheduler_org_updater.py,sha256=hT6sbdGUx8LGnMVvNI7wOVHcoan1YurckytlvtJdzGk,4347
 reconcile/test/test_openshift_base.py,sha256=uVsnMghAQhHaJTreeOw4x2INTKJ6qeiZiiteWeKflW8,33874
 reconcile/test/test_openshift_cluster_bots.py,sha256=L-yuKvMgB0LBCdfLu7wozh_lk6S_m3umXt3m_ECfLEI,8023
 reconcile/test/test_openshift_namespace_labels.py,sha256=P1hqi6P88NijNrurdXG_QR2usyo3EYZSy9zpwYHvDsM,12104
 reconcile/test/test_openshift_namespaces.py,sha256=HmRnCE5EnFt3MYceVEFHmk8wWRtCrxu2AFGFkY9pdyA,9214
 reconcile/test/test_openshift_resource.py,sha256=lbTf48jX1q6rGnRiA5pPvfU0uPfY8zhNylMtryn0sLI,12995
-reconcile/test/test_openshift_resources_base.py,sha256=4UucdsD0nCMFT1WmgNXf4r7ZZ11cJ_MP13IcK7_Vs0g,15042
+reconcile/test/test_openshift_resources_base.py,sha256=LtlR9x3o7KkSEw0JN0fZhinFeAAxBAQlB_9PpBnKwOM,14353
 reconcile/test/test_openshift_saas_deploy.py,sha256=YLJGkc--u5aP0UkQ-b9ZGEFGS2gw25jjcSgknQdI3Ic,5892
 reconcile/test/test_openshift_saas_deploy_change_tester.py,sha256=1yVe54Hx9YdVjn6qdnKge5Sa_s732c-8uZqCnuT1gGI,12871
 reconcile/test/test_openshift_tekton_resources.py,sha256=RtRWsdm51S13OSkENC9nY_rOH0QELSCaO5tjF0XqIDI,11222
@@ -465,6 +465,7 @@ reconcile/test/test_terraform_vpc_peerings.py,sha256=ubcsKh0TrUIwuI1-W3ETIgzsFvz
 reconcile/test/test_terraform_vpc_peerings_build_desired_state.py,sha256=DAfpb12I0PlqnuVUHK2vh4LH4d1OylT3H2GE_3TGZZI,47852
 reconcile/test/test_three_way_diff_strategy.py,sha256=2fjEqE2w4pIzKq18PRcADTSe01aGwsZfMGloU8xfNaE,3346
 reconcile/test/test_unleash.py,sha256=c1s_FRAZrAzzd3FbZrzHYjJzHELhoxPHBZnEzqsfMQg,6416
+reconcile/test/test_utils_jinja2.py,sha256=TpzQlpFnLGzNEZp5WOh0o7AuBiGEktqO4MuwiiJW2YY,3895
 reconcile/test/test_vault_replication.py,sha256=wlc4jm9f8P641UvvxIFFFc5_unJysNkOVrKJscjhQr0,16867
 reconcile/test/test_vault_utils.py,sha256=vbJnc89XAuE07qbTuWxHM5o9F6R9SO5aHXA38fwxT7A,1122
 reconcile/test/test_version_bump.py,sha256=q6-3Y1roriI6YWpFwaHOMN7emEP3yL33sh_0VdbmG7E,511
@@ -549,7 +550,7 @@ reconcile/utils/environ.py,sha256=VnW3zp6Un_UJn5BU4FU8RfhuqtZp0s-VeuuHnqC_WcQ,51
 reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4,635
 reconcile/utils/expiration.py,sha256=BXwKE50sNIV-Lszke97fxitNkLxYszoOLW1LBgp_yqg,1246
 reconcile/utils/extended_early_exit.py,sha256=2qvw9W2PW0t4JCZFD8wD3BINWIy02lkvpOrd2YDaYHE,6195
-reconcile/utils/external_resource_spec.py,sha256=OGPKH3IKXgJszRTgE5U_QKgU-s4BHQnx97Lj-Krz46k,6655
+reconcile/utils/external_resource_spec.py,sha256=IRY8MCsyWKzt-Qj_hXiFKgkCvZu6VVyj6IYtqEb05BA,6618
 reconcile/utils/external_resources.py,sha256=a2CkJ3KLociYBnc_9F2VWfZGWMhzDl6fDNhwo2U-MWU,7501
 reconcile/utils/filtering.py,sha256=zZnHH0u0SaTDyzuFXZ_mREURGLvjEqQIQy4z-7QBVlc,419
 reconcile/utils/git.py,sha256=Qad7mfPuS9s7eKODeWSewehwSGgJPCbQuLda1qg_6GA,1522
@@ -564,7 +565,6 @@ reconcile/utils/helpers.py,sha256=k9svgFFZG7H5FvHYY0g5jJyvgvh2UDZxf0Ib221teag,11
 reconcile/utils/imap_client.py,sha256=byFAJATbITJPsGECSbvXBOcCnoeTUpDFiEjzOAxLm_U,1975
 reconcile/utils/instrumented_wrappers.py,sha256=eVwMoa6FCrYxLv3RML3WpZF9qKVfCTjMxphgVXG03OM,1073
 reconcile/utils/jenkins_api.py,sha256=MyJSB_S3uYf3sXnt9t03-gZNQ7tbdd7Wusv3MoF2fRc,7113
-reconcile/utils/jinja2_ext.py,sha256=l628RR9r9dAGBWLVegoCbSqnjojeizNGiq9Cstt02nE,1129
 reconcile/utils/jira_client.py,sha256=f7u3xvI4tk27LOnlZxxG1OjELC5vC6wzlyk3Fo7tDnY,7007
 reconcile/utils/jjb_client.py,sha256=Pdy0dLCFvD6GPCaC0tZydYgkVJPOxYXIiwWECZaFJBU,14551
 reconcile/utils/jsonpath.py,sha256=NRpAEijKN4cMDjo7qivNPqpm0__GQQ1TiE0PBEBO45s,5572
@@ -601,7 +601,7 @@ reconcile/utils/state.py,sha256=SAa6QLHu9lr0yqLCBy2AypNx1IPCJWlrRBrvlzAKsOU,1450
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
 reconcile/utils/terraform_client.py,sha256=_jBriLBwU005bDxWlq7CRByOkVCfiH47oBzB0ArNAY8,31901
-reconcile/utils/terrascript_aws_client.py,sha256=ZV5FK0E1vdUnjsaMXnfIoE7WHU0VocKFzHL_32IDGcE,269694
+reconcile/utils/terrascript_aws_client.py,sha256=IQxaDYujdfD-AVH-_RtqklDV-RGRZSjWAZFIkP_6NFI,269716
 reconcile/utils/three_way_diff_strategy.py,sha256=nyqeQsLCoPI6e16k2CF3b9KNgQLU-rPf5RtfdUfVMwE,4468
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/unleash.py,sha256=1D56CsZfE3ShDtN3IErE1T2eeIwNmxhK-yYbCotJ99E,3601
@@ -620,6 +620,10 @@ reconcile/utils/glitchtip/models.py,sha256=_oqZXNkyRTsAnx6tF4WUURSBj0cc9UNS4okOQ
 reconcile/utils/internal_groups/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/internal_groups/client.py,sha256=abREA8RwXKybXFjCK8CAcCr-iUp2r0tAbIEJ-c-PXws,4538
 reconcile/utils/internal_groups/models.py,sha256=jlkH_hyyyuwS0J1IpuS7W1AyQSKQ2QpHelXoH36edbE,2316
+reconcile/utils/jinja2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/utils/jinja2/extensions.py,sha256=zV_x8MhSHAynKhFnG3fULXrwsm5fUG_88IygZHSnN0o,1284
+reconcile/utils/jinja2/filters.py,sha256=_kJjdMsY3lGS5PUn4NnpXUQDNrL1IwiKsB-0MhTMGYM,4521
+reconcile/utils/jinja2/utils.py,sha256=NW2BLizE-SGudgtdKqlo02CZlNda1W-swVp6uldKtUs,5779
 reconcile/utils/membershipsources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/membershipsources/app_interface_resolver.py,sha256=IlDiRtJZ0AfAGKEawybB6SvsKbm1POTXL6fpEt699E0,1979
 reconcile/utils/membershipsources/models.py,sha256=IFu6KHFe-HUTJPiAO3fEw7i22yv4_ytgBW-h_wrO6V4,2015
@@ -660,7 +664,7 @@ reconcile/utils/runtime/sharding.py,sha256=roCdbnBklhTK_g34zbgQYqzpKPaNQ8J6Xd9XL
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=XXY35h8VWQ66z3LBPxaoUAMkIW50264DQiecrzyV6oA,9076
 reconcile/utils/saasherder/models.py,sha256=PBv8DuAb6KUw_ayn5Ufiya20cCAelBv6Iv--x7hbpa4,5449
-reconcile/utils/saasherder/saasherder.py,sha256=vZ0S_7cT2FP1al2zOwLNuOedx8M7MIvokXiCsdm-5W4,85739
+reconcile/utils/saasherder/saasherder.py,sha256=72b0u-cIHg62R2uQCqlGcQfW5TbCxWDKf0dfmgVMUbY,85733
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=py-Ree-QUYD6Hvng6bM40VgSuttteehIKNgwOSoJO1o,4706
@@ -688,8 +692,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=EzJwaPxQw5CLPahLjh91oRT0pi1WCgOXZ_KgiNXZMAw,2948
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc584.dist-info/METADATA,sha256=v-BDoqNXZ9Fyh_WtSl7fukxxlwwGYE32f4hkG5CT_ag,2349
-qontract_reconcile-0.10.1rc584.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qontract_reconcile-0.10.1rc584.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
-qontract_reconcile-0.10.1rc584.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc584.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc586.dist-info/METADATA,sha256=i0GRb6YD41fyHubkOLfO3pHuPskCUvJP2_T3dpYlHlk,2349
+qontract_reconcile-0.10.1rc586.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qontract_reconcile-0.10.1rc586.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
+qontract_reconcile-0.10.1rc586.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc586.dist-info/RECORD,,

reconcile/aws_version_sync/integration.py

@@ -3,11 +3,13 @@ from collections.abc import (
     Callable,
     Iterable,
 )
+from enum import Enum
 from typing import Any
 
 import semver
 from pydantic import (
     BaseModel,
+    root_validator,
     validator,
 )
 
@@ -66,6 +68,12 @@ class ExternalResourceProvisioner(BaseModel):
     path: str | None = None
 
 
+class VersionFormat(str, Enum):
+    MAJOR = "major"
+    MAJOR_MINOR = "major_minor"
+    MAJOR_MINOR_PATCH = "major_minor_patch"
+
+
 class ExternalResource(BaseModel):
     namespace_file: str | None = None
     provider: str = "aws"
@@ -74,6 +82,8 @@ class ExternalResource(BaseModel):
     resource_identifier: str
     resource_engine: str
     resource_engine_version: semver.VersionInfo
+    # if None, it'll be set via root_validator
+    resource_engine_version_format: VersionFormat | None = None
     # used to map AWS cache name to resource_identifier
     redis_replication_group_id: str | None = None
 
@@ -98,6 +108,46 @@ class ExternalResource(BaseModel):
             return v
         return parse_semver(v, optional_minor_and_patch=True)
 
+    @root_validator(pre=True)
+    def set_resource_engine_version_format(cls, values: dict) -> dict:
+        resource_engine_version, resource_engine_version_format = (
+            values.get("resource_engine_version"),
+            values.get("resource_engine_version_format"),
+        )
+        if not resource_engine_version:
+            # make mypy happy
+            raise ValueError("resource_engine_version is required")
+
+        if resource_engine_version_format is None:
+            match resource_engine_version.count("."):
+                case 0:
+                    values["resource_engine_version_format"] = VersionFormat.MAJOR
+                case 1:
+                    values["resource_engine_version_format"] = VersionFormat.MAJOR_MINOR
+                case 2:
+                    values["resource_engine_version_format"] = (
+                        VersionFormat.MAJOR_MINOR_PATCH
+                    )
+                case _:
+                    raise ValueError(
+                        f"Invalid version format: {resource_engine_version}"
+                    )
+        return values
+
+    @property
+    def resource_engine_version_string(self) -> str:
+        match self.resource_engine_version_format:
+            case VersionFormat.MAJOR:
+                return f"{self.resource_engine_version.major}"
+            case VersionFormat.MAJOR_MINOR:
+                return f"{self.resource_engine_version.major}.{self.resource_engine_version.minor}"
+            case VersionFormat.MAJOR_MINOR_PATCH:
+                return f"{self.resource_engine_version.major}.{self.resource_engine_version.minor}.{self.resource_engine_version.patch}"
+            case _:
+                raise ValueError(
+                    f"Invalid version format: {self.resource_engine_version_format}"
+                )
+
 
 AwsExternalResources = list[ExternalResource]
 AppInterfaceExternalResources = list[ExternalResource]
@@ -319,7 +369,7 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
                 resource_provider=app_interface_resource.resource_provider,
                 resource_identifier=app_interface_resource.resource_identifier,
                 resource_engine=app_interface_resource.resource_engine,
-                resource_engine_version=str(aws_resource.resource_engine_version),
+                resource_engine_version=aws_resource.resource_engine_version_string,
             )
 
     @defer

reconcile/aws_version_sync/merge_request_manager/merge_request.py

@@ -162,7 +162,8 @@ class Renderer:
             resource_identifier,
         )
         overrides = resource.setdefault("overrides", {})
-        overrides["engine_version"] = resource_engine_version
+        # ensure that the engine version is always a string
+        overrides["engine_version"] = f"{resource_engine_version}"
         with StringIO() as stream:
             yml.dump(content, stream)
             return stream.getvalue()

reconcile/ocm_upgrade_scheduler_org_updater.py

@@ -12,7 +12,7 @@ from reconcile import (
     mr_client_gateway,
     queries,
 )
-from reconcile.openshift_resources_base import process_jinja2_template
+from reconcile.utils.jinja2.utils import process_jinja2_template
 from reconcile.utils.ocm import (
     OCMMap,
     OCMSpec,

reconcile/openshift_resources_base.py

@@ -13,7 +13,6 @@ from collections.abc import (
 )
 from contextlib import contextmanager
 from dataclasses import dataclass
-from functools import cache
 from textwrap import indent
 from threading import Lock
 from typing import (
@@ -22,14 +21,10 @@ from typing import (
     Protocol,
     Tuple,
 )
-from urllib import parse
 
 import anymarkup
-import jinja2
-import jinja2.sandbox
 from deepdiff import DeepHash
 from sretoolbox.utils import (
-    retry,
     threaded,
 )
 
@@ -37,7 +32,6 @@ import reconcile.openshift_base as ob
 from reconcile import queries
 from reconcile.change_owners.diff import IDENTIFIER_FIELD_NAME
 from reconcile.checkpoint import url_makes_sense
-from reconcile.github_users import init_github
 from reconcile.utils import (
     amtool,
     gql,
@@ -45,9 +39,12 @@ from reconcile.utils import (
 )
 from reconcile.utils.defer import defer
 from reconcile.utils.exceptions import FetchResourceError
-from reconcile.utils.jinja2_ext import (
-    B64EncodeExtension,
-    RaiseErrorExtension,
+from reconcile.utils.jinja2.utils import (
+    FetchSecretError,
+    lookup_github_file_content,
+    lookup_secret,
+    process_extracurlyjinja2_template,
+    process_jinja2_template,
 )
 from reconcile.utils.oc import (
     OC_Map,
@@ -64,7 +61,7 @@ from reconcile.utils.openshift_resource import (
 )
 from reconcile.utils.openshift_resource import OpenshiftResource as OR
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
-from reconcile.utils.secret_reader import SecretNotFound, SecretReader
+from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.sharding import is_in_shard
 from reconcile.utils.vault import (
@@ -254,21 +251,11 @@ def _locked_error_log(msg: str):
         logging.error(msg)
 
 
-class FetchSecretError(Exception):
-    def __init__(self, msg):
-        super().__init__("error fetching secret: " + str(msg))
-
-
 class FetchRouteError(Exception):
     def __init__(self, msg):
         super().__init__("error fetching route: " + str(msg))
 
 
-class Jinja2TemplateError(Exception):
-    def __init__(self, msg):
-        super().__init__("error processing jinja2 template: " + str(msg))
-
-
 class ResourceTemplateRenderError(Exception):
     pass
 
@@ -287,230 +274,6 @@ class UnknownTemplateTypeError(Exception):
         super().__init__("unknown template type error: " + str(msg))
 
 
-@retry()
-def lookup_secret(
-    path,
-    key,
-    version=None,
-    tvars=None,
-    allow_not_found=False,
-    settings=None,
-    secret_reader=None,
-):
-    if tvars is not None:
-        path = process_jinja2_template(
-            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
-        )
-        key = process_jinja2_template(
-            body=key, vars=tvars, settings=settings, secret_reader=secret_reader
-        )
-        if version and not isinstance(version, int):
-            version = process_jinja2_template(
-                body=version, vars=tvars, settings=settings, secret_reader=secret_reader
-            )
-    secret = {"path": path, "field": key, "version": version}
-    try:
-        if not secret_reader:
-            secret_reader = SecretReader(settings)
-        return secret_reader.read(secret)
-    except SecretNotFound as e:
-        if allow_not_found:
-            return None
-        raise FetchSecretError(e)
-    except Exception as e:
-        raise FetchSecretError(e)
-
-
-def lookup_github_file_content(
-    repo, path, ref, tvars=None, settings=None, secret_reader=None
-):
-    if tvars is not None:
-        repo = process_jinja2_template(
-            body=repo, vars=tvars, settings=settings, secret_reader=secret_reader
-        )
-        path = process_jinja2_template(
-            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
-        )
-        ref = process_jinja2_template(
-            body=ref, vars=tvars, settings=settings, secret_reader=secret_reader
-        )
-
-    gh = init_github()
-    c = gh.get_repo(repo).get_contents(path, ref).decoded_content
-    return c.decode("utf-8")
-
-
-def lookup_graphql_query_results(query: str, **kwargs) -> list[Any]:
-    gqlapi = gql.get_api()
-    resource = gqlapi.get_resource(query)["content"]
-    rendered_resource = jinja2.Template(resource).render(**kwargs)
-    results = list(gqlapi.query(rendered_resource).values())[0]
-    return results
-
-
-def hash_list(input: Iterable) -> str:
-    """
-    Deterministic hash of a list for jinja2 templates.
-    The order of the list doesn't matter as it is sorted
-    before hashing. Note, that the list elements
-    must be flat primitives (no dicts/lists).
-    """
-    lst = list(input)
-    str_lst = []
-    for el in lst:
-        if isinstance(el, (list, dict)):
-            raise RuntimeError(
-                f"jinja2 hash_list function received non-primitive value {el}. All values received {lst}"
-            )
-        str_lst.append(str(el))
-    msg = "a"  # keep non-empty for hashing empty list
-    msg += "".join(sorted(str_lst))
-    m = hashlib.sha256()
-    m.update(msg.encode("utf-8"))
-    return m.hexdigest()
-
-
-def eval_filter(input, **kwargs) -> str:
-    """Jinja2 filter be used when the string
-    is in itself a jinja2 template that must be
-    evaluated with kwargs. For example in the case
-    of the slo-document expression fields.
-    :param input: template string
-    :kwargs: variables that will be used to evaluate the
-             input string
-    :return: rendered string
-    """
-    return jinja2.Template(input).render(**kwargs)
-
-
-def json_to_dict(input):
-    """Jinja2 filter to parse JSON strings into dictionaries.
-       This becomes useful to access Graphql queries data (labels)
-    :param input: json string
-    :return: dict with the parsed inputs contents
-    """
-    data = json.loads(input)
-    return data
-
-
-def urlescape(
-    string: str,
-    safe: str = "/",
-    encoding: Optional[str] = None,
-) -> str:
-    """Jinja2 filter that is a simple wrapper around urllib's URL quoting
-    functions that takes a string value and makes it safe for use as URL
-    components escaping any reserved characters using URL encoding. See:
-    urllib.parse.quote() and urllib.parse.quote_plus() for reference.
-
-    :param str string: String value to escape.
-    :param str safe: Optional characters that should not be escaped.
-    :param encoding: Encoding to apply to the string to be escaped. Defaults
-        to UTF-8. Unsupported characters raise a UnicodeEncodeError error.
-    :type encoding: typing.Optional[str]
-    :returns: A string with reserved characters escaped.
-    :rtype: str
-    """
-    return parse.quote(string, safe=safe, encoding=encoding)
-
-
-def urlunescape(string: str, encoding: Optional[str] = None) -> str:
-    """Jinja2 filter that is a simple wrapper around urllib's URL unquoting
-    functions that takes an URL-encoded string value and unescapes it
-    replacing any URL-encoded values with their character equivalent. See:
-    urllib.parse.unquote() and urllib.parse.unquote_plus() for reference.
-
-    :param str string: String value to unescape.
-    :param encoding: Encoding to apply to the string to be unescaped. Defaults
-        to UTF-8. Unsupported characters are replaced by placeholder values.
-    :type encoding: typing.Optional[str]
-    :returns: A string with URL-encoded sequences unescaped.
-    :rtype: str
-    """
-    if encoding is None:
-        encoding = "utf-8"
-    return parse.unquote(string, encoding=encoding)
-
-
-@cache
-def compile_jinja2_template(body, extra_curly: bool = False):
-    env: dict = {}
-    if extra_curly:
-        env = {
-            "block_start_string": "{{%",
-            "block_end_string": "%}}",
-            "variable_start_string": "{{{",
-            "variable_end_string": "}}}",
-            "comment_start_string": "{{#",
-            "comment_end_string": "#}}",
-        }
-
-    jinja_env = jinja2.sandbox.SandboxedEnvironment(
-        extensions=[B64EncodeExtension, RaiseErrorExtension],
-        undefined=jinja2.StrictUndefined,
-        **env,
-    )
-    jinja_env.filters.update({
-        "json_to_dict": json_to_dict,
-        "urlescape": urlescape,
-        "urlunescape": urlunescape,
-        "eval": eval_filter,
-    })
-
-    return jinja_env.from_string(body)
-
-
-def process_jinja2_template(
-    body, vars=None, extra_curly: bool = False, settings=None, secret_reader=None
-):
-    if vars is None:
-        vars = {}
-    vars.update({
-        "vault": lambda p, k, v=None, allow_not_found=False: lookup_secret(
-            path=p,
-            key=k,
-            version=v,
-            tvars=vars,
-            allow_not_found=allow_not_found,
-            settings=settings,
-            secret_reader=secret_reader,
-        ),
-        "github": lambda u, p, r, v=None: lookup_github_file_content(
-            repo=u,
-            path=p,
-            ref=r,
-            tvars=vars,
-            settings=settings,
-            secret_reader=secret_reader,
-        ),
-        "urlescape": lambda u, s="/", e=None: urlescape(string=u, safe=s, encoding=e),
-        "urlunescape": lambda u, e=None: urlunescape(string=u, encoding=e),
-        "hash_list": hash_list,
-        "query": lookup_graphql_query_results,
-        "url": url_makes_sense,
-    })
-    try:
-        template = compile_jinja2_template(body, extra_curly)
-        r = template.render(vars)
-    except Exception as e:
-        raise Jinja2TemplateError(e)
-    return r
-
-
-def process_extracurlyjinja2_template(
-    body, vars=None, env=None, settings=None, secret_reader=None
-):
-    if vars is None:
-        vars = {}
-    return process_jinja2_template(
-        body,
-        vars=vars,
-        extra_curly=True,
-        settings=settings,
-        secret_reader=secret_reader,
-    )
-
-
 def check_alertmanager_config(data, path, alertmanager_config_key, decode_base64=False):
     try:
         config = data[alertmanager_config_key]

reconcile/prometheus_rules_tester/integration.py

@@ -29,6 +29,7 @@ from reconcile.utils import (
     gql,
     promtool,
 )
+from reconcile.utils.jinja2.utils import process_extracurlyjinja2_template
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.structs import CommandExecutionResult
@@ -91,7 +92,7 @@ def fetch_rule_and_tests(
         test_raw_yaml = gql.get_resource(test_path)["content"]
 
         if rule.resource["type"] == "resource-template-extracurlyjinja2":
-            test_raw_yaml = orb.process_extracurlyjinja2_template(
+            test_raw_yaml = process_extracurlyjinja2_template(
                 body=test_raw_yaml,
                 vars=variables,
                 settings=vault_settings.dict(by_alias=True),

reconcile/test/test_ocm_upgrade_scheduler_org_updater.py

@@ -3,7 +3,7 @@ import json
 import pytest
 
 from reconcile.ocm_upgrade_scheduler_org_updater import render_policy
-from reconcile.openshift_resources_base import Jinja2TemplateError
+from reconcile.utils.jinja2.utils import Jinja2TemplateError
 from reconcile.utils.ocm import (
     OCMClusterNetwork,
     OCMSpec,

reconcile/test/test_openshift_resources_base.py

@@ -13,7 +13,6 @@ from reconcile.openshift_base import CurrentStateSpec
 from reconcile.openshift_resources_base import (
     CheckClusterScopedResourceDuplicates,
     canonicalize_namespaces,
-    hash_list,
     ob,
 )
 from reconcile.test.fixtures import Fixtures
@@ -441,36 +440,6 @@ def test_check_error():
     print(e)
 
 
-def test_hash_list_empty():
-    assert hash_list([])[:6] == "ca9781"
-
-
-def test_hash_list_string():
-    assert hash_list(["a", "b"])[:6] == "38760e"
-    assert hash_list(["b", "a"])[:6] == "38760e"
-
-
-def test_hash_list_int():
-    assert hash_list([1, 2])[:6] == "f37508"
-    assert hash_list([2, 1])[:6] == "f37508"
-
-
-def test_hash_list_bool():
-    assert hash_list([True, False])[:6] == "e0ca28"
-    assert hash_list([False, True])[:6] == "e0ca28"
-
-
-def test_hash_list_error():
-    with pytest.raises(RuntimeError):
-        hash_list([{}])
-
-    with pytest.raises(RuntimeError):
-        hash_list([[]])
-
-    with pytest.raises(RuntimeError):
-        hash_list(["a", {}])
-
-
 def test_cluster_params():
     with pytest.raises(RuntimeError):
         orb.run(dry_run=False, exclude_cluster=["test-cluster"])

reconcile/test/test_utils_jinja2.py

@@ -0,0 +1,123 @@
+import pytest
+from jsonpath_ng.exceptions import JsonPathParserError
+
+from reconcile.utils.jinja2.filters import (
+    extract_jsonpath,
+    hash_list,
+    json_pointers,
+    matches_jsonpath,
+)
+
+
+def test_hash_list_empty() -> None:
+    assert hash_list([])[:6] == "ca9781"
+
+
+def test_hash_list_string() -> None:
+    assert hash_list(["a", "b"])[:6] == "38760e"
+    assert hash_list(["b", "a"])[:6] == "38760e"
+
+
+def test_hash_list_int() -> None:
+    assert hash_list([1, 2])[:6] == "f37508"
+    assert hash_list([2, 1])[:6] == "f37508"
+
+
+def test_hash_list_bool() -> None:
+    assert hash_list([True, False])[:6] == "e0ca28"
+    assert hash_list([False, True])[:6] == "e0ca28"
+
+
+def test_hash_list_error() -> None:
+    with pytest.raises(RuntimeError):
+        hash_list([{}])
+
+    with pytest.raises(RuntimeError):
+        hash_list([[]])
+
+    with pytest.raises(RuntimeError):
+        hash_list(["a", {}])
+
+
+def test_extract_jsonpath_dict_basic() -> None:
+    input = {"a": "A", "b": {"b1": "B1", "b2": ["B", 2]}}
+    assert extract_jsonpath(input, "a") == ["A"]
+    assert extract_jsonpath(input, "b.b1") == ["B1"]
+    assert extract_jsonpath(input, "b.b2") == [["B", 2]]
+    assert extract_jsonpath(input, "b.b2[0]") == ["B"]
+    assert extract_jsonpath(input, "c") == []
+
+
+def test_extract_jsonpath_dict_multiple() -> None:
+    input = {
+        "items": [
+            {"name": "a", "value": "A"},
+            {"name": "b", "value": "B1"},
+            {"name": "b", "value": "B2"},
+        ]
+    }
+    assert extract_jsonpath(input, "items[0]") == [{"name": "a", "value": "A"}]
+    assert extract_jsonpath(input, "items[?(@.name=='a')]") == [
+        {"name": "a", "value": "A"}
+    ]
+    assert extract_jsonpath(input, "items[?(@.name=='a')].value") == ["A"]
+    assert extract_jsonpath(input, "items[?(@.name=='b')]") == [
+        {"name": "b", "value": "B1"},
+        {"name": "b", "value": "B2"},
+    ]
+    assert extract_jsonpath(input, "items[?(@.name=='b')].value") == ["B1", "B2"]
+    assert extract_jsonpath(input, "items[?(@.name=='c')].value") == []
+
+
+def test_extract_jsonpath_list() -> None:
+    input = ["a", "b"]
+    assert extract_jsonpath(input, "[0]") == ["a"]
+
+
+def test_extract_jsonpath_str() -> None:
+    assert extract_jsonpath("a", "[0]") == ["a"]
+    assert extract_jsonpath("a", "something") == []
+
+
+def test_extract_jsonpath_none() -> None:
+    assert extract_jsonpath(None, "a") == []
+
+
+def test_extract_jsonpath_errors() -> None:
+    input = {"a": "A", "b": "B"}
+    with pytest.raises(JsonPathParserError):
+        extract_jsonpath(input, "THIS IS AN INVALID JSONPATH]")
+    with pytest.raises(AssertionError):
+        extract_jsonpath("a", "")
+
+
+def test_matches_jsonpath() -> None:
+    input = {"a": "A", "b": "B"}
+    assert matches_jsonpath(input, "a")
+    assert not matches_jsonpath(input, "c")
+    with pytest.raises(JsonPathParserError):
+        matches_jsonpath(input, "THIS IS AN INVALID JSONPATH]")
+    with pytest.raises(AssertionError):
+        matches_jsonpath("a", "")
+    with pytest.raises(AssertionError):
+        matches_jsonpath("a", None)
+
+
+def test_json_pointers() -> None:
+    input = {
+        "items": [
+            {"name": "a", "value": "A"},
+            {"name": "b", "value": "B1"},
+            {"name": "b", "value": "B2"},
+        ]
+    }
+    assert json_pointers(input, "items") == ["/items"]
+    assert json_pointers(input, "items[*]") == ["/items/0", "/items/1", "/items/2"]
+    assert json_pointers(input, "items[0]") == ["/items/0"]
+    assert json_pointers(input, "items[0].name") == ["/items/0/name"]
+    assert json_pointers(input, "items[4]") == []
+    assert json_pointers(input, "items[4].name") == []
+
+    assert json_pointers(input, "items[?@.name=='a']") == ["/items/0"]
+    assert json_pointers(input, "items[?@.name=='b']") == ["/items/1", "/items/2"]
+    assert json_pointers(input, "items[?@.name=='c']") == []

reconcile/utils/external_resource_spec.py

@@ -15,7 +15,7 @@ import yaml
 from pydantic import BaseModel
 from pydantic.dataclasses import dataclass
 
-from reconcile import openshift_resources_base
+from reconcile.utils.jinja2.utils import process_jinja2_template
 from reconcile.utils.metrics import GaugeMetric
 from reconcile.utils.openshift_resource import (
     SECRET_MAX_KEY_LENGTH,
@@ -58,9 +58,7 @@ class GenericSecretOutputFormatConfig(OutputFormatProcessor):
         if self.data:
             # the jinja2 rendering has the capabilitiy to change the passed
             # vars dict - make a copy to protect against it
-            rendered_data = openshift_resources_base.process_jinja2_template(
-                self.data, dict(vars)
-            )
+            rendered_data = process_jinja2_template(self.data, dict(vars))
             parsed_data = yaml.safe_load(rendered_data)
             self.validate_k8s_secret_data(parsed_data)
             return cast(dict[str, str], parsed_data)

reconcile/utils/{jinja2_ext.py → jinja2/extensions.py}

@@ -1,15 +1,17 @@
 import base64
 import textwrap
+from typing import Callable
 
 from jinja2 import nodes
 from jinja2.exceptions import TemplateRuntimeError
 from jinja2.ext import Extension
+from jinja2.parser import Parser
 
 
 class B64EncodeExtension(Extension):
     tags = {"b64encode"}
 
-    def parse(self, parser):
+    def parse(self, parser: Parser) -> nodes.CallBlock:
         lineno = next(parser.stream).lineno
 
         body = parser.parse_statements(["name:endb64encode"], drop_needle=True)
@@ -19,7 +21,7 @@ class B64EncodeExtension(Extension):
         ).set_lineno(lineno)
 
     @staticmethod
-    def _b64encode(caller):
+    def _b64encode(caller: Callable) -> str:
         content = caller()
         content = textwrap.dedent(content)
         return base64.b64encode(content.encode()).decode("utf-8")
@@ -28,7 +30,7 @@ class B64EncodeExtension(Extension):
 class RaiseErrorExtension(Extension):
     tags = {"raise_error"}
 
-    def parse(self, parser):
+    def parse(self, parser: Parser) -> nodes.CallBlock:
         lineno = next(parser.stream).lineno
 
         msg = parser.parse_expression()
@@ -42,5 +44,5 @@ class RaiseErrorExtension(Extension):
         )
 
     @staticmethod
-    def _raise_error(msg, caller):
+    def _raise_error(msg: str, caller: Callable) -> None:
         raise TemplateRuntimeError(msg)

reconcile/utils/jinja2/filters.py

@@ -0,0 +1,128 @@
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from typing import Any, Optional
+from urllib import parse
+
+import jinja2
+
+from reconcile.utils.jsonpath import parse_jsonpath
+
+
+def json_to_dict(input: str) -> Any:
+    """Jinja2 filter to parse JSON strings into dictionaries.
+       This becomes useful to access Graphql queries data (labels)
+    :param input: json string
+    :return: dict with the parsed inputs contents
+    """
+    data = json.loads(input)
+    return data
+
+
+def urlescape(string: str, safe: str = "/", encoding: Optional[str] = None) -> str:
+    """Jinja2 filter that is a simple wrapper around urllib's URL quoting
+    functions that takes a string value and makes it safe for use as URL
+    components escaping any reserved characters using URL encoding. See:
+    urllib.parse.quote() and urllib.parse.quote_plus() for reference.
+
+    :param str string: String value to escape.
+    :param str safe: Optional characters that should not be escaped.
+    :param encoding: Encoding to apply to the string to be escaped. Defaults
+        to UTF-8. Unsupported characters raise a UnicodeEncodeError error.
+    :type encoding: typing.Optional[str]
+    :returns: A string with reserved characters escaped.
+    :rtype: str
+    """
+    return parse.quote(string, safe=safe, encoding=encoding)
+
+
+def urlunescape(string: str, encoding: Optional[str] = None) -> str:
+    """Jinja2 filter that is a simple wrapper around urllib's URL unquoting
+    functions that takes an URL-encoded string value and unescapes it
+    replacing any URL-encoded values with their character equivalent. See:
+    urllib.parse.unquote() and urllib.parse.unquote_plus() for reference.
+
+    :param str string: String value to unescape.
+    :param encoding: Encoding to apply to the string to be unescaped. Defaults
+        to UTF-8. Unsupported characters are replaced by placeholder values.
+    :type encoding: typing.Optional[str]
+    :returns: A string with URL-encoded sequences unescaped.
+    :rtype: str
+    """
+    if encoding is None:
+        encoding = "utf-8"
+    return parse.unquote(string, encoding=encoding)
+
+
+def eval_filter(input: str, **kwargs: dict[str, Any]) -> str:
+    """Jinja2 filter be used when the string
+    is in itself a jinja2 template that must be
+    evaluated with kwargs. For example in the case
+    of the slo-document expression fields.
+    :param input: template string
+    :kwargs: variables that will be used to evaluate the
+             input string
+    :return: rendered string
+    """
+    return jinja2.Template(input).render(**kwargs)
+
+
+def hash_list(input: Iterable) -> str:
+    """
+    Deterministic hash of a list for jinja2 templates.
+    The order of the list doesn't matter as it is sorted
+    before hashing. Note, that the list elements
+    must be flat primitives (no dicts/lists).
+    """
+    lst = list(input)
+    str_lst = []
+    for el in lst:
+        if isinstance(el, (list, dict)):
+            raise RuntimeError(
+                f"jinja2 hash_list function received non-primitive value {el}. All values received {lst}"
+            )
+        str_lst.append(str(el))
+    msg = "a"  # keep non-empty for hashing empty list
+    msg += "".join(sorted(str_lst))
+    m = hashlib.sha256()
+    m.update(msg.encode("utf-8"))
+    return m.hexdigest()
+
+
+def _find_jsonpath(input: Any, jsonpath: str | None) -> Any:
+    assert jsonpath is not None and len(jsonpath) > 0
+    return parse_jsonpath(jsonpath).find(input)
+
+
+def extract_jsonpath(input: Any, jsonpath: str) -> Any:
+    """
+    Extracts data from the input using jsonpath.
+    The result is a list of matching elements.
+    The list will be empty if nothing matches.
+    """
+    return [i.value for i in _find_jsonpath(input, jsonpath)]
+
+
+def matches_jsonpath(input: Any, jsonpath: str | None) -> bool:
+    """
+    Returns True if the input matches the provided jsonpath
+    """
+    return len(_find_jsonpath(input, jsonpath)) > 0
+
+
+def _convert_pointer(pointer: str) -> str:
+    """
+    Converts a jsonpath_ng pointer (eg "items.[2].type.[3]")
+    to a rfc6901 one (https://www.rfc-editor.org/rfc/rfc6901)
+    "/items/2/type/3"
+    """
+    elems = [e[1:-1] if re.match(r"\[\d\]", e) else e for e in pointer.split(".")]
+    return "/" + "/".join(elems)
+
+
+def json_pointers(input: Any, jsonpath: str) -> list[str]:
+    """
+    Finds the RFC6901 JSON pointers of the input elements matching the given jsonpath
+    """
+    return [_convert_pointer(str(i.full_path)) for i in _find_jsonpath(input, jsonpath)]

reconcile/utils/jinja2/utils.py

@@ -0,0 +1,188 @@
+from functools import cache
+from typing import Any, Optional
+
+import jinja2
+from jinja2.sandbox import SandboxedEnvironment
+from sretoolbox.utils import retry
+
+from reconcile.checkpoint import url_makes_sense
+from reconcile.github_users import init_github
+from reconcile.utils import gql
+from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
+from reconcile.utils.jinja2.filters import (
+    eval_filter,
+    extract_jsonpath,
+    hash_list,
+    json_pointers,
+    json_to_dict,
+    matches_jsonpath,
+    urlescape,
+    urlunescape,
+)
+from reconcile.utils.secret_reader import SecretNotFound, SecretReader, SecretReaderBase
+
+
+class Jinja2TemplateError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("error processing jinja2 template: " + str(msg))
+
+
+@cache
+def compile_jinja2_template(body: str, extra_curly: bool = False) -> Any:
+    env: dict = {}
+    if extra_curly:
+        env = {
+            "block_start_string": "{{%",
+            "block_end_string": "%}}",
+            "variable_start_string": "{{{",
+            "variable_end_string": "}}}",
+            "comment_start_string": "{{#",
+            "comment_end_string": "#}}",
+        }
+
+    jinja_env = SandboxedEnvironment(
+        extensions=[B64EncodeExtension, RaiseErrorExtension],
+        undefined=jinja2.StrictUndefined,
+        **env,
+    )
+    jinja_env.filters.update({
+        "json_to_dict": json_to_dict,
+        "urlescape": urlescape,
+        "urlunescape": urlunescape,
+        "eval": eval_filter,
+        "extract_jsonpath": extract_jsonpath,
+        "matches_jsonpath": matches_jsonpath,
+        "json_pointers": json_pointers,
+    })
+
+    return jinja_env.from_string(body)
+
+
+def lookup_github_file_content(
+    repo: str,
+    path: str,
+    ref: str,
+    tvars: Optional[dict[str, Any]] = None,
+    settings: Optional[dict[str, Any]] = None,
+    secret_reader: Optional[SecretReaderBase] = None,
+) -> str:
+    if tvars is not None:
+        repo = process_jinja2_template(
+            body=repo, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        path = process_jinja2_template(
+            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        ref = process_jinja2_template(
+            body=ref, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+
+    gh = init_github()
+    c = gh.get_repo(repo).get_contents(path, ref).decoded_content
+    return c.decode("utf-8")
+
+
+def lookup_graphql_query_results(query: str, **kwargs: dict[str, Any]) -> list[Any]:
+    gqlapi = gql.get_api()
+    resource = gqlapi.get_resource(query)["content"]
+    rendered_resource = jinja2.Template(resource).render(**kwargs)
+    results = list(gqlapi.query(rendered_resource).values())[0]
+    return results
+
+
+@retry()
+def lookup_secret(
+    path: str,
+    key: str,
+    version: Optional[str] = None,
+    tvars: Optional[dict[str, Any]] = None,
+    allow_not_found: bool = False,
+    settings: Optional[dict[str, Any]] = None,
+    secret_reader: Optional[SecretReaderBase] = None,
+) -> Optional[str]:
+    if tvars is not None:
+        path = process_jinja2_template(
+            body=path, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        key = process_jinja2_template(
+            body=key, vars=tvars, settings=settings, secret_reader=secret_reader
+        )
+        if version and not isinstance(version, int):
+            version = process_jinja2_template(
+                body=version, vars=tvars, settings=settings, secret_reader=secret_reader
+            )
+    secret = {"path": path, "field": key, "version": version}
+    try:
+        if not secret_reader:
+            secret_reader = SecretReader(settings)
+        return secret_reader.read(secret)
+    except SecretNotFound as e:
+        if allow_not_found:
+            return None
+        raise FetchSecretError(e)
+    except Exception as e:
+        raise FetchSecretError(e)
+
+
+def process_jinja2_template(
+    body: str,
+    vars: Optional[dict[str, Any]] = None,
+    extra_curly: bool = False,
+    settings: Optional[dict[str, Any]] = None,
+    secret_reader: Optional[SecretReaderBase] = None,
+) -> Any:
+    if vars is None:
+        vars = {}
+    vars.update({
+        "vault": lambda p, k, v=None: lookup_secret(
+            path=p,
+            key=k,
+            version=v,
+            tvars=vars,
+            allow_not_found=False,
+            settings=settings,
+            secret_reader=secret_reader,
+        ),
+        "github": lambda u, p, r, v=None: lookup_github_file_content(
+            repo=u,
+            path=p,
+            ref=r,
+            tvars=vars,
+            settings=settings,
+            secret_reader=secret_reader,
+        ),
+        "urlescape": lambda u, s="/", e=None: urlescape(string=u, safe=s, encoding=e),
+        "urlunescape": lambda u, e=None: urlunescape(string=u, encoding=e),
+        "hash_list": hash_list,
+        "query": lookup_graphql_query_results,
+        "url": url_makes_sense,
+    })
+    try:
+        template = compile_jinja2_template(body, extra_curly)
+        r = template.render(vars)
+    except Exception as e:
+        raise Jinja2TemplateError(e)
+    return r
+
+
+def process_extracurlyjinja2_template(
+    body: str,
+    vars: Optional[dict[str, Any]] = None,
+    extra_curly: bool = True,
+    settings: Optional[dict[str, Any]] = None,
+    secret_reader: Optional[SecretReaderBase] = None,
+) -> Any:
+    if vars is None:
+        vars = {}
+    return process_jinja2_template(
+        body,
+        vars=vars,
+        extra_curly=True,
+        settings=settings,
+        secret_reader=secret_reader,
+    )
+
+
+class FetchSecretError(Exception):
+    def __init__(self, msg: Any):
+        super().__init__("error fetching secret: " + str(msg))

reconcile/utils/saasherder/saasherder.py

@@ -2031,7 +2031,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
     @staticmethod
     def resolve_templated_parameters(saas_files: Iterable[SaasFile]) -> None:
         """Resolve templated target parameters in saas files."""
-        from reconcile.openshift_resources_base import (  # noqa: PLC0415 - # avoid circular import
+        from reconcile.utils.jinja2.utils import (  # noqa: PLC0415 - # avoid circular import
             compile_jinja2_template,
         )
 

reconcile/utils/terrascript_aws_client.py

@@ -140,7 +140,6 @@ from terrascript.resource import (
     random_id,
 )
 
-import reconcile.openshift_resources_base as orb
 import reconcile.utils.aws_helper as awsh
 from reconcile import queries
 from reconcile.github_org import get_default_config
@@ -176,6 +175,7 @@ from reconcile.utils.external_resources import (
 from reconcile.utils.git import is_file_in_git_repo
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi
+from reconcile.utils.jinja2.utils import process_extracurlyjinja2_template
 from reconcile.utils.ocm import OCMMap
 from reconcile.utils.password_validator import (
     PasswordPolicy,
@@ -5342,7 +5342,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             part = []
             for c in cloudinit_configs:
                 raw = self.get_raw_values(c["content"])
-                content = orb.process_extracurlyjinja2_template(
+                content = process_extracurlyjinja2_template(
                     body=raw["content"], vars=vars, secret_reader=self.secret_reader
                 )
                 # https://www.terraform.io/docs/language/expressions/strings.html#escape-sequences