infrahub-server 1.2.9rc0__py3-none-any.whl → 1.2.11__py3-none-any.whl

infrahub/task_manager/task.py

@@ -35,6 +35,9 @@ from .models import FlowLogs, FlowProgress, RelatedNodesInfo
 
 log = get_logger()
 
+NB_LOGS_LIMIT = 10_000
+PREFECT_MAX_LOGS_PER_CALL = 200
+
 
 class PrefectTask:
     @classmethod
@@ -83,9 +86,42 @@ class PrefectTask:
         return related_nodes
 
     @classmethod
-    async def _get_logs(cls, client: PrefectClient, flow_ids: list[UUID]) -> FlowLogs:
+    async def _get_logs(
+        cls, client: PrefectClient, flow_ids: list[UUID], log_limit: int | None, log_offset: int | None
+    ) -> FlowLogs:
+        """
+        Return the logs for a flow run, based on log_limit and log_offset.
+        At most, NB_LOGS_LIMIT logs will be returned per flow.
+        """
+
         logs_flow = FlowLogs()
-        all_logs = await client.read_logs(log_filter=LogFilter(flow_run_id=LogFilterFlowRunId(any_=flow_ids)))
+
+        log_limit = log_limit if log_limit is not None else NB_LOGS_LIMIT
+        log_offset = log_offset or 0
+        current_offset = log_offset
+
+        if log_limit > NB_LOGS_LIMIT:
+            raise ValueError(f"log_limit cannot be greater than {NB_LOGS_LIMIT}")
+
+        all_logs = []
+
+        # Fetch the logs in batches of PREFECT_MAX_LOGS_PER_CALL, as prefect does not allow to fetch more logs at once.
+        remaining = min(log_limit, NB_LOGS_LIMIT)
+        while remaining > 0:
+            batch_limit = min(PREFECT_MAX_LOGS_PER_CALL, remaining)
+            logs_batch = await client.read_logs(
+                log_filter=LogFilter(flow_run_id=LogFilterFlowRunId(any_=flow_ids)),
+                offset=current_offset,
+                limit=batch_limit,
+            )
+            all_logs.extend(logs_batch)
+            nb_fetched = len(logs_batch)
+            if nb_fetched < batch_limit:
+                break  # No more logs to fetch
+
+            current_offset += nb_fetched
+            remaining -= nb_fetched
+
         for flow_log in all_logs:
             if flow_log.flow_run_id and flow_log.message not in ["Finished in state Completed()"]:
                 logs_flow.logs[flow_log.flow_run_id].append(flow_log)
@@ -188,6 +224,8 @@ class PrefectTask:
         branch: str | None = None,
         limit: int | None = None,
         offset: int | None = None,
+        log_limit: int | None = None,
+        log_offset: int | None = None,
     ) -> dict[str, Any]:
         nodes: list[dict] = []
         count = None
@@ -219,7 +257,9 @@ class PrefectTask:
                     sort=FlowRunSort.START_TIME_DESC,
                 )
                 if log_fields:
-                    logs_flow = await cls._get_logs(client=client, flow_ids=[flow.id for flow in flows])
+                    logs_flow = await cls._get_logs(
+                        client=client, flow_ids=[flow.id for flow in flows], log_limit=log_limit, log_offset=log_offset
+                    )
 
                 if "progress" in node_fields:
                     progress_flow = await cls._get_progress(client=client, flow_ids=[flow.id for flow in flows])
@@ -265,7 +305,7 @@ class PrefectTask:
                                 "updated_at": flow.updated.to_iso8601_string(),  # type: ignore
                                 "start_time": flow.start_time.to_iso8601_string() if flow.start_time else None,
                                 "id": flow.id,
-                                "logs": {"edges": logs},
+                                "logs": {"edges": logs, "count": len(logs)},
                             }
                         }
                     )

infrahub/telemetry/database.py

@@ -49,7 +49,7 @@ async def get_system_info(db: InfrahubDatabase) -> TelemetryDatabaseSystemInfoDa
 
 @task(name="telemetry-gather-db", task_run_name="Gather Database Information", cache_policy=NONE)
 async def gather_database_information(db: InfrahubDatabase) -> TelemetryDatabaseData:
-    async with db.start_session() as dbs:
+    async with db.start_session(read_only=True) as dbs:
         server_info = []
         system_info = None
         database_type = db.db_type.value

infrahub/telemetry/tasks.py

@@ -38,7 +38,7 @@ async def gather_schema_information(branch: Branch) -> TelemetrySchemaData:
 
 @task(name="telemetry-feature-information", task_run_name="Gather Feature Information", cache_policy=NONE)
 async def gather_feature_information(service: InfrahubServices) -> dict[str, int]:
-    async with service.database.start_session() as db:
+    async with service.database.start_session(read_only=True) as db:
         data = {}
         features_to_count = [
             InfrahubKind.ARTIFACT,

infrahub/trigger/models.py

@@ -5,8 +5,11 @@ from enum import Enum
 from typing import TYPE_CHECKING, Any
 
 from prefect.events.actions import RunDeployment
+from prefect.events.schemas.automations import (
+    Automation,  # noqa: TC002
+    Posture,
+)
 from prefect.events.schemas.automations import EventTrigger as PrefectEventTrigger
-from prefect.events.schemas.automations import Posture
 from prefect.events.schemas.events import ResourceSpecification
 from pydantic import BaseModel, Field
 
@@ -19,6 +22,13 @@ if TYPE_CHECKING:
     from uuid import UUID
 
 
+class TriggerSetupReport(BaseModel):
+    created: list[TriggerDefinition] = Field(default_factory=list)
+    updated: list[TriggerDefinition] = Field(default_factory=list)
+    deleted: list[Automation] = Field(default_factory=list)
+    unchanged: list[TriggerDefinition] = Field(default_factory=list)
+
+
 class TriggerType(str, Enum):
     BUILTIN = "builtin"
     WEBHOOK = "webhook"

infrahub/trigger/setup.py

@@ -5,27 +5,43 @@ from prefect.automations import AutomationCore
 from prefect.cache_policies import NONE
 from prefect.client.orchestration import PrefectClient
 from prefect.client.schemas.filters import DeploymentFilter, DeploymentFilterName
+from prefect.events.schemas.automations import Automation
 
 from infrahub.trigger.models import TriggerDefinition
 
-from .models import TriggerType
+from .models import TriggerSetupReport, TriggerType
 
 if TYPE_CHECKING:
     from uuid import UUID
 
 
+def compare_automations(target: AutomationCore, existing: Automation) -> bool:
+    """Compare an AutomationCore with an existing Automation object to identify if they are identical or not
+
+    Return True if the target is identical to the existing automation
+    """
+
+    target_dump = target.model_dump(exclude_defaults=True, exclude_none=True)
+    existing_dump = existing.model_dump(exclude_defaults=True, exclude_none=True, exclude={"id"})
+
+    return target_dump == existing_dump
+
+
 @task(name="trigger-setup", task_run_name="Setup triggers", cache_policy=NONE)  # type: ignore[arg-type]
 async def setup_triggers(
     client: PrefectClient,
     triggers: list[TriggerDefinition],
     trigger_type: TriggerType | None = None,
-) -> None:
+    force_update: bool = False,
+) -> TriggerSetupReport:
     log = get_run_logger()
 
+    report = TriggerSetupReport()
+
     if trigger_type:
-        log.info(f"Setting up triggers of type {trigger_type.value}")
+        log.debug(f"Setting up triggers of type {trigger_type.value}")
     else:
-        log.info("Setting up all triggers")
+        log.debug("Setting up all triggers")
 
     # -------------------------------------------------------------
     # Retrieve existing Deployments and Automation from the server
@@ -38,23 +54,24 @@ async def setup_triggers(
         )
     }
     deployments_mapping: dict[str, UUID] = {name: item.id for name, item in deployments.items()}
-    existing_automations = {item.name: item for item in await client.read_automations()}
 
     # If a trigger type is provided, narrow down the list of existing triggers to know which one to delete
+    existing_automations: dict[str, Automation] = {}
     if trigger_type:
-        trigger_automations = [
-            item.name for item in await client.read_automations() if item.name.startswith(trigger_type.value)
-        ]
+        existing_automations = {
+            item.name: item for item in await client.read_automations() if item.name.startswith(trigger_type.value)
+        }
     else:
-        trigger_automations = [item.name for item in await client.read_automations()]
+        existing_automations = {item.name: item for item in await client.read_automations()}
 
     trigger_names = [trigger.generate_name() for trigger in triggers]
+    automation_names = list(existing_automations.keys())
 
-    log.debug(f"{len(trigger_automations)} existing triggers ({trigger_automations})")
-    log.debug(f"{len(trigger_names)}  triggers to configure ({trigger_names})")
+    log.debug(f"{len(automation_names)} existing triggers ({automation_names})")
+    log.debug(f"{len(trigger_names)} triggers to configure ({trigger_names})")
 
-    to_delete = set(trigger_automations) - set(trigger_names)
-    log.debug(f"{len(trigger_names)} triggers to delete ({to_delete})")
+    to_delete = set(automation_names) - set(trigger_names)
+    log.debug(f"{len(to_delete)} triggers to delete ({to_delete})")
 
     # -------------------------------------------------------------
     # Create or Update all triggers
@@ -71,11 +88,16 @@ async def setup_triggers(
         existing_automation = existing_automations.get(trigger.generate_name(), None)
 
         if existing_automation:
-            await client.update_automation(automation_id=existing_automation.id, automation=automation)
-            log.info(f"{trigger.generate_name()} Updated")
+            if force_update or not compare_automations(target=automation, existing=existing_automation):
+                await client.update_automation(automation_id=existing_automation.id, automation=automation)
+                log.info(f"{trigger.generate_name()} Updated")
+                report.updated.append(trigger)
+            else:
+                report.unchanged.append(trigger)
         else:
             await client.create_automation(automation=automation)
             log.info(f"{trigger.generate_name()} Created")
+            report.created.append(trigger)
 
     # -------------------------------------------------------------
     # Delete Triggers that shouldn't be there
@@ -86,5 +108,19 @@ async def setup_triggers(
         if not existing_automation:
             continue
 
+        report.deleted.append(existing_automation)
         await client.delete_automation(automation_id=existing_automation.id)
         log.info(f"{item_to_delete} Deleted")
+
+    if trigger_type:
+        log.info(
+            f"Processed triggers of type {trigger_type.value}: "
+            f"{len(report.created)} created, {len(report.updated)} updated, {len(report.unchanged)} unchanged, {len(report.deleted)} deleted"
+        )
+    else:
+        log.info(
+            f"Processed all triggers: "
+            f"{len(report.created)} created, {len(report.updated)} updated, {len(report.unchanged)} unchanged, {len(report.deleted)} deleted"
+        )
+
+    return report

infrahub/trigger/tasks.py

@@ -31,7 +31,4 @@ async def trigger_configure_all(service: InfrahubServices) -> None:
         )
 
         async with get_client(sync_client=False) as prefect_client:
-            await setup_triggers(
-                client=prefect_client,
-                triggers=triggers,
-            )
+            await setup_triggers(client=prefect_client, triggers=triggers, force_update=True)

infrahub/types.py

@@ -366,7 +366,7 @@ ATTRIBUTE_PYTHON_TYPES: dict[str, type] = {
 ATTRIBUTE_KIND_LABELS = list(ATTRIBUTE_TYPES.keys())
 
 # Data types supporting large values, which can therefore not be indexed in neo4j.
-LARGE_ATTRIBUTE_TYPES = [TextArea, JSON]
+LARGE_ATTRIBUTE_TYPES = [TextArea, JSON, List]
 
 
 def get_attribute_type(kind: str = "Default") -> type[InfrahubDataType]:

infrahub/webhook/models.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import base64
 import hashlib
 import hmac
+import json
 from typing import TYPE_CHECKING, Any
 from uuid import UUID, uuid4
 
@@ -170,7 +171,7 @@ class StandardWebhook(Webhook):
     def _assign_headers(self, uuid: UUID | None = None, at: Timestamp | None = None) -> None:
         message_id = f"msg_{uuid.hex}" if uuid else f"msg_{uuid4().hex}"
         timestamp = str(at.to_timestamp()) if at else str(Timestamp().to_timestamp())
-        payload = self._payload or {}
+        payload = json.dumps(self._payload or {})
         unsigned_data = f"{message_id}.{timestamp}.{payload}".encode()
         signature = self._sign(data=unsigned_data)
 

infrahub/workflows/catalogue.py

@@ -251,6 +251,14 @@ COMPUTED_ATTRIBUTE_PROCESS_JINJA2 = WorkflowDefinition(
     tags=[WorkflowTag.DATABASE_CHANGE],
 )
 
+COMPUTED_ATTRIBUTE_JINJA2_UPDATE_VALUE = WorkflowDefinition(
+    name="computed-attribute-jinja2-update-value",
+    type=WorkflowType.CORE,
+    module="infrahub.computed_attribute.tasks",
+    function="computed_attribute_jinja2_update_value",
+    tags=[WorkflowTag.DATABASE_CHANGE],
+)
+
 TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES = WorkflowDefinition(
     name="trigger_update_jinja2_computed_attributes",
     type=WorkflowType.CORE,
@@ -443,6 +451,7 @@ workflows = [
     BRANCH_MERGE_POST_PROCESS,
     BRANCH_REBASE,
     BRANCH_VALIDATE,
+    COMPUTED_ATTRIBUTE_JINJA2_UPDATE_VALUE,
     COMPUTED_ATTRIBUTE_PROCESS_JINJA2,
     COMPUTED_ATTRIBUTE_PROCESS_TRANSFORM,
     COMPUTED_ATTRIBUTE_SETUP_JINJA2,

infrahub/workflows/initialization.py

@@ -71,7 +71,5 @@ async def setup_task_manager() -> None:
         await setup_worker_pools(client=client)
         await setup_deployments(client=client)
         await setup_triggers(
-            client=client,
-            triggers=builtin_triggers,
-            trigger_type=TriggerType.BUILTIN,
+            client=client, triggers=builtin_triggers, trigger_type=TriggerType.BUILTIN, force_update=True
         )

infrahub_sdk/timestamp.py

@@ -153,7 +153,7 @@ class Timestamp:
         nanoseconds: int = 0,
         disambiguate: Literal["compatible"] = "compatible",
     ) -> Timestamp:
-        return Timestamp(
+        return self.__class__(
             self._obj.add(
                 years=years,
                 months=months,
@@ -183,7 +183,7 @@ class Timestamp:
         nanoseconds: int = 0,
         disambiguate: Literal["compatible"] = "compatible",
     ) -> Timestamp:
-        return Timestamp(
+        return self.__class__(
             self._obj.subtract(
                 years=years,
                 months=months,

{infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: infrahub-server
-Version: 1.2.9rc0
+Version: 1.2.11
 Summary: Infrahub is taking a new approach to Infrastructure Management by providing a new generation of datastore to organize and control all the data that defines how an infrastructure should run.
 Home-page: https://opsmill.com
 License: AGPL-3.0-only
@@ -77,7 +77,7 @@ Description-Content-Type: text/markdown
 
 Infrahub from [OpsMill](https://opsmill.com) is taking a new approach to Infrastructure Management by providing a new generation of datastore to organize and control all the data that defines how an infrastructure should run. Infrahub offers a central hub to manage the data, templates and playbooks that powers your infrastructure by combining the version control and branch management capabilities similar to Git with the flexible data model and UI of a graph database.
 
-If you just want to try Infrahub out, you can use our [Always-On Sandbox](https://demo.infrahub.app/) to get started.
+If you just want to try Infrahub out, you can use our [Infrahub Sandbox](https://sandbox.infrahub.app/) to get started.
 
 ![infrahub screenshot](docs/docs/media/infrahub-readme.gif)
 
@@ -103,7 +103,7 @@ If you just want to try Infrahub out, you can use our [Always-On Sandbox](https:
 
 ## Quick Start
 
-[Always-On Sandbox](https://demo.infrahub.app/) - Instantly login to the UI of a demo environment of Infrahub with sample data pre-loaded.
+[Infrahub Sandbox](https://sandbox.infrahub.app/) - Instantly login to the UI of a demo environment of Infrahub with sample data pre-loaded.
 
 [Getting Started Environment & Tutorial](https://opsmill.instruqt.com/pages/labs) - It spins up an instance of Infrahub on our cloud, provides a browser, terminal, code editor and walks you through the basic concepts: