infrahub-server 1.4.12__py3-none-any.whl → 1.5.0__py3-none-any.whl

infrahub/cli/dev.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import importlib
+import logging
+from pathlib import Path  # noqa: TC003
+from typing import TYPE_CHECKING
+
+import typer
+from graphql import parse, print_ast, print_schema
+from infrahub_sdk.async_typer import AsyncTyper
+from rich.logging import RichHandler
+
+from infrahub import config
+from infrahub.core.initialization import (
+    first_time_initialization,
+    initialization,
+)
+from infrahub.core.schema import SchemaRoot, core_models, internal_schema
+from infrahub.core.schema.schema_branch import SchemaBranch
+from infrahub.core.utils import delete_all_nodes
+from infrahub.graphql.manager import GraphQLSchemaManager
+from infrahub.graphql.schema_sort import sort_schema_ast
+from infrahub.log import get_logger
+
+if TYPE_CHECKING:
+    from infrahub.cli.context import CliContext
+
+app = AsyncTyper()
+
+
+@app.command(name="export-graphql-schema")
+async def export_graphql_schema(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Option("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    out: Path = typer.Option("schema.graphql"),  # noqa: B008
+) -> None:
+    """Export the Core GraphQL schema to a file."""
+
+    config.load_and_exit(config_file_name=config_file)
+
+    schema = SchemaRoot(**internal_schema)
+    full_schema = schema.merge(schema=SchemaRoot(**core_models))
+
+    schema_branch = SchemaBranch(cache={}, name="default")
+    schema_branch.load_schema(schema=full_schema)
+
+    schema_branch.process()
+
+    gqlm = GraphQLSchemaManager(schema=schema_branch)
+    gql_schema = gqlm.generate()
+
+    schema_str = print_schema(gql_schema)
+    schema_ast = parse(schema_str)
+    sorted_schema_ast = sort_schema_ast(schema_ast)
+    sorted_schema_str = print_ast(sorted_schema_ast)
+
+    out.write_text(sorted_schema_str)
+
+
+@app.command(name="db-init")
+async def database_init(
+    ctx: typer.Context,
+    config_file: str = typer.Option(
+        "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
+    ),
+) -> None:
+    """Erase the content of the database and initialize it with the core schema."""
+
+    log = get_logger()
+
+    # --------------------------------------------------
+    # CLEANUP
+    #  - For now we delete everything in the database
+    #   TODO, if possible try to implement this in an idempotent way
+    # --------------------------------------------------
+
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    config.load_and_exit(config_file_name=config_file)
+
+    context: CliContext = ctx.obj
+    dbdriver = await context.init_db(retry=1)
+    async with dbdriver.start_transaction() as db:
+        log.info("Delete All Nodes")
+        await delete_all_nodes(db=db)
+        await first_time_initialization(db=db)
+
+    await dbdriver.close()
+
+
+@app.command(name="load-test-data")
+async def load_test_data(
+    ctx: typer.Context,
+    config_file: str = typer.Option(
+        "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
+    ),
+    dataset: str = "dataset01",
+) -> None:
+    """Load test data into the database from the `test_data` directory."""
+
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    config.load_and_exit(config_file_name=config_file)
+
+    context: CliContext = ctx.obj
+    dbdriver = await context.init_db(retry=1)
+
+    async with dbdriver.start_session() as db:
+        await initialization(db=db)
+
+        log_level = "DEBUG"
+
+        FORMAT = "%(message)s"
+        logging.basicConfig(level=log_level, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
+        logging.getLogger("infrahub")
+
+        dataset_module = importlib.import_module(f"infrahub.test_data.{dataset}")
+        await dataset_module.load_data(db=db)
+
+    await dbdriver.close()

infrahub/cli/tasks.py

@@ -3,9 +3,11 @@ import logging
 import typer
 from infrahub_sdk.async_typer import AsyncTyper
 from prefect.client.orchestration import get_client
+from prefect.client.schemas.objects import StateType
 
 from infrahub import config
 from infrahub.services.adapters.workflow.worker import WorkflowWorkerExecution
+from infrahub.task_manager.task import PrefectTask
 from infrahub.tasks.dummy import DUMMY_FLOW, DummyInput
 from infrahub.workflows.initialization import setup_task_manager
 from infrahub.workflows.models import WorkerPoolDefinition
@@ -50,3 +52,47 @@ async def execute(
             workflow=DUMMY_FLOW, parameters={"data": DummyInput(firstname="John", lastname="Doe")}
         )  # type: ignore[var-annotated]
         print(result)
+
+
+flush_app = AsyncTyper()
+
+app.add_typer(flush_app, name="flush")
+
+
+@flush_app.command()
+async def flow_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 30,
+    batch_size: int = 100,
+) -> None:
+    """Flush old task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        days_to_keep=days_to_keep,
+        batch_size=batch_size,
+    )
+
+
+@flush_app.command()
+async def stale_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 2,
+    batch_size: int = 100,
+) -> None:
+    """Flush stale task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
+    )

infrahub/cli/upgrade.py

@@ -8,13 +8,19 @@ import typer
 from deepdiff import DeepDiff
 from infrahub_sdk.async_typer import AsyncTyper
 from prefect.client.orchestration import get_client
-from rich import print as rprint
 
 from infrahub import config
-from infrahub.core.initialization import create_anonymous_role, create_default_account_groups, initialize_registry
+from infrahub.core.initialization import (
+    create_anonymous_role,
+    create_default_account_groups,
+    get_root_node,
+    initialize_registry,
+)
 from infrahub.core.manager import NodeManager
+from infrahub.core.migrations.shared import get_migration_console
 from infrahub.core.protocols import CoreAccount, CoreObjectPermission
 from infrahub.dependencies.registry import build_component_registry
+from infrahub.lock import initialize_lock
 from infrahub.menu.menu import default_menu
 from infrahub.menu.models import MenuDict
 from infrahub.menu.repository import MenuRepository
@@ -26,13 +32,22 @@ from infrahub.workflows.initialization import (
     setup_worker_pools,
 )
 
-from .db import initialize_internal_schema, migrate_database, update_core_schema
+from .db import (
+    detect_migration_to_run,
+    initialize_internal_schema,
+    mark_branches_needing_rebase,
+    migrate_database,
+    trigger_rebase_branches,
+    update_core_schema,
+)
 
 if TYPE_CHECKING:
     from infrahub.cli.context import CliContext
+    from infrahub.core.branch.models import Branch
     from infrahub.database import InfrahubDatabase
 
 app = AsyncTyper()
+console = get_migration_console()
 
 
 @app.command(name="upgrade")
@@ -40,6 +55,10 @@ async def upgrade_cmd(
     ctx: typer.Context,
     config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
     check: bool = typer.Option(False, help="Check the state of the system without upgrading."),
+    rebase_branches: bool = typer.Option(False, help="Rebase and apply migrations to branches if required."),
+    interactive: bool = typer.Option(
+        False, help="Use interactive prompt to accept or deny rebase of individual branches."
+    ),
 ) -> None:
     """Upgrade Infrahub to the latest version."""
 
@@ -54,9 +73,12 @@ async def upgrade_cmd(
     dbdriver = await context.init_db(retry=1)
 
     await initialize_registry(db=dbdriver)
+    initialize_lock()
 
     build_component_registry()
 
+    root_node = await get_root_node(db=dbdriver)
+
     # NOTE add step to validate if the database and the task manager are reachable
 
     # -------------------------------------------
@@ -67,9 +89,14 @@ async def upgrade_cmd(
     # Upgrade Infrahub Database and Schema
     # -------------------------------------------
 
-    if not await migrate_database(db=dbdriver, initialize=False, check=check):
+    migrations = await detect_migration_to_run(current_graph_version=root_node.graph_version)
+    if check:
+        await dbdriver.close()
+        return
+
+    if not await migrate_database(db=dbdriver, initialize=False, migrations=migrations):
         # A migration failed, stop the upgrade process
-        rprint("Upgrade cancelled due to migration failure.")
+        console.log("Upgrade cancelled due to migration failure.")
         await dbdriver.close()
         return
 
@@ -91,6 +118,26 @@ async def upgrade_cmd(
         await setup_deployments(client=client)
         await trigger_configure_all()
 
+    # -------------------------------------------
+    # Perform branch rebase and apply migrations to them
+    # -------------------------------------------
+    branches = await mark_branches_needing_rebase(db=dbdriver)
+    plural = len(branches) != 1
+    get_migration_console().log(
+        f"Found {len(branches)} {'branches' if plural else 'branch'} that {'need' if plural else 'needs'} to be rebased"
+    )
+
+    if rebase_branches:
+        branches_to_rebase: list[Branch] = []
+        if not interactive:
+            branches_to_rebase = branches
+        else:
+            for branch in branches:
+                if typer.confirm(f"Rebase branch {branch.name}?"):
+                    branches_to_rebase.append(branch)
+
+        await trigger_rebase_branches(db=dbdriver, branches=branches_to_rebase)
+
     await dbdriver.close()
 
 
@@ -107,21 +154,21 @@ async def upgrade_menu(db: InfrahubDatabase) -> None:
     diff_menu = DeepDiff(menu_items.to_rest(), default_menu_dict.to_rest(), ignore_order=True)
 
     if not diff_menu:
-        rprint("Menu Up to date, nothing to update")
+        console.log("Menu Up to date, nothing to update")
         return
 
     await menu_repository.update_menu(existing_menu=menu_items, new_menu=default_menu_dict, menu_nodes=menu_nodes)
-    rprint("Menu has been updated")
+    console.log("Menu has been updated")
 
 
 async def upgrade_permissions(db: InfrahubDatabase) -> None:
     existing_permissions = await NodeManager.query(schema=CoreObjectPermission, db=db, limit=1)
     if existing_permissions:
-        rprint("Permissions Up to date, nothing to update")
+        console.log("Permissions Up to date, nothing to update")
         return
 
     await setup_permissions(db=db)
-    rprint("Permissions have been updated")
+    console.log("Permissions have been updated")
 
 
 async def setup_permissions(db: InfrahubDatabase) -> None:

infrahub/computed_attribute/tasks.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from infrahub_sdk.exceptions import URLNotFoundError
 from infrahub_sdk.protocols import CoreTransformPython
 from infrahub_sdk.template import Jinja2Template
 from prefect import flow
@@ -177,12 +178,17 @@ async def computed_attribute_jinja2_update_value(
         log.debug(f"Ignoring to update {obj} with existing value on {attribute_name}={value}")
         return
 
-    await client.execute_graphql(
-        query=UPDATE_ATTRIBUTE,
-        variables={"id": obj.node_id, "kind": node_kind, "attribute": attribute_name, "value": value},
-        branch_name=branch_name,
-    )
-    log.info(f"Updating computed attribute {node_kind}.{attribute_name}='{value}' ({obj.node_id})")
+    try:
+        await client.execute_graphql(
+            query=UPDATE_ATTRIBUTE,
+            variables={"id": obj.node_id, "kind": node_kind, "attribute": attribute_name, "value": value},
+            branch_name=branch_name,
+        )
+        log.info(f"Updating computed attribute {node_kind}.{attribute_name}='{value}' ({obj.node_id})")
+    except URLNotFoundError:
+        log.warning(
+            f"Update of computed attribute {node_kind}.{attribute_name} failed for branch {branch_name} (not found)"
+        )
 
 
 @flow(
@@ -229,7 +235,13 @@ async def process_jinja2(
 
         for id_filter in computed_macro.node_filters:
             query = attribute_graphql.render_graphql_query(query_filter=id_filter, filter_id=object_id)
-            response = await client.execute_graphql(query=query, branch_name=branch_name)
+            try:
+                response = await client.execute_graphql(query=query, branch_name=branch_name)
+            except URLNotFoundError:
+                log.warning(
+                    f"Process computed attributes for {computed_attribute_kind}.{computed_attribute_name} failed for branch {branch_name} (not found)"
+                )
+                return
             output = attribute_graphql.parse_response(response=response)
             found.extend(output)
 

infrahub/config.py

@@ -8,7 +8,7 @@ from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-import toml
+import tomllib
 from infrahub_sdk.utils import generate_uuid
 from pydantic import (
     AliasChoices,
@@ -371,6 +371,11 @@ class CacheSettings(BaseSettings):
     tls_enabled: bool = Field(default=False, description="Indicates if TLS is enabled for the connection")
     tls_insecure: bool = Field(default=False, description="Indicates if TLS certificates are verified")
     tls_ca_file: str | None = Field(default=None, description="File path to CA cert or bundle in PEM format")
+    clean_up_deadlocks_interval_mins: int = Field(
+        default=15,
+        ge=1,
+        description="Age threshold in minutes: locks older than this and owned by inactive workers are deleted by the cleanup task.",
+    )
 
     @property
     def service_port(self) -> int:
@@ -975,7 +980,7 @@ def load(config_file_name: Path | str = "infrahub.toml", config_data: dict[str,
 
     if config_file.exists():
         config_string = config_file.read_text(encoding="utf-8")
-        config_tmp = toml.loads(config_string)
+        config_tmp = tomllib.loads(config_string)
 
         return Settings(**config_tmp)
 

infrahub/core/attribute.py

@@ -18,6 +18,7 @@ from infrahub.core.changelog.models import AttributeChangelog
 from infrahub.core.constants import NULL_VALUE, AttributeDBNodeType, BranchSupportType, RelationshipStatus
 from infrahub.core.property import FlagPropertyMixin, NodePropertyData, NodePropertyMixin
 from infrahub.core.query.attribute import (
+    AttributeClearNodePropertyQuery,
     AttributeGetQuery,
     AttributeUpdateFlagQuery,
     AttributeUpdateNodePropertyQuery,
@@ -36,7 +37,7 @@ from .schema.attribute_parameters import NumberAttributeParameters
 if TYPE_CHECKING:
     from infrahub.core.branch import Branch
     from infrahub.core.node import Node
-    from infrahub.core.schema import AttributeSchema
+    from infrahub.core.schema import AttributeSchema, MainSchemaTypes
     from infrahub.database import InfrahubDatabase
 
 
@@ -324,7 +325,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
 
         save_at = Timestamp(at)
 
-        if not self.id or self.is_from_profile:
+        if not self.id:
             return None
 
         return await self._update(at=save_at, db=db)
@@ -395,7 +396,6 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
 
         Get the current value
          - If the value is the same, do nothing
-         - If the value is inherited and is different, raise error (for now just ignore)
          - If the value is different, create new node and update relationship
 
         """
@@ -470,28 +470,38 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
 
         # ---------- Update the Node Properties ----------
         for prop_name in self._node_properties:
-            if getattr(self, f"{prop_name}_id") and not (
-                prop_name in current_attr_data.node_properties
-                and current_attr_data.node_properties[prop_name].uuid == getattr(self, f"{prop_name}_id")
-            ):
-                previous_attribute_node_property = current_attr_data.node_properties.get(prop_name)
-                previous_value = None
-                if previous_attribute_node_property:
-                    previous_value = previous_attribute_node_property.uuid
+            current_prop_id = getattr(self, f"{prop_name}_id")
+            database_prop_id: str | None = None
+            if prop_name in current_attr_data.node_properties:
+                database_prop_id = current_attr_data.node_properties[prop_name].uuid
+            needs_update = current_prop_id is not None and current_prop_id != database_prop_id
+            needs_clear = self.is_clear(prop_name) and database_prop_id
+
+            if not needs_update and not needs_clear:
+                continue
 
-                changelog.add_property(
-                    name=prop_name,
-                    value_current=getattr(self, f"{prop_name}_id"),
-                    value_previous=previous_value,
-                )
+            changelog.add_property(
+                name=prop_name,
+                value_current=current_prop_id,
+                value_previous=database_prop_id,
+            )
+
+            if needs_update:
                 query = await AttributeUpdateNodePropertyQuery.init(
-                    db=db, attr=self, at=update_at, prop_name=prop_name, prop_id=getattr(self, f"{prop_name}_id")
+                    db=db, attr=self, at=update_at, prop_name=prop_name, prop_id=current_prop_id
                 )
                 await query.execute(db=db)
 
-                rel = current_attr_result.get(f"rel_{prop_name}")
-                if rel and rel.get("branch") == branch.name:
-                    await update_relationships_to([rel.element_id], to=update_at, db=db)
+            if needs_clear:
+                query = await AttributeClearNodePropertyQuery.init(
+                    db=db, attr=self, at=update_at, prop_name=prop_name, prop_id=database_prop_id
+                )
+                await query.execute(db=db)
+
+            # set the to time on the previously active edge
+            rel = current_attr_result.get(f"rel_{prop_name}")
+            if rel and rel.get("branch") == branch.name:
+                await update_relationships_to([rel.element_id], to=update_at, db=db)
 
         if changelog.has_updates:
             return changelog
@@ -578,7 +588,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
 
         return value
 
-    async def from_graphql(self, data: dict, db: InfrahubDatabase) -> bool:
+    async def from_graphql(self, data: dict, db: InfrahubDatabase, process_pools: bool = True) -> bool:
         """Update attr from GraphQL payload"""
 
         changed = False
@@ -592,7 +602,8 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
                 changed = True
         elif "from_pool" in data:
             self.from_pool = data["from_pool"]
-            await self.node.handle_pool(db=db, attribute=self, errors=[])
+            if process_pools:
+                await self.node.handle_pool(db=db, attribute=self, errors=[])
             changed = True
 
         if changed and self.is_from_profile:
@@ -627,7 +638,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
             return AttributeDBNodeType.DEFAULT
         return AttributeDBNodeType.INDEXED
 
-    def get_create_data(self) -> AttributeCreateData:
+    def get_create_data(self, node_schema: MainSchemaTypes) -> AttributeCreateData:
         branch = self.branch
         hierarchy_level = branch.hierarchy_level
         if self.schema.branch == BranchSupportType.AGNOSTIC:
@@ -642,7 +653,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
             branch=branch.name,
             status="active",
             branch_level=hierarchy_level,
-            branch_support=self.schema.branch.value,
+            branch_support=self.schema.branch.value if self.schema.branch is not None else node_schema.branch,
             content=self.to_db(),
             is_default=self.is_default,
             is_protected=self.is_protected,

infrahub/core/branch/enums.py

@@ -4,5 +4,5 @@ from infrahub.utils import InfrahubStringEnum
 class BranchStatus(InfrahubStringEnum):
     OPEN = "OPEN"
     NEED_REBASE = "NEED_REBASE"
-    CLOSED = "CLOSED"
+    NEED_UPGRADE_REBASE = "NEED_UPGRADE_REBASE"
     DELETING = "DELETING"

infrahub/core/branch/models.py

@@ -5,9 +5,9 @@ from typing import TYPE_CHECKING, Any, Optional, Self, Union
 
 from pydantic import Field, field_validator
 
-from infrahub.core.constants import (
-    GLOBAL_BRANCH_NAME,
-)
+from infrahub.core.branch.enums import BranchStatus
+from infrahub.core.constants import GLOBAL_BRANCH_NAME
+from infrahub.core.graph import GRAPH_VERSION
 from infrahub.core.models import SchemaBranchHash  # noqa: TC001
 from infrahub.core.node.standard import StandardNode
 from infrahub.core.query import QueryType
@@ -21,8 +21,6 @@ from infrahub.core.registry import registry
 from infrahub.core.timestamp import Timestamp
 from infrahub.exceptions import BranchNotFoundError, InitializationError, ValidationError
 
-from .enums import BranchStatus
-
 if TYPE_CHECKING:
     from infrahub.database import InfrahubDatabase
 
@@ -47,6 +45,7 @@ class Branch(StandardNode):
     is_isolated: bool = True
     schema_changed_at: Optional[str] = None
     schema_hash: Optional[SchemaBranchHash] = None
+    graph_version: int | None = None
 
     _exclude_attrs: list[str] = ["id", "uuid", "owner"]
 
@@ -262,6 +261,10 @@ class Branch(StandardNode):
 
         return start, end
 
+    async def create(self, db: InfrahubDatabase) -> bool:
+        self.graph_version = GRAPH_VERSION
+        return await super().create(db=db)
+
     async def delete(self, db: InfrahubDatabase) -> None:
         if self.is_default:
             raise ValidationError(f"Unable to delete {self.name} it is the default branch.")
@@ -485,6 +488,7 @@ class Branch(StandardNode):
         # FIXME, we must ensure that there is no conflict before rebasing a branch
         #   Otherwise we could endup with a complicated situation
         self.branched_from = at.to_string()
+        self.status = BranchStatus.OPEN
         await self.save(db=db)
 
         # Update the branch in the registry after the rebase

infrahub/core/branch/needs_rebase_status.py

@@ -0,0 +1,11 @@
+from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
+
+
+def raise_needs_rebase_error(branch_name: str) -> None:
+    raise ValueError(f"Branch {branch_name} must be rebased before any updates can be made")
+
+
+def check_need_rebase_status(branch: Branch) -> None:
+    if branch.status == BranchStatus.NEED_REBASE:
+        raise_needs_rebase_error(branch_name=branch.name)