infrahub-server 1.5.0b1__py3-none-any.whl → 1.5.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. infrahub/api/internal.py +2 -0
  2. infrahub/api/oauth2.py +13 -19
  3. infrahub/api/oidc.py +15 -21
  4. infrahub/api/schema.py +24 -3
  5. infrahub/artifacts/models.py +2 -1
  6. infrahub/auth.py +137 -3
  7. infrahub/cli/__init__.py +2 -0
  8. infrahub/cli/db.py +83 -102
  9. infrahub/cli/dev.py +118 -0
  10. infrahub/cli/tasks.py +46 -0
  11. infrahub/cli/upgrade.py +30 -3
  12. infrahub/computed_attribute/tasks.py +20 -8
  13. infrahub/core/attribute.py +10 -2
  14. infrahub/core/branch/enums.py +1 -1
  15. infrahub/core/branch/models.py +7 -3
  16. infrahub/core/branch/tasks.py +68 -7
  17. infrahub/core/constants/__init__.py +3 -0
  18. infrahub/core/diff/query/artifact.py +1 -0
  19. infrahub/core/diff/query/field_summary.py +1 -0
  20. infrahub/core/graph/__init__.py +1 -1
  21. infrahub/core/initialization.py +5 -2
  22. infrahub/core/migrations/__init__.py +3 -0
  23. infrahub/core/migrations/exceptions.py +4 -0
  24. infrahub/core/migrations/graph/__init__.py +10 -13
  25. infrahub/core/migrations/graph/load_schema_branch.py +21 -0
  26. infrahub/core/migrations/graph/m013_convert_git_password_credential.py +1 -1
  27. infrahub/core/migrations/graph/m040_duplicated_attributes.py +81 -0
  28. infrahub/core/migrations/graph/m041_profile_attrs_in_db.py +145 -0
  29. infrahub/core/migrations/graph/m042_create_hfid_display_label_in_db.py +164 -0
  30. infrahub/core/migrations/graph/m043_backfill_hfid_display_label_in_db.py +866 -0
  31. infrahub/core/migrations/query/__init__.py +7 -8
  32. infrahub/core/migrations/query/attribute_add.py +8 -6
  33. infrahub/core/migrations/query/attribute_remove.py +134 -0
  34. infrahub/core/migrations/runner.py +54 -0
  35. infrahub/core/migrations/schema/attribute_kind_update.py +9 -3
  36. infrahub/core/migrations/schema/attribute_supports_profile.py +90 -0
  37. infrahub/core/migrations/schema/node_attribute_add.py +30 -2
  38. infrahub/core/migrations/schema/node_attribute_remove.py +13 -109
  39. infrahub/core/migrations/schema/node_kind_update.py +2 -1
  40. infrahub/core/migrations/schema/node_remove.py +2 -1
  41. infrahub/core/migrations/schema/placeholder_dummy.py +3 -2
  42. infrahub/core/migrations/shared.py +48 -14
  43. infrahub/core/node/__init__.py +16 -11
  44. infrahub/core/node/create.py +46 -63
  45. infrahub/core/node/lock_utils.py +70 -44
  46. infrahub/core/node/resource_manager/ip_address_pool.py +2 -1
  47. infrahub/core/node/resource_manager/ip_prefix_pool.py +2 -1
  48. infrahub/core/node/resource_manager/number_pool.py +2 -1
  49. infrahub/core/query/attribute.py +55 -0
  50. infrahub/core/query/ipam.py +1 -0
  51. infrahub/core/query/node.py +9 -3
  52. infrahub/core/query/relationship.py +1 -0
  53. infrahub/core/schema/__init__.py +56 -0
  54. infrahub/core/schema/attribute_schema.py +4 -0
  55. infrahub/core/schema/definitions/internal.py +2 -2
  56. infrahub/core/schema/generated/attribute_schema.py +2 -2
  57. infrahub/core/schema/manager.py +22 -1
  58. infrahub/core/schema/schema_branch.py +180 -22
  59. infrahub/database/graph.py +21 -0
  60. infrahub/display_labels/tasks.py +13 -7
  61. infrahub/events/branch_action.py +27 -1
  62. infrahub/generators/tasks.py +3 -7
  63. infrahub/git/base.py +4 -1
  64. infrahub/git/integrator.py +1 -1
  65. infrahub/git/models.py +2 -1
  66. infrahub/git/repository.py +22 -5
  67. infrahub/git/tasks.py +66 -10
  68. infrahub/git/utils.py +123 -1
  69. infrahub/graphql/api/endpoints.py +14 -4
  70. infrahub/graphql/manager.py +4 -9
  71. infrahub/graphql/mutations/convert_object_type.py +11 -1
  72. infrahub/graphql/mutations/display_label.py +17 -10
  73. infrahub/graphql/mutations/hfid.py +17 -10
  74. infrahub/graphql/mutations/ipam.py +54 -35
  75. infrahub/graphql/mutations/main.py +27 -28
  76. infrahub/graphql/schema_sort.py +170 -0
  77. infrahub/graphql/types/branch.py +4 -1
  78. infrahub/graphql/types/enums.py +3 -0
  79. infrahub/hfid/tasks.py +13 -7
  80. infrahub/lock.py +52 -12
  81. infrahub/message_bus/types.py +2 -1
  82. infrahub/permissions/constants.py +2 -0
  83. infrahub/proposed_change/tasks.py +25 -16
  84. infrahub/server.py +6 -2
  85. infrahub/services/__init__.py +2 -2
  86. infrahub/services/adapters/http/__init__.py +5 -0
  87. infrahub/services/adapters/workflow/worker.py +14 -3
  88. infrahub/task_manager/event.py +5 -0
  89. infrahub/task_manager/models.py +7 -0
  90. infrahub/task_manager/task.py +73 -0
  91. infrahub/trigger/setup.py +13 -4
  92. infrahub/trigger/tasks.py +3 -0
  93. infrahub/workers/dependencies.py +10 -1
  94. infrahub/workers/infrahub_async.py +10 -2
  95. infrahub/workflows/catalogue.py +8 -0
  96. infrahub/workflows/initialization.py +5 -0
  97. infrahub/workflows/utils.py +2 -1
  98. infrahub_sdk/client.py +13 -10
  99. infrahub_sdk/config.py +29 -2
  100. infrahub_sdk/ctl/schema.py +22 -7
  101. infrahub_sdk/schema/__init__.py +32 -4
  102. infrahub_sdk/spec/models.py +7 -0
  103. infrahub_sdk/spec/object.py +37 -102
  104. infrahub_sdk/spec/processors/__init__.py +0 -0
  105. infrahub_sdk/spec/processors/data_processor.py +10 -0
  106. infrahub_sdk/spec/processors/factory.py +34 -0
  107. infrahub_sdk/spec/processors/range_expand_processor.py +56 -0
  108. {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/METADATA +3 -1
  109. {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/RECORD +115 -101
  110. infrahub_testcontainers/container.py +114 -2
  111. infrahub_testcontainers/docker-compose-cluster.test.yml +5 -0
  112. infrahub_testcontainers/docker-compose.test.yml +5 -0
  113. infrahub/core/migrations/graph/m040_profile_attrs_in_db.py +0 -166
  114. infrahub/core/migrations/graph/m041_create_hfid_display_label_in_db.py +0 -97
  115. infrahub/core/migrations/graph/m042_backfill_hfid_display_label_in_db.py +0 -86
  116. {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/LICENSE.txt +0 -0
  117. {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/WHEEL +0 -0
  118. {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/entry_points.txt +0 -0
infrahub/cli/db.py CHANGED
@@ -1,6 +1,5 @@
1
1
  from __future__ import annotations
2
2
 
3
- import importlib
4
3
  import logging
5
4
  import os
6
5
  from collections import defaultdict
@@ -16,11 +15,15 @@ from infrahub_sdk.async_typer import AsyncTyper
16
15
  from prefect.testing.utilities import prefect_test_harness
17
16
  from rich import print as rprint
18
17
  from rich.console import Console
19
- from rich.logging import RichHandler
20
18
  from rich.table import Table
21
19
 
22
20
  from infrahub import config
21
+ from infrahub.auth import AccountSession, AuthType
22
+ from infrahub.context import InfrahubContext
23
23
  from infrahub.core import registry
24
+ from infrahub.core.branch import Branch
25
+ from infrahub.core.branch.tasks import rebase_branch
26
+ from infrahub.core.constants import GLOBAL_BRANCH_NAME
24
27
  from infrahub.core.graph import GRAPH_VERSION
25
28
  from infrahub.core.graph.constraints import ConstraintManagerBase, ConstraintManagerMemgraph, ConstraintManagerNeo4j
26
29
  from infrahub.core.graph.index import node_indexes, rel_indexes
@@ -32,25 +35,20 @@ from infrahub.core.graph.schema import (
32
35
  GraphRelationshipIsPartOf,
33
36
  GraphRelationshipProperties,
34
37
  )
35
- from infrahub.core.initialization import (
36
- first_time_initialization,
37
- get_root_node,
38
- initialization,
39
- initialize_registry,
40
- )
38
+ from infrahub.core.initialization import get_root_node, initialize_registry
39
+ from infrahub.core.migrations.exceptions import MigrationFailureError
41
40
  from infrahub.core.migrations.graph import get_graph_migrations, get_migration_by_number
42
41
  from infrahub.core.migrations.schema.models import SchemaApplyMigrationData
43
42
  from infrahub.core.migrations.schema.tasks import schema_apply_migrations
44
43
  from infrahub.core.schema import SchemaRoot, core_models, internal_schema
45
44
  from infrahub.core.schema.definitions.deprecated import deprecated_models
46
45
  from infrahub.core.schema.manager import SchemaManager
47
- from infrahub.core.utils import delete_all_nodes
48
46
  from infrahub.core.validators.models.validate_migration import SchemaValidateMigrationData
49
47
  from infrahub.core.validators.tasks import schema_validate_migrations
50
48
  from infrahub.database import DatabaseType
51
49
  from infrahub.database.memgraph import IndexManagerMemgraph
52
50
  from infrahub.database.neo4j import IndexManagerNeo4j
53
- from infrahub.log import get_logger
51
+ from infrahub.exceptions import ValidationError
54
52
 
55
53
  from .constants import ERROR_BADGE, FAILED_BADGE, SUCCESS_BADGE
56
54
  from .db_commands.check_inheritance import check_inheritance
@@ -65,7 +63,7 @@ def get_timestamp_string() -> str:
65
63
 
66
64
  if TYPE_CHECKING:
67
65
  from infrahub.cli.context import CliContext
68
- from infrahub.core.migrations.shared import ArbitraryMigration, GraphMigration, InternalSchemaMigration
66
+ from infrahub.core.migrations.shared import MigrationTypes
69
67
  from infrahub.database import InfrahubDatabase
70
68
  from infrahub.database.index import IndexManagerBase
71
69
 
@@ -94,67 +92,6 @@ def callback() -> None:
94
92
  """
95
93
 
96
94
 
97
- @app.command()
98
- async def init(
99
- ctx: typer.Context,
100
- config_file: str = typer.Option(
101
- "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
102
- ),
103
- ) -> None:
104
- """Erase the content of the database and initialize it with the core schema."""
105
-
106
- log = get_logger()
107
-
108
- # --------------------------------------------------
109
- # CLEANUP
110
- # - For now we delete everything in the database
111
- # TODO, if possible try to implement this in an idempotent way
112
- # --------------------------------------------------
113
-
114
- logging.getLogger("neo4j").setLevel(logging.ERROR)
115
- config.load_and_exit(config_file_name=config_file)
116
-
117
- context: CliContext = ctx.obj
118
- dbdriver = await context.init_db(retry=1)
119
- async with dbdriver.start_transaction() as db:
120
- log.info("Delete All Nodes")
121
- await delete_all_nodes(db=db)
122
- await first_time_initialization(db=db)
123
-
124
- await dbdriver.close()
125
-
126
-
127
- @app.command()
128
- async def load_test_data(
129
- ctx: typer.Context,
130
- config_file: str = typer.Option(
131
- "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
132
- ),
133
- dataset: str = "dataset01",
134
- ) -> None:
135
- """Load test data into the database from the `test_data` directory."""
136
-
137
- logging.getLogger("neo4j").setLevel(logging.ERROR)
138
- config.load_and_exit(config_file_name=config_file)
139
-
140
- context: CliContext = ctx.obj
141
- dbdriver = await context.init_db(retry=1)
142
-
143
- async with dbdriver.start_session() as db:
144
- await initialization(db=db)
145
-
146
- log_level = "DEBUG"
147
-
148
- FORMAT = "%(message)s"
149
- logging.basicConfig(level=log_level, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
150
- logging.getLogger("infrahub")
151
-
152
- dataset_module = importlib.import_module(f"infrahub.test_data.{dataset}")
153
- await dataset_module.load_data(db=db)
154
-
155
- await dbdriver.close()
156
-
157
-
158
95
  @app.command(name="migrate")
159
96
  async def migrate_cmd(
160
97
  ctx: typer.Context,
@@ -172,7 +109,15 @@ async def migrate_cmd(
172
109
  context: CliContext = ctx.obj
173
110
  dbdriver = await context.init_db(retry=1)
174
111
 
175
- await migrate_database(db=dbdriver, initialize=True, check=check, migration_number=migration_number)
112
+ root_node = await get_root_node(db=dbdriver)
113
+ migrations = await detect_migration_to_run(
114
+ current_graph_version=root_node.graph_version, migration_number=migration_number
115
+ )
116
+
117
+ if check or not migrations:
118
+ return
119
+
120
+ await migrate_database(db=dbdriver, migrations=migrations, initialize=True)
176
121
 
177
122
  await dbdriver.close()
178
123
 
@@ -335,8 +280,38 @@ async def index(
335
280
  await dbdriver.close()
336
281
 
337
282
 
283
+ async def detect_migration_to_run(
284
+ current_graph_version: int, migration_number: int | str | None = None
285
+ ) -> Sequence[MigrationTypes]:
286
+ """Return a sequence of migrations to apply to upgrade the database."""
287
+ rprint("Checking current state of the database")
288
+ migrations: list[MigrationTypes] = []
289
+
290
+ if migration_number:
291
+ migration = get_migration_by_number(migration_number)
292
+ migrations.append(migration)
293
+ if current_graph_version > migration.minimum_version:
294
+ rprint(
295
+ f"Migration {migration_number} already applied. To apply again, run the command without the --check flag."
296
+ )
297
+ return []
298
+ rprint(
299
+ f"Migration {migration_number} needs to be applied. Run `infrahub db migrate` to apply all outstanding migrations."
300
+ )
301
+ else:
302
+ migrations.extend(await get_graph_migrations(current_graph_version=current_graph_version))
303
+ if not migrations:
304
+ rprint(f"Database up-to-date (v{current_graph_version}), no migration to execute.")
305
+ return []
306
+
307
+ rprint(
308
+ f"Database needs to be updated (v{current_graph_version} -> v{GRAPH_VERSION}), {len(migrations)} migrations pending"
309
+ )
310
+ return migrations
311
+
312
+
338
313
  async def migrate_database(
339
- db: InfrahubDatabase, initialize: bool = False, check: bool = False, migration_number: int | str | None = None
314
+ db: InfrahubDatabase, migrations: Sequence[MigrationTypes], initialize: bool = False
340
315
  ) -> bool:
341
316
  """Apply the latest migrations to the database, this function will print the status directly in the console.
342
317
 
@@ -344,40 +319,16 @@ async def migrate_database(
344
319
 
345
320
  Args:
346
321
  db: The database object.
347
- check: If True, the function will only check the status of the database and not apply the migrations. Defaults to False.
348
- migration_number: If provided, the function will only apply the migration with the given number. Defaults to None.
322
+ migrations: Sequence of migrations to apply.
323
+ initialize: Whether to initialize the registry before running migrations.
349
324
  """
350
- rprint("Checking current state of the Database")
325
+ if not migrations:
326
+ return True
351
327
 
352
328
  if initialize:
353
329
  await initialize_registry(db=db)
354
330
 
355
331
  root_node = await get_root_node(db=db)
356
- if migration_number:
357
- migration = get_migration_by_number(migration_number)
358
- migrations: Sequence[GraphMigration | InternalSchemaMigration | ArbitraryMigration] = [migration]
359
- if check:
360
- if root_node.graph_version > migration.minimum_version:
361
- rprint(
362
- f"Migration {migration_number} already applied. To apply again, run the command without the --check flag."
363
- )
364
- return True
365
- rprint(
366
- f"Migration {migration_number} needs to be applied. Run `infrahub db migrate` to apply all outstanding migrations."
367
- )
368
- return False
369
- else:
370
- migrations = await get_graph_migrations(root=root_node)
371
- if not migrations:
372
- rprint(f"Database up-to-date (v{root_node.graph_version}), no migration to execute.")
373
- return True
374
-
375
- rprint(
376
- f"Database needs to be updated (v{root_node.graph_version} -> v{GRAPH_VERSION}), {len(migrations)} migrations pending"
377
- )
378
-
379
- if check:
380
- return True
381
332
 
382
333
  for migration in migrations:
383
334
  execution_result = await migration.execute(db=db)
@@ -402,6 +353,36 @@ async def migrate_database(
402
353
  return True
403
354
 
404
355
 
356
+ async def trigger_rebase_branches(db: InfrahubDatabase) -> None:
357
+ """Trigger rebase of non-default branches, also triggering migrations in the process."""
358
+ branches = [b for b in await Branch.get_list(db=db) if b.name not in [registry.default_branch, GLOBAL_BRANCH_NAME]]
359
+ if not branches:
360
+ return
361
+
362
+ rprint(f"Planning rebase and migrations for {len(branches)} branches: {', '.join([b.name for b in branches])}")
363
+
364
+ for branch in branches:
365
+ if branch.graph_version == GRAPH_VERSION:
366
+ rprint(
367
+ f"Ignoring branch rebase and migrations for '{branch.name}' (ID: {branch.uuid}), it is already up-to-date"
368
+ )
369
+ continue
370
+
371
+ rprint(f"Rebasing branch '{branch.name}' (ID: {branch.uuid})...", end="")
372
+ try:
373
+ await registry.schema.load_schema(db=db, branch=branch)
374
+ await rebase_branch(
375
+ branch=branch.name,
376
+ context=InfrahubContext.init(
377
+ branch=branch, account=AccountSession(auth_type=AuthType.NONE, authenticated=False, account_id="")
378
+ ),
379
+ send_events=False,
380
+ )
381
+ rprint(SUCCESS_BADGE)
382
+ except (ValidationError, MigrationFailureError):
383
+ rprint(FAILED_BADGE)
384
+
385
+
405
386
  async def initialize_internal_schema() -> None:
406
387
  registry.schema = SchemaManager()
407
388
  schema = SchemaRoot(**internal_schema)
infrahub/cli/dev.py ADDED
@@ -0,0 +1,118 @@
1
+ from __future__ import annotations
2
+
3
+ import importlib
4
+ import logging
5
+ from pathlib import Path # noqa: TC003
6
+ from typing import TYPE_CHECKING
7
+
8
+ import typer
9
+ from graphql import parse, print_ast, print_schema
10
+ from infrahub_sdk.async_typer import AsyncTyper
11
+ from rich.logging import RichHandler
12
+
13
+ from infrahub import config
14
+ from infrahub.core.initialization import (
15
+ first_time_initialization,
16
+ initialization,
17
+ )
18
+ from infrahub.core.schema import SchemaRoot, core_models, internal_schema
19
+ from infrahub.core.schema.schema_branch import SchemaBranch
20
+ from infrahub.core.utils import delete_all_nodes
21
+ from infrahub.graphql.manager import GraphQLSchemaManager
22
+ from infrahub.graphql.schema_sort import sort_schema_ast
23
+ from infrahub.log import get_logger
24
+
25
+ if TYPE_CHECKING:
26
+ from infrahub.cli.context import CliContext
27
+
28
+ app = AsyncTyper()
29
+
30
+
31
+ @app.command(name="export-graphql-schema")
32
+ async def export_graphql_schema(
33
+ ctx: typer.Context, # noqa: ARG001
34
+ config_file: str = typer.Option("infrahub.toml", envvar="INFRAHUB_CONFIG"),
35
+ out: Path = typer.Option("schema.graphql"), # noqa: B008
36
+ ) -> None:
37
+ """Export the Core GraphQL schema to a file."""
38
+
39
+ config.load_and_exit(config_file_name=config_file)
40
+
41
+ schema = SchemaRoot(**internal_schema)
42
+ full_schema = schema.merge(schema=SchemaRoot(**core_models))
43
+
44
+ schema_branch = SchemaBranch(cache={}, name="default")
45
+ schema_branch.load_schema(schema=full_schema)
46
+
47
+ schema_branch.process()
48
+
49
+ gqlm = GraphQLSchemaManager(schema=schema_branch)
50
+ gql_schema = gqlm.generate()
51
+
52
+ schema_str = print_schema(gql_schema)
53
+ schema_ast = parse(schema_str)
54
+ sorted_schema_ast = sort_schema_ast(schema_ast)
55
+ sorted_schema_str = print_ast(sorted_schema_ast)
56
+
57
+ out.write_text(sorted_schema_str)
58
+
59
+
60
+ @app.command(name="db-init")
61
+ async def database_init(
62
+ ctx: typer.Context,
63
+ config_file: str = typer.Option(
64
+ "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
65
+ ),
66
+ ) -> None:
67
+ """Erase the content of the database and initialize it with the core schema."""
68
+
69
+ log = get_logger()
70
+
71
+ # --------------------------------------------------
72
+ # CLEANUP
73
+ # - For now we delete everything in the database
74
+ # TODO, if possible try to implement this in an idempotent way
75
+ # --------------------------------------------------
76
+
77
+ logging.getLogger("neo4j").setLevel(logging.ERROR)
78
+ config.load_and_exit(config_file_name=config_file)
79
+
80
+ context: CliContext = ctx.obj
81
+ dbdriver = await context.init_db(retry=1)
82
+ async with dbdriver.start_transaction() as db:
83
+ log.info("Delete All Nodes")
84
+ await delete_all_nodes(db=db)
85
+ await first_time_initialization(db=db)
86
+
87
+ await dbdriver.close()
88
+
89
+
90
+ @app.command(name="load-test-data")
91
+ async def load_test_data(
92
+ ctx: typer.Context,
93
+ config_file: str = typer.Option(
94
+ "infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
95
+ ),
96
+ dataset: str = "dataset01",
97
+ ) -> None:
98
+ """Load test data into the database from the `test_data` directory."""
99
+
100
+ logging.getLogger("neo4j").setLevel(logging.ERROR)
101
+ config.load_and_exit(config_file_name=config_file)
102
+
103
+ context: CliContext = ctx.obj
104
+ dbdriver = await context.init_db(retry=1)
105
+
106
+ async with dbdriver.start_session() as db:
107
+ await initialization(db=db)
108
+
109
+ log_level = "DEBUG"
110
+
111
+ FORMAT = "%(message)s"
112
+ logging.basicConfig(level=log_level, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
113
+ logging.getLogger("infrahub")
114
+
115
+ dataset_module = importlib.import_module(f"infrahub.test_data.{dataset}")
116
+ await dataset_module.load_data(db=db)
117
+
118
+ await dbdriver.close()
infrahub/cli/tasks.py CHANGED
@@ -3,9 +3,11 @@ import logging
3
3
  import typer
4
4
  from infrahub_sdk.async_typer import AsyncTyper
5
5
  from prefect.client.orchestration import get_client
6
+ from prefect.client.schemas.objects import StateType
6
7
 
7
8
  from infrahub import config
8
9
  from infrahub.services.adapters.workflow.worker import WorkflowWorkerExecution
10
+ from infrahub.task_manager.task import PrefectTask
9
11
  from infrahub.tasks.dummy import DUMMY_FLOW, DummyInput
10
12
  from infrahub.workflows.initialization import setup_task_manager
11
13
  from infrahub.workflows.models import WorkerPoolDefinition
@@ -50,3 +52,47 @@ async def execute(
50
52
  workflow=DUMMY_FLOW, parameters={"data": DummyInput(firstname="John", lastname="Doe")}
51
53
  ) # type: ignore[var-annotated]
52
54
  print(result)
55
+
56
+
57
+ flush_app = AsyncTyper()
58
+
59
+ app.add_typer(flush_app, name="flush")
60
+
61
+
62
+ @flush_app.command()
63
+ async def flow_runs(
64
+ ctx: typer.Context, # noqa: ARG001
65
+ config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
66
+ days_to_keep: int = 30,
67
+ batch_size: int = 100,
68
+ ) -> None:
69
+ """Flush old task runs"""
70
+ logging.getLogger("infrahub").setLevel(logging.WARNING)
71
+ logging.getLogger("neo4j").setLevel(logging.ERROR)
72
+ logging.getLogger("prefect").setLevel(logging.ERROR)
73
+
74
+ config.load_and_exit(config_file_name=config_file)
75
+
76
+ await PrefectTask.delete_flow_runs(
77
+ days_to_keep=days_to_keep,
78
+ batch_size=batch_size,
79
+ )
80
+
81
+
82
+ @flush_app.command()
83
+ async def stale_runs(
84
+ ctx: typer.Context, # noqa: ARG001
85
+ config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
86
+ days_to_keep: int = 2,
87
+ batch_size: int = 100,
88
+ ) -> None:
89
+ """Flush stale task runs"""
90
+ logging.getLogger("infrahub").setLevel(logging.WARNING)
91
+ logging.getLogger("neo4j").setLevel(logging.ERROR)
92
+ logging.getLogger("prefect").setLevel(logging.ERROR)
93
+
94
+ config.load_and_exit(config_file_name=config_file)
95
+
96
+ await PrefectTask.delete_flow_runs(
97
+ states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
98
+ )
infrahub/cli/upgrade.py CHANGED
@@ -11,10 +11,16 @@ from prefect.client.orchestration import get_client
11
11
  from rich import print as rprint
12
12
 
13
13
  from infrahub import config
14
- from infrahub.core.initialization import create_anonymous_role, create_default_account_groups, initialize_registry
14
+ from infrahub.core.initialization import (
15
+ create_anonymous_role,
16
+ create_default_account_groups,
17
+ get_root_node,
18
+ initialize_registry,
19
+ )
15
20
  from infrahub.core.manager import NodeManager
16
21
  from infrahub.core.protocols import CoreAccount, CoreObjectPermission
17
22
  from infrahub.dependencies.registry import build_component_registry
23
+ from infrahub.lock import initialize_lock
18
24
  from infrahub.menu.menu import default_menu
19
25
  from infrahub.menu.models import MenuDict
20
26
  from infrahub.menu.repository import MenuRepository
@@ -26,7 +32,13 @@ from infrahub.workflows.initialization import (
26
32
  setup_worker_pools,
27
33
  )
28
34
 
29
- from .db import initialize_internal_schema, migrate_database, update_core_schema
35
+ from .db import (
36
+ detect_migration_to_run,
37
+ initialize_internal_schema,
38
+ migrate_database,
39
+ trigger_rebase_branches,
40
+ update_core_schema,
41
+ )
30
42
 
31
43
  if TYPE_CHECKING:
32
44
  from infrahub.cli.context import CliContext
@@ -40,6 +52,7 @@ async def upgrade_cmd(
40
52
  ctx: typer.Context,
41
53
  config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
42
54
  check: bool = typer.Option(False, help="Check the state of the system without upgrading."),
55
+ rebase_branches: bool = typer.Option(False, help="Rebase and apply migrations to branches if required."),
43
56
  ) -> None:
44
57
  """Upgrade Infrahub to the latest version."""
45
58
 
@@ -54,9 +67,12 @@ async def upgrade_cmd(
54
67
  dbdriver = await context.init_db(retry=1)
55
68
 
56
69
  await initialize_registry(db=dbdriver)
70
+ initialize_lock()
57
71
 
58
72
  build_component_registry()
59
73
 
74
+ root_node = await get_root_node(db=dbdriver)
75
+
60
76
  # NOTE add step to validate if the database and the task manager are reachable
61
77
 
62
78
  # -------------------------------------------
@@ -67,7 +83,12 @@ async def upgrade_cmd(
67
83
  # Upgrade Infrahub Database and Schema
68
84
  # -------------------------------------------
69
85
 
70
- if not await migrate_database(db=dbdriver, initialize=False, check=check):
86
+ migrations = await detect_migration_to_run(current_graph_version=root_node.graph_version)
87
+ if check:
88
+ await dbdriver.close()
89
+ return
90
+
91
+ if not await migrate_database(db=dbdriver, initialize=False, migrations=migrations):
71
92
  # A migration failed, stop the upgrade process
72
93
  rprint("Upgrade cancelled due to migration failure.")
73
94
  await dbdriver.close()
@@ -91,6 +112,12 @@ async def upgrade_cmd(
91
112
  await setup_deployments(client=client)
92
113
  await trigger_configure_all()
93
114
 
115
+ # -------------------------------------------
116
+ # Perform branch rebase and apply migrations to them
117
+ # -------------------------------------------
118
+ if rebase_branches:
119
+ await trigger_rebase_branches(db=dbdriver)
120
+
94
121
  await dbdriver.close()
95
122
 
96
123
 
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  from typing import TYPE_CHECKING
4
4
 
5
+ from infrahub_sdk.exceptions import URLNotFoundError
5
6
  from infrahub_sdk.protocols import CoreTransformPython
6
7
  from infrahub_sdk.template import Jinja2Template
7
8
  from prefect import flow
@@ -104,7 +105,7 @@ async def process_transform(
104
105
  ) # type: ignore[misc]
105
106
 
106
107
  data = await client.query_gql_query(
107
- name=transform.query.peer.name.value,
108
+ name=transform.query.id,
108
109
  branch_name=branch_name,
109
110
  variables={"id": object_id},
110
111
  update_group=True,
@@ -177,12 +178,17 @@ async def computed_attribute_jinja2_update_value(
177
178
  log.debug(f"Ignoring to update {obj} with existing value on {attribute_name}={value}")
178
179
  return
179
180
 
180
- await client.execute_graphql(
181
- query=UPDATE_ATTRIBUTE,
182
- variables={"id": obj.node_id, "kind": node_kind, "attribute": attribute_name, "value": value},
183
- branch_name=branch_name,
184
- )
185
- log.info(f"Updating computed attribute {node_kind}.{attribute_name}='{value}' ({obj.node_id})")
181
+ try:
182
+ await client.execute_graphql(
183
+ query=UPDATE_ATTRIBUTE,
184
+ variables={"id": obj.node_id, "kind": node_kind, "attribute": attribute_name, "value": value},
185
+ branch_name=branch_name,
186
+ )
187
+ log.info(f"Updating computed attribute {node_kind}.{attribute_name}='{value}' ({obj.node_id})")
188
+ except URLNotFoundError:
189
+ log.warning(
190
+ f"Update of computed attribute {node_kind}.{attribute_name} failed for branch {branch_name} (not found)"
191
+ )
186
192
 
187
193
 
188
194
  @flow(
@@ -229,7 +235,13 @@ async def process_jinja2(
229
235
 
230
236
  for id_filter in computed_macro.node_filters:
231
237
  query = attribute_graphql.render_graphql_query(query_filter=id_filter, filter_id=object_id)
232
- response = await client.execute_graphql(query=query, branch_name=branch_name)
238
+ try:
239
+ response = await client.execute_graphql(query=query, branch_name=branch_name)
240
+ except URLNotFoundError:
241
+ log.warning(
242
+ f"Process computed attributes for {computed_attribute_kind}.{computed_attribute_name} failed for branch {branch_name} (not found)"
243
+ )
244
+ return
233
245
  output = attribute_graphql.parse_response(response=response)
234
246
  found.extend(output)
235
247
 
@@ -18,6 +18,7 @@ from infrahub.core.changelog.models import AttributeChangelog
18
18
  from infrahub.core.constants import NULL_VALUE, AttributeDBNodeType, BranchSupportType, RelationshipStatus
19
19
  from infrahub.core.property import FlagPropertyMixin, NodePropertyData, NodePropertyMixin
20
20
  from infrahub.core.query.attribute import (
21
+ AttributeClearNodePropertyQuery,
21
22
  AttributeGetQuery,
22
23
  AttributeUpdateFlagQuery,
23
24
  AttributeUpdateNodePropertyQuery,
@@ -491,6 +492,12 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
491
492
  )
492
493
  await query.execute(db=db)
493
494
 
495
+ if needs_clear:
496
+ query = await AttributeClearNodePropertyQuery.init(
497
+ db=db, attr=self, at=update_at, prop_name=prop_name, prop_id=database_prop_id
498
+ )
499
+ await query.execute(db=db)
500
+
494
501
  # set the to time on the previously active edge
495
502
  rel = current_attr_result.get(f"rel_{prop_name}")
496
503
  if rel and rel.get("branch") == branch.name:
@@ -581,7 +588,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
581
588
 
582
589
  return value
583
590
 
584
- async def from_graphql(self, data: dict, db: InfrahubDatabase) -> bool:
591
+ async def from_graphql(self, data: dict, db: InfrahubDatabase, process_pools: bool = True) -> bool:
585
592
  """Update attr from GraphQL payload"""
586
593
 
587
594
  changed = False
@@ -595,7 +602,8 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
595
602
  changed = True
596
603
  elif "from_pool" in data:
597
604
  self.from_pool = data["from_pool"]
598
- await self.node.handle_pool(db=db, attribute=self, errors=[])
605
+ if process_pools:
606
+ await self.node.handle_pool(db=db, attribute=self, errors=[])
599
607
  changed = True
600
608
 
601
609
  if changed and self.is_from_profile:
@@ -4,5 +4,5 @@ from infrahub.utils import InfrahubStringEnum
4
4
  class BranchStatus(InfrahubStringEnum):
5
5
  OPEN = "OPEN"
6
6
  NEED_REBASE = "NEED_REBASE"
7
- CLOSED = "CLOSED"
7
+ NEED_UPGRADE_REBASE = "NEED_UPGRADE_REBASE"
8
8
  DELETING = "DELETING"
@@ -6,9 +6,8 @@ from typing import TYPE_CHECKING, Any, Optional, Self, Union
6
6
  from pydantic import Field, field_validator
7
7
 
8
8
  from infrahub.core.branch.enums import BranchStatus
9
- from infrahub.core.constants import (
10
- GLOBAL_BRANCH_NAME,
11
- )
9
+ from infrahub.core.constants import GLOBAL_BRANCH_NAME
10
+ from infrahub.core.graph import GRAPH_VERSION
12
11
  from infrahub.core.models import SchemaBranchHash # noqa: TC001
13
12
  from infrahub.core.node.standard import StandardNode
14
13
  from infrahub.core.query import QueryType
@@ -46,6 +45,7 @@ class Branch(StandardNode):
46
45
  is_isolated: bool = True
47
46
  schema_changed_at: Optional[str] = None
48
47
  schema_hash: Optional[SchemaBranchHash] = None
48
+ graph_version: int | None = None
49
49
 
50
50
  _exclude_attrs: list[str] = ["id", "uuid", "owner"]
51
51
 
@@ -261,6 +261,10 @@ class Branch(StandardNode):
261
261
 
262
262
  return start, end
263
263
 
264
+ async def create(self, db: InfrahubDatabase) -> bool:
265
+ self.graph_version = GRAPH_VERSION
266
+ return await super().create(db=db)
267
+
264
268
  async def delete(self, db: InfrahubDatabase) -> None:
265
269
  if self.is_default:
266
270
  raise ValidationError(f"Unable to delete {self.name} it is the default branch.")