infrahub-server 1.5.0b1__py3-none-any.whl → 1.5.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/api/internal.py +2 -0
- infrahub/api/oauth2.py +13 -19
- infrahub/api/oidc.py +15 -21
- infrahub/api/schema.py +24 -3
- infrahub/artifacts/models.py +2 -1
- infrahub/auth.py +137 -3
- infrahub/cli/__init__.py +2 -0
- infrahub/cli/db.py +83 -102
- infrahub/cli/dev.py +118 -0
- infrahub/cli/tasks.py +46 -0
- infrahub/cli/upgrade.py +30 -3
- infrahub/computed_attribute/tasks.py +20 -8
- infrahub/core/attribute.py +10 -2
- infrahub/core/branch/enums.py +1 -1
- infrahub/core/branch/models.py +7 -3
- infrahub/core/branch/tasks.py +68 -7
- infrahub/core/constants/__init__.py +3 -0
- infrahub/core/diff/query/artifact.py +1 -0
- infrahub/core/diff/query/field_summary.py +1 -0
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/initialization.py +5 -2
- infrahub/core/migrations/__init__.py +3 -0
- infrahub/core/migrations/exceptions.py +4 -0
- infrahub/core/migrations/graph/__init__.py +10 -13
- infrahub/core/migrations/graph/load_schema_branch.py +21 -0
- infrahub/core/migrations/graph/m013_convert_git_password_credential.py +1 -1
- infrahub/core/migrations/graph/m040_duplicated_attributes.py +81 -0
- infrahub/core/migrations/graph/m041_profile_attrs_in_db.py +145 -0
- infrahub/core/migrations/graph/m042_create_hfid_display_label_in_db.py +164 -0
- infrahub/core/migrations/graph/m043_backfill_hfid_display_label_in_db.py +866 -0
- infrahub/core/migrations/query/__init__.py +7 -8
- infrahub/core/migrations/query/attribute_add.py +8 -6
- infrahub/core/migrations/query/attribute_remove.py +134 -0
- infrahub/core/migrations/runner.py +54 -0
- infrahub/core/migrations/schema/attribute_kind_update.py +9 -3
- infrahub/core/migrations/schema/attribute_supports_profile.py +90 -0
- infrahub/core/migrations/schema/node_attribute_add.py +30 -2
- infrahub/core/migrations/schema/node_attribute_remove.py +13 -109
- infrahub/core/migrations/schema/node_kind_update.py +2 -1
- infrahub/core/migrations/schema/node_remove.py +2 -1
- infrahub/core/migrations/schema/placeholder_dummy.py +3 -2
- infrahub/core/migrations/shared.py +48 -14
- infrahub/core/node/__init__.py +16 -11
- infrahub/core/node/create.py +46 -63
- infrahub/core/node/lock_utils.py +70 -44
- infrahub/core/node/resource_manager/ip_address_pool.py +2 -1
- infrahub/core/node/resource_manager/ip_prefix_pool.py +2 -1
- infrahub/core/node/resource_manager/number_pool.py +2 -1
- infrahub/core/query/attribute.py +55 -0
- infrahub/core/query/ipam.py +1 -0
- infrahub/core/query/node.py +9 -3
- infrahub/core/query/relationship.py +1 -0
- infrahub/core/schema/__init__.py +56 -0
- infrahub/core/schema/attribute_schema.py +4 -0
- infrahub/core/schema/definitions/internal.py +2 -2
- infrahub/core/schema/generated/attribute_schema.py +2 -2
- infrahub/core/schema/manager.py +22 -1
- infrahub/core/schema/schema_branch.py +180 -22
- infrahub/database/graph.py +21 -0
- infrahub/display_labels/tasks.py +13 -7
- infrahub/events/branch_action.py +27 -1
- infrahub/generators/tasks.py +3 -7
- infrahub/git/base.py +4 -1
- infrahub/git/integrator.py +1 -1
- infrahub/git/models.py +2 -1
- infrahub/git/repository.py +22 -5
- infrahub/git/tasks.py +66 -10
- infrahub/git/utils.py +123 -1
- infrahub/graphql/api/endpoints.py +14 -4
- infrahub/graphql/manager.py +4 -9
- infrahub/graphql/mutations/convert_object_type.py +11 -1
- infrahub/graphql/mutations/display_label.py +17 -10
- infrahub/graphql/mutations/hfid.py +17 -10
- infrahub/graphql/mutations/ipam.py +54 -35
- infrahub/graphql/mutations/main.py +27 -28
- infrahub/graphql/schema_sort.py +170 -0
- infrahub/graphql/types/branch.py +4 -1
- infrahub/graphql/types/enums.py +3 -0
- infrahub/hfid/tasks.py +13 -7
- infrahub/lock.py +52 -12
- infrahub/message_bus/types.py +2 -1
- infrahub/permissions/constants.py +2 -0
- infrahub/proposed_change/tasks.py +25 -16
- infrahub/server.py +6 -2
- infrahub/services/__init__.py +2 -2
- infrahub/services/adapters/http/__init__.py +5 -0
- infrahub/services/adapters/workflow/worker.py +14 -3
- infrahub/task_manager/event.py +5 -0
- infrahub/task_manager/models.py +7 -0
- infrahub/task_manager/task.py +73 -0
- infrahub/trigger/setup.py +13 -4
- infrahub/trigger/tasks.py +3 -0
- infrahub/workers/dependencies.py +10 -1
- infrahub/workers/infrahub_async.py +10 -2
- infrahub/workflows/catalogue.py +8 -0
- infrahub/workflows/initialization.py +5 -0
- infrahub/workflows/utils.py +2 -1
- infrahub_sdk/client.py +13 -10
- infrahub_sdk/config.py +29 -2
- infrahub_sdk/ctl/schema.py +22 -7
- infrahub_sdk/schema/__init__.py +32 -4
- infrahub_sdk/spec/models.py +7 -0
- infrahub_sdk/spec/object.py +37 -102
- infrahub_sdk/spec/processors/__init__.py +0 -0
- infrahub_sdk/spec/processors/data_processor.py +10 -0
- infrahub_sdk/spec/processors/factory.py +34 -0
- infrahub_sdk/spec/processors/range_expand_processor.py +56 -0
- {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/METADATA +3 -1
- {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/RECORD +115 -101
- infrahub_testcontainers/container.py +114 -2
- infrahub_testcontainers/docker-compose-cluster.test.yml +5 -0
- infrahub_testcontainers/docker-compose.test.yml +5 -0
- infrahub/core/migrations/graph/m040_profile_attrs_in_db.py +0 -166
- infrahub/core/migrations/graph/m041_create_hfid_display_label_in_db.py +0 -97
- infrahub/core/migrations/graph/m042_backfill_hfid_display_label_in_db.py +0 -86
- {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/WHEEL +0 -0
- {infrahub_server-1.5.0b1.dist-info → infrahub_server-1.5.0b2.dist-info}/entry_points.txt +0 -0
infrahub/cli/db.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import importlib
|
|
4
3
|
import logging
|
|
5
4
|
import os
|
|
6
5
|
from collections import defaultdict
|
|
@@ -16,11 +15,15 @@ from infrahub_sdk.async_typer import AsyncTyper
|
|
|
16
15
|
from prefect.testing.utilities import prefect_test_harness
|
|
17
16
|
from rich import print as rprint
|
|
18
17
|
from rich.console import Console
|
|
19
|
-
from rich.logging import RichHandler
|
|
20
18
|
from rich.table import Table
|
|
21
19
|
|
|
22
20
|
from infrahub import config
|
|
21
|
+
from infrahub.auth import AccountSession, AuthType
|
|
22
|
+
from infrahub.context import InfrahubContext
|
|
23
23
|
from infrahub.core import registry
|
|
24
|
+
from infrahub.core.branch import Branch
|
|
25
|
+
from infrahub.core.branch.tasks import rebase_branch
|
|
26
|
+
from infrahub.core.constants import GLOBAL_BRANCH_NAME
|
|
24
27
|
from infrahub.core.graph import GRAPH_VERSION
|
|
25
28
|
from infrahub.core.graph.constraints import ConstraintManagerBase, ConstraintManagerMemgraph, ConstraintManagerNeo4j
|
|
26
29
|
from infrahub.core.graph.index import node_indexes, rel_indexes
|
|
@@ -32,25 +35,20 @@ from infrahub.core.graph.schema import (
|
|
|
32
35
|
GraphRelationshipIsPartOf,
|
|
33
36
|
GraphRelationshipProperties,
|
|
34
37
|
)
|
|
35
|
-
from infrahub.core.initialization import
|
|
36
|
-
|
|
37
|
-
get_root_node,
|
|
38
|
-
initialization,
|
|
39
|
-
initialize_registry,
|
|
40
|
-
)
|
|
38
|
+
from infrahub.core.initialization import get_root_node, initialize_registry
|
|
39
|
+
from infrahub.core.migrations.exceptions import MigrationFailureError
|
|
41
40
|
from infrahub.core.migrations.graph import get_graph_migrations, get_migration_by_number
|
|
42
41
|
from infrahub.core.migrations.schema.models import SchemaApplyMigrationData
|
|
43
42
|
from infrahub.core.migrations.schema.tasks import schema_apply_migrations
|
|
44
43
|
from infrahub.core.schema import SchemaRoot, core_models, internal_schema
|
|
45
44
|
from infrahub.core.schema.definitions.deprecated import deprecated_models
|
|
46
45
|
from infrahub.core.schema.manager import SchemaManager
|
|
47
|
-
from infrahub.core.utils import delete_all_nodes
|
|
48
46
|
from infrahub.core.validators.models.validate_migration import SchemaValidateMigrationData
|
|
49
47
|
from infrahub.core.validators.tasks import schema_validate_migrations
|
|
50
48
|
from infrahub.database import DatabaseType
|
|
51
49
|
from infrahub.database.memgraph import IndexManagerMemgraph
|
|
52
50
|
from infrahub.database.neo4j import IndexManagerNeo4j
|
|
53
|
-
from infrahub.
|
|
51
|
+
from infrahub.exceptions import ValidationError
|
|
54
52
|
|
|
55
53
|
from .constants import ERROR_BADGE, FAILED_BADGE, SUCCESS_BADGE
|
|
56
54
|
from .db_commands.check_inheritance import check_inheritance
|
|
@@ -65,7 +63,7 @@ def get_timestamp_string() -> str:
|
|
|
65
63
|
|
|
66
64
|
if TYPE_CHECKING:
|
|
67
65
|
from infrahub.cli.context import CliContext
|
|
68
|
-
from infrahub.core.migrations.shared import
|
|
66
|
+
from infrahub.core.migrations.shared import MigrationTypes
|
|
69
67
|
from infrahub.database import InfrahubDatabase
|
|
70
68
|
from infrahub.database.index import IndexManagerBase
|
|
71
69
|
|
|
@@ -94,67 +92,6 @@ def callback() -> None:
|
|
|
94
92
|
"""
|
|
95
93
|
|
|
96
94
|
|
|
97
|
-
@app.command()
|
|
98
|
-
async def init(
|
|
99
|
-
ctx: typer.Context,
|
|
100
|
-
config_file: str = typer.Option(
|
|
101
|
-
"infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
|
|
102
|
-
),
|
|
103
|
-
) -> None:
|
|
104
|
-
"""Erase the content of the database and initialize it with the core schema."""
|
|
105
|
-
|
|
106
|
-
log = get_logger()
|
|
107
|
-
|
|
108
|
-
# --------------------------------------------------
|
|
109
|
-
# CLEANUP
|
|
110
|
-
# - For now we delete everything in the database
|
|
111
|
-
# TODO, if possible try to implement this in an idempotent way
|
|
112
|
-
# --------------------------------------------------
|
|
113
|
-
|
|
114
|
-
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
115
|
-
config.load_and_exit(config_file_name=config_file)
|
|
116
|
-
|
|
117
|
-
context: CliContext = ctx.obj
|
|
118
|
-
dbdriver = await context.init_db(retry=1)
|
|
119
|
-
async with dbdriver.start_transaction() as db:
|
|
120
|
-
log.info("Delete All Nodes")
|
|
121
|
-
await delete_all_nodes(db=db)
|
|
122
|
-
await first_time_initialization(db=db)
|
|
123
|
-
|
|
124
|
-
await dbdriver.close()
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
@app.command()
|
|
128
|
-
async def load_test_data(
|
|
129
|
-
ctx: typer.Context,
|
|
130
|
-
config_file: str = typer.Option(
|
|
131
|
-
"infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
|
|
132
|
-
),
|
|
133
|
-
dataset: str = "dataset01",
|
|
134
|
-
) -> None:
|
|
135
|
-
"""Load test data into the database from the `test_data` directory."""
|
|
136
|
-
|
|
137
|
-
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
138
|
-
config.load_and_exit(config_file_name=config_file)
|
|
139
|
-
|
|
140
|
-
context: CliContext = ctx.obj
|
|
141
|
-
dbdriver = await context.init_db(retry=1)
|
|
142
|
-
|
|
143
|
-
async with dbdriver.start_session() as db:
|
|
144
|
-
await initialization(db=db)
|
|
145
|
-
|
|
146
|
-
log_level = "DEBUG"
|
|
147
|
-
|
|
148
|
-
FORMAT = "%(message)s"
|
|
149
|
-
logging.basicConfig(level=log_level, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
|
|
150
|
-
logging.getLogger("infrahub")
|
|
151
|
-
|
|
152
|
-
dataset_module = importlib.import_module(f"infrahub.test_data.{dataset}")
|
|
153
|
-
await dataset_module.load_data(db=db)
|
|
154
|
-
|
|
155
|
-
await dbdriver.close()
|
|
156
|
-
|
|
157
|
-
|
|
158
95
|
@app.command(name="migrate")
|
|
159
96
|
async def migrate_cmd(
|
|
160
97
|
ctx: typer.Context,
|
|
@@ -172,7 +109,15 @@ async def migrate_cmd(
|
|
|
172
109
|
context: CliContext = ctx.obj
|
|
173
110
|
dbdriver = await context.init_db(retry=1)
|
|
174
111
|
|
|
175
|
-
await
|
|
112
|
+
root_node = await get_root_node(db=dbdriver)
|
|
113
|
+
migrations = await detect_migration_to_run(
|
|
114
|
+
current_graph_version=root_node.graph_version, migration_number=migration_number
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
if check or not migrations:
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
await migrate_database(db=dbdriver, migrations=migrations, initialize=True)
|
|
176
121
|
|
|
177
122
|
await dbdriver.close()
|
|
178
123
|
|
|
@@ -335,8 +280,38 @@ async def index(
|
|
|
335
280
|
await dbdriver.close()
|
|
336
281
|
|
|
337
282
|
|
|
283
|
+
async def detect_migration_to_run(
|
|
284
|
+
current_graph_version: int, migration_number: int | str | None = None
|
|
285
|
+
) -> Sequence[MigrationTypes]:
|
|
286
|
+
"""Return a sequence of migrations to apply to upgrade the database."""
|
|
287
|
+
rprint("Checking current state of the database")
|
|
288
|
+
migrations: list[MigrationTypes] = []
|
|
289
|
+
|
|
290
|
+
if migration_number:
|
|
291
|
+
migration = get_migration_by_number(migration_number)
|
|
292
|
+
migrations.append(migration)
|
|
293
|
+
if current_graph_version > migration.minimum_version:
|
|
294
|
+
rprint(
|
|
295
|
+
f"Migration {migration_number} already applied. To apply again, run the command without the --check flag."
|
|
296
|
+
)
|
|
297
|
+
return []
|
|
298
|
+
rprint(
|
|
299
|
+
f"Migration {migration_number} needs to be applied. Run `infrahub db migrate` to apply all outstanding migrations."
|
|
300
|
+
)
|
|
301
|
+
else:
|
|
302
|
+
migrations.extend(await get_graph_migrations(current_graph_version=current_graph_version))
|
|
303
|
+
if not migrations:
|
|
304
|
+
rprint(f"Database up-to-date (v{current_graph_version}), no migration to execute.")
|
|
305
|
+
return []
|
|
306
|
+
|
|
307
|
+
rprint(
|
|
308
|
+
f"Database needs to be updated (v{current_graph_version} -> v{GRAPH_VERSION}), {len(migrations)} migrations pending"
|
|
309
|
+
)
|
|
310
|
+
return migrations
|
|
311
|
+
|
|
312
|
+
|
|
338
313
|
async def migrate_database(
|
|
339
|
-
db: InfrahubDatabase,
|
|
314
|
+
db: InfrahubDatabase, migrations: Sequence[MigrationTypes], initialize: bool = False
|
|
340
315
|
) -> bool:
|
|
341
316
|
"""Apply the latest migrations to the database, this function will print the status directly in the console.
|
|
342
317
|
|
|
@@ -344,40 +319,16 @@ async def migrate_database(
|
|
|
344
319
|
|
|
345
320
|
Args:
|
|
346
321
|
db: The database object.
|
|
347
|
-
|
|
348
|
-
|
|
322
|
+
migrations: Sequence of migrations to apply.
|
|
323
|
+
initialize: Whether to initialize the registry before running migrations.
|
|
349
324
|
"""
|
|
350
|
-
|
|
325
|
+
if not migrations:
|
|
326
|
+
return True
|
|
351
327
|
|
|
352
328
|
if initialize:
|
|
353
329
|
await initialize_registry(db=db)
|
|
354
330
|
|
|
355
331
|
root_node = await get_root_node(db=db)
|
|
356
|
-
if migration_number:
|
|
357
|
-
migration = get_migration_by_number(migration_number)
|
|
358
|
-
migrations: Sequence[GraphMigration | InternalSchemaMigration | ArbitraryMigration] = [migration]
|
|
359
|
-
if check:
|
|
360
|
-
if root_node.graph_version > migration.minimum_version:
|
|
361
|
-
rprint(
|
|
362
|
-
f"Migration {migration_number} already applied. To apply again, run the command without the --check flag."
|
|
363
|
-
)
|
|
364
|
-
return True
|
|
365
|
-
rprint(
|
|
366
|
-
f"Migration {migration_number} needs to be applied. Run `infrahub db migrate` to apply all outstanding migrations."
|
|
367
|
-
)
|
|
368
|
-
return False
|
|
369
|
-
else:
|
|
370
|
-
migrations = await get_graph_migrations(root=root_node)
|
|
371
|
-
if not migrations:
|
|
372
|
-
rprint(f"Database up-to-date (v{root_node.graph_version}), no migration to execute.")
|
|
373
|
-
return True
|
|
374
|
-
|
|
375
|
-
rprint(
|
|
376
|
-
f"Database needs to be updated (v{root_node.graph_version} -> v{GRAPH_VERSION}), {len(migrations)} migrations pending"
|
|
377
|
-
)
|
|
378
|
-
|
|
379
|
-
if check:
|
|
380
|
-
return True
|
|
381
332
|
|
|
382
333
|
for migration in migrations:
|
|
383
334
|
execution_result = await migration.execute(db=db)
|
|
@@ -402,6 +353,36 @@ async def migrate_database(
|
|
|
402
353
|
return True
|
|
403
354
|
|
|
404
355
|
|
|
356
|
+
async def trigger_rebase_branches(db: InfrahubDatabase) -> None:
|
|
357
|
+
"""Trigger rebase of non-default branches, also triggering migrations in the process."""
|
|
358
|
+
branches = [b for b in await Branch.get_list(db=db) if b.name not in [registry.default_branch, GLOBAL_BRANCH_NAME]]
|
|
359
|
+
if not branches:
|
|
360
|
+
return
|
|
361
|
+
|
|
362
|
+
rprint(f"Planning rebase and migrations for {len(branches)} branches: {', '.join([b.name for b in branches])}")
|
|
363
|
+
|
|
364
|
+
for branch in branches:
|
|
365
|
+
if branch.graph_version == GRAPH_VERSION:
|
|
366
|
+
rprint(
|
|
367
|
+
f"Ignoring branch rebase and migrations for '{branch.name}' (ID: {branch.uuid}), it is already up-to-date"
|
|
368
|
+
)
|
|
369
|
+
continue
|
|
370
|
+
|
|
371
|
+
rprint(f"Rebasing branch '{branch.name}' (ID: {branch.uuid})...", end="")
|
|
372
|
+
try:
|
|
373
|
+
await registry.schema.load_schema(db=db, branch=branch)
|
|
374
|
+
await rebase_branch(
|
|
375
|
+
branch=branch.name,
|
|
376
|
+
context=InfrahubContext.init(
|
|
377
|
+
branch=branch, account=AccountSession(auth_type=AuthType.NONE, authenticated=False, account_id="")
|
|
378
|
+
),
|
|
379
|
+
send_events=False,
|
|
380
|
+
)
|
|
381
|
+
rprint(SUCCESS_BADGE)
|
|
382
|
+
except (ValidationError, MigrationFailureError):
|
|
383
|
+
rprint(FAILED_BADGE)
|
|
384
|
+
|
|
385
|
+
|
|
405
386
|
async def initialize_internal_schema() -> None:
|
|
406
387
|
registry.schema = SchemaManager()
|
|
407
388
|
schema = SchemaRoot(**internal_schema)
|
infrahub/cli/dev.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import logging
|
|
5
|
+
from pathlib import Path # noqa: TC003
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from graphql import parse, print_ast, print_schema
|
|
10
|
+
from infrahub_sdk.async_typer import AsyncTyper
|
|
11
|
+
from rich.logging import RichHandler
|
|
12
|
+
|
|
13
|
+
from infrahub import config
|
|
14
|
+
from infrahub.core.initialization import (
|
|
15
|
+
first_time_initialization,
|
|
16
|
+
initialization,
|
|
17
|
+
)
|
|
18
|
+
from infrahub.core.schema import SchemaRoot, core_models, internal_schema
|
|
19
|
+
from infrahub.core.schema.schema_branch import SchemaBranch
|
|
20
|
+
from infrahub.core.utils import delete_all_nodes
|
|
21
|
+
from infrahub.graphql.manager import GraphQLSchemaManager
|
|
22
|
+
from infrahub.graphql.schema_sort import sort_schema_ast
|
|
23
|
+
from infrahub.log import get_logger
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from infrahub.cli.context import CliContext
|
|
27
|
+
|
|
28
|
+
app = AsyncTyper()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@app.command(name="export-graphql-schema")
|
|
32
|
+
async def export_graphql_schema(
|
|
33
|
+
ctx: typer.Context, # noqa: ARG001
|
|
34
|
+
config_file: str = typer.Option("infrahub.toml", envvar="INFRAHUB_CONFIG"),
|
|
35
|
+
out: Path = typer.Option("schema.graphql"), # noqa: B008
|
|
36
|
+
) -> None:
|
|
37
|
+
"""Export the Core GraphQL schema to a file."""
|
|
38
|
+
|
|
39
|
+
config.load_and_exit(config_file_name=config_file)
|
|
40
|
+
|
|
41
|
+
schema = SchemaRoot(**internal_schema)
|
|
42
|
+
full_schema = schema.merge(schema=SchemaRoot(**core_models))
|
|
43
|
+
|
|
44
|
+
schema_branch = SchemaBranch(cache={}, name="default")
|
|
45
|
+
schema_branch.load_schema(schema=full_schema)
|
|
46
|
+
|
|
47
|
+
schema_branch.process()
|
|
48
|
+
|
|
49
|
+
gqlm = GraphQLSchemaManager(schema=schema_branch)
|
|
50
|
+
gql_schema = gqlm.generate()
|
|
51
|
+
|
|
52
|
+
schema_str = print_schema(gql_schema)
|
|
53
|
+
schema_ast = parse(schema_str)
|
|
54
|
+
sorted_schema_ast = sort_schema_ast(schema_ast)
|
|
55
|
+
sorted_schema_str = print_ast(sorted_schema_ast)
|
|
56
|
+
|
|
57
|
+
out.write_text(sorted_schema_str)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@app.command(name="db-init")
|
|
61
|
+
async def database_init(
|
|
62
|
+
ctx: typer.Context,
|
|
63
|
+
config_file: str = typer.Option(
|
|
64
|
+
"infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
|
|
65
|
+
),
|
|
66
|
+
) -> None:
|
|
67
|
+
"""Erase the content of the database and initialize it with the core schema."""
|
|
68
|
+
|
|
69
|
+
log = get_logger()
|
|
70
|
+
|
|
71
|
+
# --------------------------------------------------
|
|
72
|
+
# CLEANUP
|
|
73
|
+
# - For now we delete everything in the database
|
|
74
|
+
# TODO, if possible try to implement this in an idempotent way
|
|
75
|
+
# --------------------------------------------------
|
|
76
|
+
|
|
77
|
+
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
78
|
+
config.load_and_exit(config_file_name=config_file)
|
|
79
|
+
|
|
80
|
+
context: CliContext = ctx.obj
|
|
81
|
+
dbdriver = await context.init_db(retry=1)
|
|
82
|
+
async with dbdriver.start_transaction() as db:
|
|
83
|
+
log.info("Delete All Nodes")
|
|
84
|
+
await delete_all_nodes(db=db)
|
|
85
|
+
await first_time_initialization(db=db)
|
|
86
|
+
|
|
87
|
+
await dbdriver.close()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@app.command(name="load-test-data")
|
|
91
|
+
async def load_test_data(
|
|
92
|
+
ctx: typer.Context,
|
|
93
|
+
config_file: str = typer.Option(
|
|
94
|
+
"infrahub.toml", envvar="INFRAHUB_CONFIG", help="Location of the configuration file to use for Infrahub"
|
|
95
|
+
),
|
|
96
|
+
dataset: str = "dataset01",
|
|
97
|
+
) -> None:
|
|
98
|
+
"""Load test data into the database from the `test_data` directory."""
|
|
99
|
+
|
|
100
|
+
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
101
|
+
config.load_and_exit(config_file_name=config_file)
|
|
102
|
+
|
|
103
|
+
context: CliContext = ctx.obj
|
|
104
|
+
dbdriver = await context.init_db(retry=1)
|
|
105
|
+
|
|
106
|
+
async with dbdriver.start_session() as db:
|
|
107
|
+
await initialization(db=db)
|
|
108
|
+
|
|
109
|
+
log_level = "DEBUG"
|
|
110
|
+
|
|
111
|
+
FORMAT = "%(message)s"
|
|
112
|
+
logging.basicConfig(level=log_level, format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
|
|
113
|
+
logging.getLogger("infrahub")
|
|
114
|
+
|
|
115
|
+
dataset_module = importlib.import_module(f"infrahub.test_data.{dataset}")
|
|
116
|
+
await dataset_module.load_data(db=db)
|
|
117
|
+
|
|
118
|
+
await dbdriver.close()
|
infrahub/cli/tasks.py
CHANGED
|
@@ -3,9 +3,11 @@ import logging
|
|
|
3
3
|
import typer
|
|
4
4
|
from infrahub_sdk.async_typer import AsyncTyper
|
|
5
5
|
from prefect.client.orchestration import get_client
|
|
6
|
+
from prefect.client.schemas.objects import StateType
|
|
6
7
|
|
|
7
8
|
from infrahub import config
|
|
8
9
|
from infrahub.services.adapters.workflow.worker import WorkflowWorkerExecution
|
|
10
|
+
from infrahub.task_manager.task import PrefectTask
|
|
9
11
|
from infrahub.tasks.dummy import DUMMY_FLOW, DummyInput
|
|
10
12
|
from infrahub.workflows.initialization import setup_task_manager
|
|
11
13
|
from infrahub.workflows.models import WorkerPoolDefinition
|
|
@@ -50,3 +52,47 @@ async def execute(
|
|
|
50
52
|
workflow=DUMMY_FLOW, parameters={"data": DummyInput(firstname="John", lastname="Doe")}
|
|
51
53
|
) # type: ignore[var-annotated]
|
|
52
54
|
print(result)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
flush_app = AsyncTyper()
|
|
58
|
+
|
|
59
|
+
app.add_typer(flush_app, name="flush")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@flush_app.command()
|
|
63
|
+
async def flow_runs(
|
|
64
|
+
ctx: typer.Context, # noqa: ARG001
|
|
65
|
+
config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
|
|
66
|
+
days_to_keep: int = 30,
|
|
67
|
+
batch_size: int = 100,
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Flush old task runs"""
|
|
70
|
+
logging.getLogger("infrahub").setLevel(logging.WARNING)
|
|
71
|
+
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
72
|
+
logging.getLogger("prefect").setLevel(logging.ERROR)
|
|
73
|
+
|
|
74
|
+
config.load_and_exit(config_file_name=config_file)
|
|
75
|
+
|
|
76
|
+
await PrefectTask.delete_flow_runs(
|
|
77
|
+
days_to_keep=days_to_keep,
|
|
78
|
+
batch_size=batch_size,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@flush_app.command()
|
|
83
|
+
async def stale_runs(
|
|
84
|
+
ctx: typer.Context, # noqa: ARG001
|
|
85
|
+
config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
|
|
86
|
+
days_to_keep: int = 2,
|
|
87
|
+
batch_size: int = 100,
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Flush stale task runs"""
|
|
90
|
+
logging.getLogger("infrahub").setLevel(logging.WARNING)
|
|
91
|
+
logging.getLogger("neo4j").setLevel(logging.ERROR)
|
|
92
|
+
logging.getLogger("prefect").setLevel(logging.ERROR)
|
|
93
|
+
|
|
94
|
+
config.load_and_exit(config_file_name=config_file)
|
|
95
|
+
|
|
96
|
+
await PrefectTask.delete_flow_runs(
|
|
97
|
+
states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
|
|
98
|
+
)
|
infrahub/cli/upgrade.py
CHANGED
|
@@ -11,10 +11,16 @@ from prefect.client.orchestration import get_client
|
|
|
11
11
|
from rich import print as rprint
|
|
12
12
|
|
|
13
13
|
from infrahub import config
|
|
14
|
-
from infrahub.core.initialization import
|
|
14
|
+
from infrahub.core.initialization import (
|
|
15
|
+
create_anonymous_role,
|
|
16
|
+
create_default_account_groups,
|
|
17
|
+
get_root_node,
|
|
18
|
+
initialize_registry,
|
|
19
|
+
)
|
|
15
20
|
from infrahub.core.manager import NodeManager
|
|
16
21
|
from infrahub.core.protocols import CoreAccount, CoreObjectPermission
|
|
17
22
|
from infrahub.dependencies.registry import build_component_registry
|
|
23
|
+
from infrahub.lock import initialize_lock
|
|
18
24
|
from infrahub.menu.menu import default_menu
|
|
19
25
|
from infrahub.menu.models import MenuDict
|
|
20
26
|
from infrahub.menu.repository import MenuRepository
|
|
@@ -26,7 +32,13 @@ from infrahub.workflows.initialization import (
|
|
|
26
32
|
setup_worker_pools,
|
|
27
33
|
)
|
|
28
34
|
|
|
29
|
-
from .db import
|
|
35
|
+
from .db import (
|
|
36
|
+
detect_migration_to_run,
|
|
37
|
+
initialize_internal_schema,
|
|
38
|
+
migrate_database,
|
|
39
|
+
trigger_rebase_branches,
|
|
40
|
+
update_core_schema,
|
|
41
|
+
)
|
|
30
42
|
|
|
31
43
|
if TYPE_CHECKING:
|
|
32
44
|
from infrahub.cli.context import CliContext
|
|
@@ -40,6 +52,7 @@ async def upgrade_cmd(
|
|
|
40
52
|
ctx: typer.Context,
|
|
41
53
|
config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
|
|
42
54
|
check: bool = typer.Option(False, help="Check the state of the system without upgrading."),
|
|
55
|
+
rebase_branches: bool = typer.Option(False, help="Rebase and apply migrations to branches if required."),
|
|
43
56
|
) -> None:
|
|
44
57
|
"""Upgrade Infrahub to the latest version."""
|
|
45
58
|
|
|
@@ -54,9 +67,12 @@ async def upgrade_cmd(
|
|
|
54
67
|
dbdriver = await context.init_db(retry=1)
|
|
55
68
|
|
|
56
69
|
await initialize_registry(db=dbdriver)
|
|
70
|
+
initialize_lock()
|
|
57
71
|
|
|
58
72
|
build_component_registry()
|
|
59
73
|
|
|
74
|
+
root_node = await get_root_node(db=dbdriver)
|
|
75
|
+
|
|
60
76
|
# NOTE add step to validate if the database and the task manager are reachable
|
|
61
77
|
|
|
62
78
|
# -------------------------------------------
|
|
@@ -67,7 +83,12 @@ async def upgrade_cmd(
|
|
|
67
83
|
# Upgrade Infrahub Database and Schema
|
|
68
84
|
# -------------------------------------------
|
|
69
85
|
|
|
70
|
-
|
|
86
|
+
migrations = await detect_migration_to_run(current_graph_version=root_node.graph_version)
|
|
87
|
+
if check:
|
|
88
|
+
await dbdriver.close()
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
if not await migrate_database(db=dbdriver, initialize=False, migrations=migrations):
|
|
71
92
|
# A migration failed, stop the upgrade process
|
|
72
93
|
rprint("Upgrade cancelled due to migration failure.")
|
|
73
94
|
await dbdriver.close()
|
|
@@ -91,6 +112,12 @@ async def upgrade_cmd(
|
|
|
91
112
|
await setup_deployments(client=client)
|
|
92
113
|
await trigger_configure_all()
|
|
93
114
|
|
|
115
|
+
# -------------------------------------------
|
|
116
|
+
# Perform branch rebase and apply migrations to them
|
|
117
|
+
# -------------------------------------------
|
|
118
|
+
if rebase_branches:
|
|
119
|
+
await trigger_rebase_branches(db=dbdriver)
|
|
120
|
+
|
|
94
121
|
await dbdriver.close()
|
|
95
122
|
|
|
96
123
|
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
4
|
|
|
5
|
+
from infrahub_sdk.exceptions import URLNotFoundError
|
|
5
6
|
from infrahub_sdk.protocols import CoreTransformPython
|
|
6
7
|
from infrahub_sdk.template import Jinja2Template
|
|
7
8
|
from prefect import flow
|
|
@@ -104,7 +105,7 @@ async def process_transform(
|
|
|
104
105
|
) # type: ignore[misc]
|
|
105
106
|
|
|
106
107
|
data = await client.query_gql_query(
|
|
107
|
-
name=transform.query.
|
|
108
|
+
name=transform.query.id,
|
|
108
109
|
branch_name=branch_name,
|
|
109
110
|
variables={"id": object_id},
|
|
110
111
|
update_group=True,
|
|
@@ -177,12 +178,17 @@ async def computed_attribute_jinja2_update_value(
|
|
|
177
178
|
log.debug(f"Ignoring to update {obj} with existing value on {attribute_name}={value}")
|
|
178
179
|
return
|
|
179
180
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
181
|
+
try:
|
|
182
|
+
await client.execute_graphql(
|
|
183
|
+
query=UPDATE_ATTRIBUTE,
|
|
184
|
+
variables={"id": obj.node_id, "kind": node_kind, "attribute": attribute_name, "value": value},
|
|
185
|
+
branch_name=branch_name,
|
|
186
|
+
)
|
|
187
|
+
log.info(f"Updating computed attribute {node_kind}.{attribute_name}='{value}' ({obj.node_id})")
|
|
188
|
+
except URLNotFoundError:
|
|
189
|
+
log.warning(
|
|
190
|
+
f"Update of computed attribute {node_kind}.{attribute_name} failed for branch {branch_name} (not found)"
|
|
191
|
+
)
|
|
186
192
|
|
|
187
193
|
|
|
188
194
|
@flow(
|
|
@@ -229,7 +235,13 @@ async def process_jinja2(
|
|
|
229
235
|
|
|
230
236
|
for id_filter in computed_macro.node_filters:
|
|
231
237
|
query = attribute_graphql.render_graphql_query(query_filter=id_filter, filter_id=object_id)
|
|
232
|
-
|
|
238
|
+
try:
|
|
239
|
+
response = await client.execute_graphql(query=query, branch_name=branch_name)
|
|
240
|
+
except URLNotFoundError:
|
|
241
|
+
log.warning(
|
|
242
|
+
f"Process computed attributes for {computed_attribute_kind}.{computed_attribute_name} failed for branch {branch_name} (not found)"
|
|
243
|
+
)
|
|
244
|
+
return
|
|
233
245
|
output = attribute_graphql.parse_response(response=response)
|
|
234
246
|
found.extend(output)
|
|
235
247
|
|
infrahub/core/attribute.py
CHANGED
|
@@ -18,6 +18,7 @@ from infrahub.core.changelog.models import AttributeChangelog
|
|
|
18
18
|
from infrahub.core.constants import NULL_VALUE, AttributeDBNodeType, BranchSupportType, RelationshipStatus
|
|
19
19
|
from infrahub.core.property import FlagPropertyMixin, NodePropertyData, NodePropertyMixin
|
|
20
20
|
from infrahub.core.query.attribute import (
|
|
21
|
+
AttributeClearNodePropertyQuery,
|
|
21
22
|
AttributeGetQuery,
|
|
22
23
|
AttributeUpdateFlagQuery,
|
|
23
24
|
AttributeUpdateNodePropertyQuery,
|
|
@@ -491,6 +492,12 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
|
|
|
491
492
|
)
|
|
492
493
|
await query.execute(db=db)
|
|
493
494
|
|
|
495
|
+
if needs_clear:
|
|
496
|
+
query = await AttributeClearNodePropertyQuery.init(
|
|
497
|
+
db=db, attr=self, at=update_at, prop_name=prop_name, prop_id=database_prop_id
|
|
498
|
+
)
|
|
499
|
+
await query.execute(db=db)
|
|
500
|
+
|
|
494
501
|
# set the to time on the previously active edge
|
|
495
502
|
rel = current_attr_result.get(f"rel_{prop_name}")
|
|
496
503
|
if rel and rel.get("branch") == branch.name:
|
|
@@ -581,7 +588,7 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
|
|
|
581
588
|
|
|
582
589
|
return value
|
|
583
590
|
|
|
584
|
-
async def from_graphql(self, data: dict, db: InfrahubDatabase) -> bool:
|
|
591
|
+
async def from_graphql(self, data: dict, db: InfrahubDatabase, process_pools: bool = True) -> bool:
|
|
585
592
|
"""Update attr from GraphQL payload"""
|
|
586
593
|
|
|
587
594
|
changed = False
|
|
@@ -595,7 +602,8 @@ class BaseAttribute(FlagPropertyMixin, NodePropertyMixin):
|
|
|
595
602
|
changed = True
|
|
596
603
|
elif "from_pool" in data:
|
|
597
604
|
self.from_pool = data["from_pool"]
|
|
598
|
-
|
|
605
|
+
if process_pools:
|
|
606
|
+
await self.node.handle_pool(db=db, attribute=self, errors=[])
|
|
599
607
|
changed = True
|
|
600
608
|
|
|
601
609
|
if changed and self.is_from_profile:
|
infrahub/core/branch/enums.py
CHANGED
infrahub/core/branch/models.py
CHANGED
|
@@ -6,9 +6,8 @@ from typing import TYPE_CHECKING, Any, Optional, Self, Union
|
|
|
6
6
|
from pydantic import Field, field_validator
|
|
7
7
|
|
|
8
8
|
from infrahub.core.branch.enums import BranchStatus
|
|
9
|
-
from infrahub.core.constants import
|
|
10
|
-
|
|
11
|
-
)
|
|
9
|
+
from infrahub.core.constants import GLOBAL_BRANCH_NAME
|
|
10
|
+
from infrahub.core.graph import GRAPH_VERSION
|
|
12
11
|
from infrahub.core.models import SchemaBranchHash # noqa: TC001
|
|
13
12
|
from infrahub.core.node.standard import StandardNode
|
|
14
13
|
from infrahub.core.query import QueryType
|
|
@@ -46,6 +45,7 @@ class Branch(StandardNode):
|
|
|
46
45
|
is_isolated: bool = True
|
|
47
46
|
schema_changed_at: Optional[str] = None
|
|
48
47
|
schema_hash: Optional[SchemaBranchHash] = None
|
|
48
|
+
graph_version: int | None = None
|
|
49
49
|
|
|
50
50
|
_exclude_attrs: list[str] = ["id", "uuid", "owner"]
|
|
51
51
|
|
|
@@ -261,6 +261,10 @@ class Branch(StandardNode):
|
|
|
261
261
|
|
|
262
262
|
return start, end
|
|
263
263
|
|
|
264
|
+
async def create(self, db: InfrahubDatabase) -> bool:
|
|
265
|
+
self.graph_version = GRAPH_VERSION
|
|
266
|
+
return await super().create(db=db)
|
|
267
|
+
|
|
264
268
|
async def delete(self, db: InfrahubDatabase) -> None:
|
|
265
269
|
if self.is_default:
|
|
266
270
|
raise ValidationError(f"Unable to delete {self.name} it is the default branch.")
|