infrahub-server 1.4.11__py3-none-any.whl → 1.4.13__py3-none-any.whl

infrahub/cli/tasks.py

@@ -3,9 +3,11 @@ import logging
 import typer
 from infrahub_sdk.async_typer import AsyncTyper
 from prefect.client.orchestration import get_client
+from prefect.client.schemas.objects import StateType
 
 from infrahub import config
 from infrahub.services.adapters.workflow.worker import WorkflowWorkerExecution
+from infrahub.task_manager.task import PrefectTask
 from infrahub.tasks.dummy import DUMMY_FLOW, DummyInput
 from infrahub.workflows.initialization import setup_task_manager
 from infrahub.workflows.models import WorkerPoolDefinition
@@ -50,3 +52,47 @@ async def execute(
             workflow=DUMMY_FLOW, parameters={"data": DummyInput(firstname="John", lastname="Doe")}
         )  # type: ignore[var-annotated]
         print(result)
+
+
+flush_app = AsyncTyper()
+
+app.add_typer(flush_app, name="flush")
+
+
+@flush_app.command()
+async def flow_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 30,
+    batch_size: int = 100,
+) -> None:
+    """Flush old task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        days_to_keep=days_to_keep,
+        batch_size=batch_size,
+    )
+
+
+@flush_app.command()
+async def stale_runs(
+    ctx: typer.Context,  # noqa: ARG001
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+    days_to_keep: int = 2,
+    batch_size: int = 100,
+) -> None:
+    """Flush stale task runs"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    await PrefectTask.delete_flow_runs(
+        states=[StateType.RUNNING], delete=False, days_to_keep=days_to_keep, batch_size=batch_size
+    )

infrahub/core/diff/calculator.py

@@ -141,8 +141,8 @@ class DiffCalculator:
             to_time=to_time,
             previous_node_field_specifiers=previous_node_specifiers,
         )
-        node_limit = int(config.SETTINGS.database.query_size_limit / 10)
-        fields_limit = int(config.SETTINGS.database.query_size_limit / 3)
+        node_limit = max(int(config.SETTINGS.database.query_size_limit / 10), 1)
+        fields_limit = max(int(config.SETTINGS.database.query_size_limit / 3), 1)
         properties_limit = config.SETTINGS.database.query_size_limit
 
         calculation_request = DiffCalculationRequest(

infrahub/core/diff/query/delete_query.py

@@ -20,10 +20,14 @@ class EnrichedDiffDeleteQuery(Query):
             diff_filter = "WHERE d_root.uuid IN $diff_root_uuids"
 
         query = """
-        MATCH (d_root:DiffRoot)
-        %(diff_filter)s
-        OPTIONAL MATCH (d_root)-[*]->(diff_thing)
-        DETACH DELETE diff_thing
-        DETACH DELETE d_root
+MATCH (d_root:DiffRoot)
+%(diff_filter)s
+OPTIONAL MATCH (d_root)-[*]->(diff_thing)
+WITH DISTINCT d_root, diff_thing
+ORDER BY elementId(diff_thing)
+CALL (diff_thing) {
+    DETACH DELETE diff_thing
+} IN TRANSACTIONS
+DETACH DELETE d_root
         """ % {"diff_filter": diff_filter}
         self.add_to_query(query=query)

infrahub/core/diff/query/merge.py

@@ -49,11 +49,18 @@ WITH node_diff_map, is_node_kind_migration, CASE
     WHEN is_node_kind_migration THEN $migrated_kinds_id_map[node_diff_map.uuid]
     ELSE NULL
 END AS node_db_id
+
+// ------------------------------
+// find the correct Node if the Node had its kind/inheritance migrated
+// and there are multiple Nodes with the same UUID
+// ------------------------------
 CALL (node_diff_map, node_db_id) {
-    MATCH (n:Node {uuid: node_diff_map.uuid})
-    WHERE node_db_id IS NULL
-    OR %(id_func)s(n) = node_db_id
+    MATCH (n:Node {uuid: node_diff_map.uuid})-[n_is_part_of:IS_PART_OF]->(:Root)
+    WHERE node_db_id IS NULL OR %(id_func)s(n) = node_db_id
+    AND n_is_part_of.branch IN [$source_branch, $target_branch]
     RETURN n
+    ORDER BY n_is_part_of.branch_level DESC, n_is_part_of.from DESC, n_is_part_of.status ASC
+    LIMIT 1
 }
 WITH n, node_diff_map, is_node_kind_migration
 CALL (n, node_diff_map, is_node_kind_migration) {
@@ -224,16 +231,28 @@ CALL (n, node_diff_map, is_node_kind_migration) {
                     ELSE NULL
                 END AS rel_peer_db_id
             // ------------------------------
+            // find the correct relationship peer if the peer had its kind/inheritance migrated
+            // and there are multiple Nodes with the same UUID
+            // ------------------------------
+            CALL (rel_peer_id, rel_peer_db_id) {
+                MATCH (rel_peer:Node {uuid: rel_peer_id})-[target_is_part_of:IS_PART_OF]->(:Root)
+                WHERE (rel_peer_db_id IS NULL OR %(id_func)s(rel_peer) = rel_peer_db_id)
+                AND target_is_part_of.branch IN [$source_branch, $target_branch]
+                RETURN rel_peer
+                ORDER BY target_is_part_of.branch_level DESC, target_is_part_of.from DESC, target_is_part_of.status ASC
+                LIMIT 1
+            }
+            WITH rel_name, related_rel_status, rel_peer
+            // ------------------------------
             // determine the directions of each IS_RELATED
             // ------------------------------
-            CALL (n, rel_name, rel_peer_id, rel_peer_db_id, related_rel_status) {
+            CALL (n, rel_name, rel_peer, related_rel_status) {
                 MATCH (n)
                     -[source_r_rel_1:IS_RELATED]
                     -(r:Relationship {name: rel_name})
                     -[source_r_rel_2:IS_RELATED]
-                    -(rel_peer:Node {uuid: rel_peer_id})
-                WHERE (rel_peer_db_id IS NULL OR %(id_func)s(rel_peer) = rel_peer_db_id)
-                AND source_r_rel_1.branch IN [$source_branch, $target_branch]
+                    -(rel_peer)
+                WHERE source_r_rel_1.branch IN [$source_branch, $target_branch]
                 AND source_r_rel_2.branch IN [$source_branch, $target_branch]
                 AND source_r_rel_1.from <= $at AND source_r_rel_1.to IS NULL
                 AND source_r_rel_2.from <= $at AND source_r_rel_2.to IS NULL
@@ -251,37 +270,34 @@ CALL (n, node_diff_map, is_node_kind_migration) {
                 source_r_rel_1.hierarchy AS r1_hierarchy,
                 source_r_rel_2.hierarchy AS r2_hierarchy
             }
-            WITH n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer_id, rel_peer_db_id, related_rel_status
-            CALL (n, rel_name, rel_peer_id, rel_peer_db_id, related_rel_status) {
+            WITH n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer, related_rel_status
+            CALL (n, rel_name, rel_peer, related_rel_status) {
                 OPTIONAL MATCH (n)
                     -[target_r_rel_1:IS_RELATED {branch: $target_branch, status: "active"}]
                     -(:Relationship {name: rel_name})
                     -[target_r_rel_2:IS_RELATED {branch: $target_branch, status: "active"}]
-                    -(rel_peer:Node {uuid: rel_peer_id})
+                    -(rel_peer)
                 WHERE related_rel_status = "deleted"
-                AND (rel_peer_db_id IS NULL OR %(id_func)s(rel_peer) = rel_peer_db_id)
                 AND target_r_rel_1.from <= $at AND target_r_rel_1.to IS NULL
                 AND target_r_rel_2.from <= $at AND target_r_rel_2.to IS NULL
                 SET target_r_rel_1.to = $at
                 SET target_r_rel_2.to = $at
             }
-            WITH n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer_id, rel_peer_db_id, related_rel_status
+            WITH n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer, related_rel_status
             // ------------------------------
             // conditionally create new IS_RELATED relationships on target_branch, if necessary
             // ------------------------------
-            CALL (n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer_id, rel_peer_db_id, related_rel_status) {
-                MATCH (p:Node {uuid: rel_peer_id})
-                WHERE rel_peer_db_id IS NULL OR %(id_func)s(p) = rel_peer_db_id
+            CALL (n, r, r1_dir, r2_dir, r1_hierarchy, r2_hierarchy, rel_name, rel_peer, related_rel_status) {
                 OPTIONAL MATCH (n)
                     -[r_rel_1:IS_RELATED {branch: $target_branch, status: related_rel_status}]
                     -(:Relationship {name: rel_name})
                     -[r_rel_2:IS_RELATED {branch: $target_branch, status: related_rel_status}]
-                    -(p)
+                    -(rel_peer)
                 WHERE r_rel_1.from <= $at
                 AND (r_rel_1.to >= $at OR r_rel_1.to IS NULL)
                 AND r_rel_2.from <= $at
                 AND (r_rel_2.to >= $at OR r_rel_2.to IS NULL)
-                WITH p, r_rel_1, r_rel_2
+                WITH rel_peer, r_rel_1, r_rel_2
                 WHERE r_rel_1 IS NULL
                 AND r_rel_2 IS NULL
                 // ------------------------------
@@ -301,19 +317,19 @@ CALL (n, node_diff_map, is_node_kind_migration) {
                         <-[:IS_RELATED {branch: $target_branch, branch_level: $branch_level, from: $at, status: related_rel_status, hierarchy: r1_hierarchy}]
                         -(r)
                 }
-                CALL (r, p, r2_dir, r2_hierarchy, related_rel_status) {
-                    WITH r, p, r2_dir, r2_hierarchy, related_rel_status
+                CALL (r, rel_peer, r2_dir, r2_hierarchy, related_rel_status) {
+                    WITH r, rel_peer, r2_dir, r2_hierarchy, related_rel_status
                     WHERE r2_dir = "r"
                     CREATE (r)
                         -[:IS_RELATED {branch: $target_branch, branch_level: $branch_level, from: $at, status: related_rel_status, hierarchy: r2_hierarchy}]
-                        ->(p)
+                        ->(rel_peer)
                 }
-                CALL (r, p, r2_dir, r2_hierarchy, related_rel_status) {
-                    WITH r, p, r2_dir, r2_hierarchy, related_rel_status
+                CALL (r, rel_peer, r2_dir, r2_hierarchy, related_rel_status) {
+                    WITH r, rel_peer, r2_dir, r2_hierarchy, related_rel_status
                     WHERE r2_dir = "l"
                     CREATE (r)
                         <-[:IS_RELATED {branch: $target_branch, branch_level: $branch_level, from: $at, status: related_rel_status, hierarchy: r2_hierarchy}]
-                        -(p)
+                        -(rel_peer)
                 }
             }
         }

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 39
+GRAPH_VERSION = 40

infrahub/core/migrations/graph/__init__.py

@@ -41,6 +41,8 @@ from .m036_drop_attr_value_index import Migration036
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
+from .m040_duplicated_attributes import Migration040
+from .m041_deleted_dup_edges import Migration041
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -87,6 +89,8 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration037,
     Migration038,
     Migration039,
+    Migration040,
+    Migration041,
 ]
 
 

infrahub/core/migrations/graph/m040_duplicated_attributes.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class DeleteDuplicatedAttributesQuery(Query):
+    name: str = "delete_duplicated_attributes"
+    type: QueryType = QueryType.WRITE
+    insert_return: bool = False
+    insert_limit: bool = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// -------------
+// get all the Nodes linked to multiple Attributes with the same name to drastically reduce the search space
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(attr:Attribute)
+WITH DISTINCT n, attr
+WITH n, attr.name AS attr_name, count(*) AS num_attrs
+WHERE num_attrs > 1
+// -------------
+// for each Node-attr_name pair, get the possible duplicate Attributes
+// -------------
+MATCH (n)-[:HAS_ATTRIBUTE]->(dup_attr:Attribute {name: attr_name})
+WITH DISTINCT n, dup_attr
+// -------------
+// get the branch(es) for each possible duplicate Attribute
+// -------------
+CALL (n, dup_attr) {
+    MATCH (n)-[r:HAS_ATTRIBUTE {status: "active"}]->(dup_attr)
+    WHERE r.to IS NULL
+    AND NOT exists((n)-[:HAS_ATTRIBUTE {status: "deleted", branch: r.branch}]->(dup_attr))
+    RETURN r.branch AS branch
+}
+// -------------
+// get the latest update time for each duplicate Attribute on each branch
+// -------------
+CALL (dup_attr, branch) {
+    MATCH (dup_attr)-[r {branch: branch}]-()
+    RETURN max(r.from) AS latest_update
+}
+// -------------
+// order the duplicate Attributes by latest update time
+// -------------
+WITH n, dup_attr, branch, latest_update
+ORDER BY n, branch, dup_attr.name, latest_update DESC
+// -------------
+// for any Node-dup_attr_name pairs with multiple duplicate Attributes, keep the Attribute with the latest update
+// on this branch and delete all the other edges on this branch for this Attribute
+// -------------
+WITH n, branch, dup_attr.name AS dup_attr_name, collect(dup_attr) AS dup_attrs_reverse_chronological
+WHERE size(dup_attrs_reverse_chronological) > 1
+WITH branch, tail(dup_attrs_reverse_chronological) AS dup_attrs_to_delete
+UNWIND dup_attrs_to_delete AS dup_attr_to_delete
+MATCH (dup_attr_to_delete)-[r {branch: branch}]-()
+DELETE r
+// -------------
+// delete any orphaned Attributes
+// -------------
+WITH DISTINCT dup_attr_to_delete
+WHERE NOT exists((dup_attr_to_delete)--())
+DELETE dup_attr_to_delete
+        """
+        self.add_to_query(query)
+
+
+class Migration040(GraphMigration):
+    name: str = "040_duplicated_attributes"
+    queries: Sequence[type[Query]] = [DeleteDuplicatedAttributesQuery]
+    minimum_version: int = 39
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()

infrahub/core/migrations/graph/m041_deleted_dup_edges.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich import print as rprint
+
+from infrahub.core.branch import Branch
+from infrahub.core.diff.repository.repository import DiffRepository
+from infrahub.core.initialization import get_root_node
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+from infrahub.dependencies.registry import build_component_registry, get_component_registry
+from infrahub.log import get_logger
+
+from ..shared import ArbitraryMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class DeletePosthumousEdges(Query):
+    name = "delete_posthumous_edges_query"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+        query = """
+// ------------
+// find deleted nodes
+// ------------
+MATCH (n:Node)-[e:IS_PART_OF]->(:Root)
+WHERE e.status = "deleted" OR e.to IS NOT NULL
+WITH DISTINCT n, e.branch AS delete_branch, e.branch_level AS delete_branch_level, CASE
+    WHEN e.status = "deleted" THEN e.from
+    ELSE e.to
+END AS delete_time
+// ------------
+// find the edges added to the deleted node after the delete time
+// ------------
+MATCH (n)-[added_e]-(peer)
+WHERE added_e.from > delete_time
+AND type(added_e) <> "IS_PART_OF"
+// if the node was deleted on a branch (delete_branch_level > 1), and then updated on main/global (added_e.branch_level = 1), we can ignore it
+AND added_e.branch_level >= delete_branch_level
+AND (added_e.branch = delete_branch OR delete_branch_level = 1)
+WITH DISTINCT n, delete_branch, delete_time, added_e, peer
+// ------------
+// get the branched_from for the branch on which the node was deleted
+// ------------
+CALL (added_e) {
+    MATCH (b:Branch {name: added_e.branch})
+    RETURN b.branched_from AS added_e_branched_from
+}
+// ------------
+// account for the following situations, given that the edge update time is after the node delete time
+//  - deleted on main/global, updated on branch
+//    - illegal if the delete is before branch.branched_from
+//  - deleted on branch, updated on branch
+//    - illegal
+// ------------
+WITH n, delete_branch, delete_time, added_e, peer
+WHERE delete_branch = added_e.branch
+OR delete_time < added_e_branched_from
+DELETE added_e
+        """
+        self.add_to_query(query)
+
+
+class DeleteDuplicateEdgesForMigratedKindNodes(Query):
+    name = "delete_duplicate_edges_for_migrated_kind_nodes_query"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+        query = """
+// ------------
+// get UUIDs for migrated kind/inheritance nodes
+// ------------
+MATCH (n:Node)
+WITH n.uuid AS node_uuid, count(*) AS num_nodes_with_uuid
+WHERE num_nodes_with_uuid > 1
+CALL (node_uuid) {
+    // ------------
+    // find any Relationships for these nodes
+    // ------------
+    MATCH (n:Node {uuid: node_uuid})-[:IS_RELATED]-(rel:Relationship)
+    WITH DISTINCT rel
+    MATCH (rel)-[e]->(peer)
+    WITH
+        type(e) AS e_type,
+        e.branch AS e_branch,
+        e.from AS e_from,
+        e.to AS e_to,
+        e.status AS e_status,
+        e.peer AS e_peer,
+        CASE
+            WHEN startNode(e) = rel THEN "out" ELSE "in"
+        END AS direction,
+        collect(e) AS duplicate_edges
+    WHERE size(duplicate_edges) > 1
+    WITH tail(duplicate_edges) AS duplicate_edges_to_delete
+    UNWIND duplicate_edges_to_delete AS edge_to_delete
+    DELETE edge_to_delete
+} IN TRANSACTIONS OF 500 ROWS
+        """
+        self.add_to_query(query)
+
+
+class Migration041(ArbitraryMigration):
+    """Clean up improper merges that duplicated edges to nodes with migrated kinds
+
+    - delete all existing diffs b/c they could contain incorrect nodes linking to deleted nodes with migrated kind/inheritance
+    - delete all edges added to any nodes AFTER they were deleted on main
+    - delete any duplicate edges touching migrated kind/inheritance nodes on main
+    """
+
+    name: str = "041_deleted_dup_edges"
+    minimum_version: int = 40
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        result = MigrationResult()
+
+        return result
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        root_node = await get_root_node(db=db)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+
+        rprint("Deleting all diffs", end="...")
+        build_component_registry()
+        component_registry = get_component_registry()
+        diff_repo = await component_registry.get_component(DiffRepository, db=db, branch=default_branch)
+        await diff_repo.delete_all_diff_roots()
+        rprint("done")
+
+        rprint("Deleting edges merged after node deleted", end="...")
+        delete_posthumous_edges_query = await DeletePosthumousEdges.init(db=db)
+        await delete_posthumous_edges_query.execute(db=db)
+        rprint("done")
+
+        rprint("Deleting duplicate edges for migrated kind/inheritance nodes", end="...")
+        delete_duplicate_edges_query = await DeleteDuplicateEdgesForMigratedKindNodes.init(db=db)
+        await delete_duplicate_edges_query.execute(db=db)
+        rprint("done")
+
+        return MigrationResult()

infrahub/core/migrations/schema/node_attribute_add.py

@@ -42,6 +42,16 @@ class NodeAttributeAddMigration(AttributeSchemaMigration):
     name: str = "node.attribute.add"
     queries: Sequence[type[AttributeMigrationQuery]] = [NodeAttributeAddMigrationQuery01]  # type: ignore[assignment]
 
+    async def execute(
+        self,
+        db: InfrahubDatabase,
+        branch: Branch,
+        at: Timestamp | str | None = None,
+    ) -> MigrationResult:
+        if self.new_attribute_schema.inherited is True:
+            return MigrationResult()
+        return await super().execute(db=db, branch=branch, at=at)
+
     async def execute_post_queries(
         self,
         db: InfrahubDatabase,

infrahub/core/node/create.py

@@ -57,14 +57,25 @@ async def extract_peer_data(
 
     for rel in template_peer.get_schema().relationship_names:
         rel_manager: RelationshipManager = getattr(template_peer, rel)
-        if (
-            rel_manager.schema.kind not in [RelationshipKind.COMPONENT, RelationshipKind.PARENT]
-            or rel_manager.schema.name not in obj_peer_schema.relationship_names
-        ):
+
+        if rel_manager.schema.name not in obj_peer_schema.relationship_names:
             continue
 
-        if list(await rel_manager.get_peers(db=db)) == [current_template.id]:
+        peers_map = await rel_manager.get_peers(db=db)
+        if rel_manager.schema.kind in [RelationshipKind.COMPONENT, RelationshipKind.PARENT] and list(
+            peers_map.keys()
+        ) == [current_template.id]:
             obj_peer_data[rel] = {"id": parent_obj.id}
+            continue
+
+        rel_peer_ids = []
+        for peer_id, peer_object in peers_map.items():
+            # deeper templates are handled in the next level of recursion
+            if peer_object.get_schema().is_template_schema:
+                continue
+            rel_peer_ids.append({"id": peer_id})
+
+        obj_peer_data[rel] = rel_peer_ids
 
     return obj_peer_data
 

infrahub/core/query/diff.py

@@ -301,13 +301,15 @@ WITH p, q, diff_rel, CASE
     ELSE $from_time
 END AS row_from_time
 ORDER BY %(id_func)s(p) DESC
-SKIP $offset
-LIMIT $limit
+SKIP toInteger($offset)
+LIMIT toInteger($limit)
 // -------------------------------------
 // Add flag to indicate if there is more data after this
 // -------------------------------------
 WITH collect([p, q, diff_rel, row_from_time]) AS limited_results
-WITH limited_results, size(limited_results) = $limit AS has_more_data
+// extra NULL row ensures that has_more_data is always returned, even if all results are filtered out below
+WITH limited_results + [[NULL, NULL, NULL, NULL]] AS limited_results
+WITH limited_results, size(limited_results) = ($limit + 1) AS has_more_data
 UNWIND limited_results AS one_result
 WITH one_result[0] AS p, one_result[1] AS q, one_result[2] AS diff_rel, one_result[3] AS row_from_time, has_more_data
 // -------------------------------------
@@ -470,14 +472,16 @@ AND (
 // Limit the number of paths
 // -------------------------------------
 WITH root, r_root, p, diff_rel, q
-ORDER BY r_root.from, p.uuid, q.uuid, diff_rel.branch, diff_rel.from
-SKIP $offset
-LIMIT $limit
+ORDER BY r_root.from, p.uuid, q.uuid, q.name, diff_rel.branch, diff_rel.from
+SKIP toInteger($offset)
+LIMIT toInteger($limit)
 // -------------------------------------
 // Add flag to indicate if there is more data after this
 // -------------------------------------
 WITH collect([root, r_root, p, diff_rel, q]) AS limited_results
-WITH limited_results, size(limited_results) = $limit AS has_more_data
+// extra NULL row ensures that has_more_data is always returned, even if all results are filtered out below
+WITH limited_results + [[NULL, NULL, NULL, NULL, NULL]] AS limited_results
+WITH limited_results, size(limited_results) = ($limit + 1) AS has_more_data
 UNWIND limited_results AS one_result
 WITH one_result[0] AS root, one_result[1] AS r_root, one_result[2] AS p, one_result[3] AS diff_rel, one_result[4] AS q, has_more_data
 // -------------------------------------
@@ -641,8 +645,28 @@ AND (
         )
         // skip paths where nodes/attrs/rels are updated after $from_time, those are handled in other queries
         AND (
-            r_root.from <= $from_time AND (r_root.to IS NULL OR r_root.branch <> diff_rel.branch OR r_root.to <= $from_time)
-            AND r_node.from <= $from_time AND (r_node.to IS NULL OR r_node.branch <> diff_rel.branch OR r_node.to <= $from_time)
+            (
+                r_root.branch = diff_rel.branch
+                AND r_root.from <= $from_time
+                AND (r_root.to IS NULL OR r_root.to >= $to_time)
+            )
+            OR (
+                r_root.branch <> diff_rel.branch
+                AND r_root.from <= $from_time
+                AND (r_root.to IS NULL OR r_root.to >= $branch_from_time)
+            )
+        )
+        AND (
+            (
+                r_node.branch = diff_rel.branch
+                AND r_node.from <= $from_time
+                AND (r_node.to IS NULL OR r_node.to >= $to_time)
+            )
+            OR (
+                r_node.branch <> diff_rel.branch
+                AND r_node.from <= $from_time
+                AND (r_node.to IS NULL OR r_node.to >= $branch_from_time)
+            )
         )
     )
     // time-based filters for new nodes
@@ -658,8 +682,27 @@ AND (
         )
         // skip paths where nodes/attrs/rels are updated after $branch_from_time, those are handled in other queries
         AND (
-            r_root.from <= $branch_from_time AND (r_root.to IS NULL OR r_root.branch <> diff_rel.branch OR r_root.to <= $branch_from_time)
-            AND r_node.from <= $branch_from_time AND (r_node.to IS NULL OR r_node.branch <> diff_rel.branch OR r_node.to <= $branch_from_time)
+            (
+                r_root.branch = diff_rel.branch
+                AND (r_root.to IS NULL OR r_root.to >= $to_time)
+            )
+            OR (
+                r_root.branch <> diff_rel.branch
+                AND r_root.from <= $branch_from_time
+                AND (r_root.to IS NULL OR r_root.to >= $branch_from_time)
+            )
+        )
+        AND (
+            (
+                r_node.branch = diff_rel.branch
+                AND r_node.from <= $branch_from_time
+                AND (r_node.to IS NULL OR r_node.to >= $to_time)
+            )
+            OR (
+                r_node.branch <> diff_rel.branch
+                AND r_node.from <= $branch_from_time
+                AND (r_node.to IS NULL OR r_node.to >= $branch_from_time)
+            )
         )
     )
 )
@@ -701,13 +744,15 @@ AND [%(id_func)s(n), type(r_node)] <> [%(id_func)s(q), type(diff_rel)]
 // -------------------------------------
 WITH diff_rel_path, r_root, n, r_node, p, diff_rel
 ORDER BY r_root.from, n.uuid, p.uuid, type(diff_rel), diff_rel.branch, diff_rel.from
-SKIP $offset
-LIMIT $limit
+SKIP toInteger($offset)
+LIMIT toInteger($limit)
 // -------------------------------------
 // Add flag to indicate if there is more data after this
 // -------------------------------------
 WITH collect([diff_rel_path, r_root, n, r_node, p, diff_rel]) AS limited_results
-WITH limited_results, size(limited_results) = $limit AS has_more_data
+// extra NULL row ensures that has_more_data is always returned, even if all results are filtered out below
+WITH limited_results + [[NULL, NULL, NULL, NULL, NULL, NULL]] AS limited_results
+WITH limited_results, size(limited_results) = ($limit + 1) AS has_more_data
 UNWIND limited_results AS one_result
 WITH one_result[0] AS diff_rel_path, one_result[1] AS r_root, one_result[2] AS n,
     one_result[3] AS r_node, one_result[4] AS p, one_result[5] AS diff_rel, has_more_data
@@ -803,8 +848,8 @@ WHERE num_nodes_with_uuid > 1
 // -------------------------------------
 WITH node_uuid
 ORDER BY node_uuid
-SKIP $offset
-LIMIT $limit
+SKIP toInteger($offset)
+LIMIT toInteger($limit)
 WITH collect(node_uuid) AS node_uuids
 WITH node_uuids, size(node_uuids) = $limit AS has_more_data
 MATCH (:Root)<-[diff_rel:IS_PART_OF {branch: $branch_name}]-(n:Node)