nidx-binding 6.3.6.post315__tar.gz → 6.3.7.post320__tar.gz

nidx_binding-6.3.7.post320/.sqlx/query-a60ec2f66f1e7b84189e5b089f2087a29ff6a64326a3743dea935bbc58ee77fa.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                -- For each index, gather the seq number before which we want to force deletion of segments\n                -- e.g: gather the seq of the 100th newest deletion, and force merge all segments older than that\n                WITH index_deletion_window AS (\n                    SELECT\n                        id,\n                        COALESCE(\n                            (\n                                -- Biggest seq excluding the N most recent\n                                SELECT MIN(seq) FROM (\n                                    SELECT seq FROM deletions WHERE index_id = indexes.id ORDER BY seq DESC LIMIT $2)\n                                    AS new_deletions\n                                ),\n                            0\n                        ) AS seq\n                    FROM indexes\n                    JOIN deletions ON index_id = indexes.id\n                    GROUP BY indexes.id\n                    HAVING COUNT(*) > $2\n                )\n                -- For each index, a list of mergeable segments with amount of records and deletions\n                SELECT\n                    indexes.id AS \"id: IndexId\",\n                    indexes.kind AS \"kind: IndexKind\",\n                    array_agg(\n                        -- segment_id, records, force_merge (has more than N deletions)\n                        (segments.id, records, segments.seq < COALESCE(index_deletion_window.seq, 0))\n                        ORDER BY records DESC\n                    ) AS \"segments!: Vec<(SegmentId, i64, bool)>\"\n                FROM segments\n                JOIN indexes ON segments.index_id = indexes.id\n                LEFT JOIN index_deletion_window ON segments.index_id = index_deletion_window.id\n                WHERE\n                    delete_at IS NULL AND merge_job_id IS NULL\n                    AND segments.seq <= $1\n                GROUP BY\n                    indexes.id,\n                    indexes.kind,\n                    -- Only merge vector segments with same tags\n                    CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END,\n                    index_deletion_window.id\n                -- Only return indexes with more than one segment or with many deletions\n                HAVING COUNT(segments) > 1 OR index_deletion_window.id IS NOT NULL\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id: IndexId",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "kind: IndexKind",
+        "type_info": {
+          "Custom": {
+            "name": "index_kind",
+            "kind": {
+              "Enum": [
+                "text",
+                "paragraph",
+                "vector",
+                "relation"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "segments!: Vec<(SegmentId, i64, bool)>",
+        "type_info": "RecordArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "a60ec2f66f1e7b84189e5b089f2087a29ff6a64326a3743dea935bbc58ee77fa"
+}

{nidx_binding-6.3.6.post315 → nidx_binding-6.3.7.post320}/Cargo.lock

@@ -559,9 +559,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.14"
+version = "0.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
 dependencies = [
  "crossbeam-utils",
 ]

{nidx_binding-6.3.6.post315 → nidx_binding-6.3.7.post320}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.6.post315
+Version: 6.3.7.post320
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.6.post315 → nidx_binding-6.3.7.post320}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.6.post315"
+version = "6.3.7.post320"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.6.post315 → nidx_binding-6.3.7.post320}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.6.post315"
+version = "6.3.7.post320"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.6.post315 → nidx_binding-6.3.7.post320}/src/scheduler/merge_task.rs

@@ -56,44 +56,48 @@ impl MergeScheduler {
         // TODO: Remove vector index split by metadata when removing tags
         let indexes = sqlx::query!(
             r#"
-            -- Only gather deletion information for indexes with many deletions since it's expensive and often not needed
-            WITH indexes_with_many_deletions AS (
-                SELECT index_id FROM deletions GROUP BY index_id HAVING COUNT(*) > $2
-
-            -- Deletions that apply to each segment (so we know which segments we need to merge to allow purging deletions)
-            ), deletions_per_segment AS (
+                -- For each index, gather the seq number before which we want to force deletion of segments
+                -- e.g: gather the seq of the 100th newest deletion, and force merge all segments older than that
+                WITH index_deletion_window AS (
+                    SELECT
+                        id,
+                        COALESCE(
+                            (
+                                -- Biggest seq excluding the N most recent
+                                SELECT MIN(seq) FROM (
+                                    SELECT seq FROM deletions WHERE index_id = indexes.id ORDER BY seq DESC LIMIT $2)
+                                    AS new_deletions
+                                ),
+                            0
+                        ) AS seq
+                    FROM indexes
+                    JOIN deletions ON index_id = indexes.id
+                    GROUP BY indexes.id
+                    HAVING COUNT(*) > $2
+                )
+                -- For each index, a list of mergeable segments with amount of records and deletions
                 SELECT
-                    segments.id,
-                    (
-                        SELECT COUNT(*)
-                        FROM deletions
-                        WHERE deletions.index_id = segments.index_id
-                        AND deletions.seq > segments.seq
-                    ) AS deletions
+                    indexes.id AS "id: IndexId",
+                    indexes.kind AS "kind: IndexKind",
+                    array_agg(
+                        -- segment_id, records, force_merge (has more than N deletions)
+                        (segments.id, records, segments.seq < COALESCE(index_deletion_window.seq, 0))
+                        ORDER BY records DESC
+                    ) AS "segments!: Vec<(SegmentId, i64, bool)>"
                 FROM segments
-                NATURAL JOIN indexes_with_many_deletions
-                GROUP BY segments.id
-            )
-
-            -- For each index, a list of mergeable segments with amount of records and deletions
-            SELECT
-                indexes.id AS "id: IndexId",
-                indexes.kind AS "kind: IndexKind",
-                array_agg(
-                    (segments.id, records, COALESCE(deletions, 0) > $2)
-                    ORDER BY records DESC
-                ) AS "segments!: Vec<(SegmentId, i64, bool)>"
-            FROM segments
-            JOIN indexes ON segments.index_id = indexes.id
-            LEFT JOIN deletions_per_segment ON segments.id = deletions_per_segment.id
-            WHERE
-                delete_at IS NULL AND merge_job_id IS NULL
-                AND seq <= $1
-            GROUP BY
-                indexes.id,
-                indexes.kind,
-                -- Only merge vector segments with same tags
-                CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END
+                JOIN indexes ON segments.index_id = indexes.id
+                LEFT JOIN index_deletion_window ON segments.index_id = index_deletion_window.id
+                WHERE
+                    delete_at IS NULL AND merge_job_id IS NULL
+                    AND segments.seq <= $1
+                GROUP BY
+                    indexes.id,
+                    indexes.kind,
+                    -- Only merge vector segments with same tags
+                    CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END,
+                    index_deletion_window.id
+                -- Only return indexes with more than one segment or with many deletions
+                HAVING COUNT(segments) > 1 OR index_deletion_window.id IS NOT NULL
             "#,
             i64::from(last_indexed_seq),
             self.settings.max_deletions as i64

nidx_binding-6.3.6.post315/.sqlx/query-77a245aed9f6f8b9fc643efbd4b1e2e8e99bbba085e3e75a4f29321cd7b2a25c.json

@@ -1,47 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            -- Only gather deletion information for indexes with many deletions since it's expensive and often not needed\n            WITH indexes_with_many_deletions AS (\n                SELECT index_id FROM deletions GROUP BY index_id HAVING COUNT(*) > $2\n\n            -- Deletions that apply to each segment (so we know which segments we need to merge to allow purging deletions)\n            ), deletions_per_segment AS (\n                SELECT\n                    segments.id,\n                    (\n                        SELECT COUNT(*)\n                        FROM deletions\n                        WHERE deletions.index_id = segments.index_id\n                        AND deletions.seq > segments.seq\n                    ) AS deletions\n                FROM segments\n                NATURAL JOIN indexes_with_many_deletions\n                GROUP BY segments.id\n            )\n\n            -- For each index, a list of mergeable segments with amount of records and deletions\n            SELECT\n                indexes.id AS \"id: IndexId\",\n                indexes.kind AS \"kind: IndexKind\",\n                array_agg(\n                    (segments.id, records, COALESCE(deletions, 0) > $2)\n                    ORDER BY records DESC\n                ) AS \"segments!: Vec<(SegmentId, i64, bool)>\"\n            FROM segments\n            JOIN indexes ON segments.index_id = indexes.id\n            LEFT JOIN deletions_per_segment ON segments.id = deletions_per_segment.id\n            WHERE\n                delete_at IS NULL AND merge_job_id IS NULL\n                AND seq <= $1\n            GROUP BY\n                indexes.id,\n                indexes.kind,\n                -- Only merge vector segments with same tags\n                CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id: IndexId",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "kind: IndexKind",
-        "type_info": {
-          "Custom": {
-            "name": "index_kind",
-            "kind": {
-              "Enum": [
-                "text",
-                "paragraph",
-                "vector",
-                "relation"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "segments!: Vec<(SegmentId, i64, bool)>",
-        "type_info": "RecordArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      null
-    ]
-  },
-  "hash": "77a245aed9f6f8b9fc643efbd4b1e2e8e99bbba085e3e75a4f29321cd7b2a25c"
-}