nidx-binding 6.3.0.post10__tar.gz → 6.3.0.post16__tar.gz

nidx_binding-6.3.0.post16/.sqlx/query-77a245aed9f6f8b9fc643efbd4b1e2e8e99bbba085e3e75a4f29321cd7b2a25c.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            -- Only gather deletion information for indexes with many deletions since it's expensive and often not needed\n            WITH indexes_with_many_deletions AS (\n                SELECT index_id FROM deletions GROUP BY index_id HAVING COUNT(*) > $2\n\n            -- Deletions that apply to each segment (so we know which segments we need to merge to allow purging deletions)\n            ), deletions_per_segment AS (\n                SELECT\n                    segments.id,\n                    (\n                        SELECT COUNT(*)\n                        FROM deletions\n                        WHERE deletions.index_id = segments.index_id\n                        AND deletions.seq > segments.seq\n                    ) AS deletions\n                FROM segments\n                NATURAL JOIN indexes_with_many_deletions\n                GROUP BY segments.id\n            )\n\n            -- For each index, a list of mergeable segments with amount of records and deletions\n            SELECT\n                indexes.id AS \"id: IndexId\",\n                indexes.kind AS \"kind: IndexKind\",\n                array_agg(\n                    (segments.id, records, COALESCE(deletions, 0) > $2)\n                    ORDER BY records DESC\n                ) AS \"segments!: Vec<(SegmentId, i64, bool)>\"\n            FROM segments\n            JOIN indexes ON segments.index_id = indexes.id\n            LEFT JOIN deletions_per_segment ON segments.id = deletions_per_segment.id\n            WHERE\n                delete_at IS NULL AND merge_job_id IS NULL\n                AND seq <= $1\n            GROUP BY\n                indexes.id,\n                indexes.kind,\n                -- Only merge vector segments with same tags\n                CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id: IndexId",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "kind: IndexKind",
+        "type_info": {
+          "Custom": {
+            "name": "index_kind",
+            "kind": {
+              "Enum": [
+                "text",
+                "paragraph",
+                "vector",
+                "relation"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "segments!: Vec<(SegmentId, i64, bool)>",
+        "type_info": "RecordArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "77a245aed9f6f8b9fc643efbd4b1e2e8e99bbba085e3e75a4f29321cd7b2a25c"
+}

nidx_binding-6.3.0.post10/.sqlx/query-e61329fbb184d48a81fbe18bb66b8f1947724b77bada24ebd56b0fbb800b2c7e.json → nidx_binding-6.3.0.post16/.sqlx/query-abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT COALESCE(MIN(seq) - 1, (SELECT last_value FROM index_requests_seq_seq)) AS \"seq!\" FROM index_requests",
+  "query": "SELECT COALESCE(MIN(seq) - 1, (SELECT last_value FROM index_requests_seq_seq)) AS \"seq!\"\n             FROM index_requests WHERE received_at > NOW() - INTERVAL '1 minute'",
   "describe": {
     "columns": [
       {
@@ -16,5 +16,5 @@
       null
     ]
   },
-  "hash": "e61329fbb184d48a81fbe18bb66b8f1947724b77bada24ebd56b0fbb800b2c7e"
+  "hash": "abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2"
 }

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.0.post10
+Version: 6.3.0.post16
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/nidx_binding/src/lib.rs

@@ -49,6 +49,10 @@ impl GetAckFloor for SeqSource {
         let seq = self.0.load(std::sync::atomic::Ordering::Relaxed);
         Ok(seq)
     }
+
+    async fn cleanup(&self) -> anyhow::Result<()> {
+        Ok(())
+    }
 }
 
 #[pyclass]

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.0.post10"
+version = "6.3.0.post16"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.0.post10"
+version = "6.3.0.post16"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/src/metadata/index_request.rs

@@ -43,7 +43,7 @@ impl IndexRequest {
         Ok(())
     }
 
-    pub async fn delete_old(&self, meta: impl Executor<'_, Database = Postgres>) -> sqlx::Result<()> {
+    pub async fn delete_old(meta: impl Executor<'_, Database = Postgres>) -> sqlx::Result<()> {
         sqlx::query!("DELETE FROM index_requests WHERE received_at < NOW() - INTERVAL '1 minute'")
             .execute(meta)
             .await?;
@@ -52,7 +52,8 @@ impl IndexRequest {
 
     pub async fn last_ack_seq(meta: impl Executor<'_, Database = Postgres>) -> sqlx::Result<i64> {
         sqlx::query_scalar!(
-            "SELECT COALESCE(MIN(seq) - 1, (SELECT last_value FROM index_requests_seq_seq)) AS \"seq!\" FROM index_requests"
+            "SELECT COALESCE(MIN(seq) - 1, (SELECT last_value FROM index_requests_seq_seq)) AS \"seq!\"
+             FROM index_requests WHERE received_at > NOW() - INTERVAL '1 minute'"
         )
         .fetch_one(meta)
         .await

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/src/scheduler/merge_task.rs

@@ -64,10 +64,16 @@ impl MergeScheduler {
 
             -- Deletions that apply to each segment (so we know which segments we need to merge to allow purging deletions)
             ), deletions_per_segment AS (
-                SELECT segments.id, COUNT(deletions) AS deletions
+                SELECT
+                    segments.id,
+                    (
+                        SELECT COUNT(*)
+                        FROM deletions
+                        WHERE deletions.index_id = segments.index_id
+                        AND deletions.seq > segments.seq
+                    ) AS deletions
                 FROM segments
                 NATURAL JOIN indexes_with_many_deletions
-                LEFT JOIN deletions ON segments.index_id = deletions.index_id AND segments.seq < deletions.seq
                 GROUP BY segments.id
             )
 

{nidx_binding-6.3.0.post10 → nidx_binding-6.3.0.post16}/src/scheduler.rs

@@ -68,6 +68,7 @@ pub async fn run(settings: Settings, shutdown: CancellationToken) -> anyhow::Res
 
 pub trait GetAckFloor {
     fn get(&mut self) -> impl Future<Output = anyhow::Result<i64>> + Send;
+    fn cleanup(&self) -> impl Future<Output = anyhow::Result<()>> + Send;
 }
 
 #[derive(Clone)]
@@ -77,6 +78,10 @@ impl GetAckFloor for NatsAckFloor {
     async fn get(&mut self) -> anyhow::Result<i64> {
         Ok(self.0.info().await?.ack_floor.stream_sequence as i64)
     }
+
+    async fn cleanup(&self) -> anyhow::Result<()> {
+        Ok(())
+    }
 }
 
 #[derive(Clone)]
@@ -86,6 +91,11 @@ impl GetAckFloor for PgAckFloor {
     async fn get(&mut self) -> anyhow::Result<i64> {
         Ok(IndexRequest::last_ack_seq(&self.0.pool).await?)
     }
+
+    async fn cleanup(&self) -> anyhow::Result<()> {
+        IndexRequest::delete_old(&self.0.pool).await?;
+        Ok(())
+    }
 }
 
 pub async fn run_tasks(
@@ -127,6 +137,16 @@ pub async fn run_tasks(
         }
     });
 
+    let ack_floor_copy = ack_floor.clone();
+    tasks.spawn(async move {
+        loop {
+            if let Err(e) = ack_floor_copy.cleanup().await {
+                warn!("Error cleaning up AckFloor: {e:?}");
+            }
+            sleep(Duration::from_secs(60)).await;
+        }
+    });
+
     let meta2 = meta.clone();
     let mut ack_floor_copy = ack_floor.clone();
     tasks.spawn(async move {

nidx_binding-6.3.0.post10/.sqlx/query-ca7618102d184f9734ba876b0ade0bbc47d2d99747d9df380b120680d77c3b83.json

@@ -1,47 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            -- Only gather deletion information for indexes with many deletions since it's expensive and often not needed\n            WITH indexes_with_many_deletions AS (\n                SELECT index_id FROM deletions GROUP BY index_id HAVING COUNT(*) > $2\n\n            -- Deletions that apply to each segment (so we know which segments we need to merge to allow purging deletions)\n            ), deletions_per_segment AS (\n                SELECT segments.id, COUNT(deletions) AS deletions\n                FROM segments\n                NATURAL JOIN indexes_with_many_deletions\n                LEFT JOIN deletions ON segments.index_id = deletions.index_id AND segments.seq < deletions.seq\n                GROUP BY segments.id\n            )\n\n            -- For each index, a list of mergeable segments with amount of records and deletions\n            SELECT\n                indexes.id AS \"id: IndexId\",\n                indexes.kind AS \"kind: IndexKind\",\n                array_agg(\n                    (segments.id, records, COALESCE(deletions, 0) > $2)\n                    ORDER BY records DESC\n                ) AS \"segments!: Vec<(SegmentId, i64, bool)>\"\n            FROM segments\n            JOIN indexes ON segments.index_id = indexes.id\n            LEFT JOIN deletions_per_segment ON segments.id = deletions_per_segment.id\n            WHERE\n                delete_at IS NULL AND merge_job_id IS NULL\n                AND seq <= $1\n            GROUP BY\n                indexes.id,\n                indexes.kind,\n                -- Only merge vector segments with same tags\n                CASE WHEN kind = 'vector' THEN index_metadata::text ELSE NULL END\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id: IndexId",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "kind: IndexKind",
-        "type_info": {
-          "Custom": {
-            "name": "index_kind",
-            "kind": {
-              "Enum": [
-                "text",
-                "paragraph",
-                "vector",
-                "relation"
-              ]
-            }
-          }
-        }
-      },
-      {
-        "ordinal": 2,
-        "name": "segments!: Vec<(SegmentId, i64, bool)>",
-        "type_info": "RecordArray"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8",
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false,
-      false,
-      null
-    ]
-  },
-  "hash": "ca7618102d184f9734ba876b0ade0bbc47d2d99747d9df380b120680d77c3b83"
-}